You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/05/22 20:48:05 UTC
[01/50] hadoop git commit: HADOOP-14361. Azure:
NativeAzureFileSystem.getDelegationToken() call fails sometimes when invoked
concurrently. Contributed by Santhosh G Nayak [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-5734 caa6a7a4b -> 8d28ca16d (forced update)
HADOOP-14361. Azure: NativeAzureFileSystem.getDelegationToken() call fails sometimes when invoked concurrently. Contributed by Santhosh G Nayak
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eed73149
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eed73149
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eed73149
Branch: refs/heads/YARN-5734
Commit: eed731496f63537350206274af09d2118b116a17
Parents: 5cb6e3e
Author: Mingliang Liu <li...@apache.org>
Authored: Wed May 10 15:05:11 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed May 10 15:05:11 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed73149/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 45c45b1..03a2782 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -2968,7 +2968,7 @@ public class NativeAzureFileSystem extends FileSystem {
* @throws IOException thrown when getting the current user.
*/
@Override
- public Token<?> getDelegationToken(final String renewer) throws IOException {
+ public synchronized Token<?> getDelegationToken(final String renewer) throws IOException {
if (kerberosSupportEnabled) {
try {
final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] hadoop git commit: YARN-6560. SLS doesn't honor node total
resource specified in sls-runner.xml. Contributed by Wangda Tan.
Posted by wa...@apache.org.
YARN-6560. SLS doesn't honor node total resource specified in sls-runner.xml. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40e6a85d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40e6a85d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40e6a85d
Branch: refs/heads/YARN-5734
Commit: 40e6a85d25387d4025585c5726b3e4e24c2c1572
Parents: ba70225
Author: Sunil G <su...@apache.org>
Authored: Thu May 18 22:49:42 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu May 18 22:49:42 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/sls/SLSRunner.java | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40e6a85d/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index f66d2d0..e679c9d 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -145,6 +145,16 @@ public class SLSRunner extends Configured implements Tool {
init(tempConf);
}
+ @Override
+ public void setConf(Configuration conf) {
+ if (null != conf) {
+ // Override setConf to make sure all conf added load sls-runner.xml, see
+ // YARN-6560
+ conf.addResource("sls-runner.xml");
+ }
+ super.setConf(conf);
+ }
+
private void init(Configuration tempConf) throws ClassNotFoundException {
nmMap = new HashMap<>();
queueAppNumMap = new HashMap<>();
@@ -152,8 +162,7 @@ public class SLSRunner extends Configured implements Tool {
amClassMap = new HashMap<>();
// runner configuration
- tempConf.addResource("sls-runner.xml");
- super.setConf(tempConf);
+ setConf(tempConf);
// runner
int poolSize = tempConf.getInt(SLSConfiguration.RUNNER_POOL_SIZE,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] hadoop git commit: HADOOP-14375. Remove tomcat support from
hadoop-functions.sh. Contributed by John Zhuge.
Posted by wa...@apache.org.
HADOOP-14375. Remove tomcat support from hadoop-functions.sh. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9e24a13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9e24a13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9e24a13
Branch: refs/heads/YARN-5734
Commit: a9e24a13b7e3cdd92898d39ec5bf03382336e597
Parents: 1411612
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri May 12 08:48:51 2017 -0500
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri May 12 08:48:51 2017 -0500
----------------------------------------------------------------------
.../src/main/bin/hadoop-functions.sh | 24 ---------
.../scripts/hadoop_finalize_catalina_opts.bats | 56 --------------------
2 files changed, 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9e24a13/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index fb8e961..8ac1b0c 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -1457,30 +1457,6 @@ function hadoop_finalize_classpath
hadoop_translate_cygwin_path CLASSPATH true
}
-## @description Finish Catalina configuration prior to execution
-## @audience private
-## @stability evolving
-## @replaceable yes
-function hadoop_finalize_catalina_opts
-{
-
- local prefix=${HADOOP_CATALINA_PREFIX}
-
- hadoop_add_param CATALINA_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_HOME}"
- if [[ -n "${JAVA_LIBRARY_PATH}" ]]; then
- hadoop_add_param CATALINA_OPTS java.library.path "-Djava.library.path=${JAVA_LIBRARY_PATH}"
- fi
- hadoop_add_param CATALINA_OPTS "${prefix}.home.dir" "-D${prefix}.home.dir=${HADOOP_HOME}"
- hadoop_add_param CATALINA_OPTS "${prefix}.config.dir" "-D${prefix}.config.dir=${HADOOP_CATALINA_CONFIG}"
- hadoop_add_param CATALINA_OPTS "${prefix}.log.dir" "-D${prefix}.log.dir=${HADOOP_CATALINA_LOG}"
- hadoop_add_param CATALINA_OPTS "${prefix}.temp.dir" "-D${prefix}.temp.dir=${HADOOP_CATALINA_TEMP}"
- hadoop_add_param CATALINA_OPTS "${prefix}.admin.port" "-D${prefix}.admin.port=${HADOOP_CATALINA_ADMIN_PORT}"
- hadoop_add_param CATALINA_OPTS "${prefix}.http.port" "-D${prefix}.http.port=${HADOOP_CATALINA_HTTP_PORT}"
- hadoop_add_param CATALINA_OPTS "${prefix}.max.threads" "-D${prefix}.max.threads=${HADOOP_CATALINA_MAX_THREADS}"
- hadoop_add_param CATALINA_OPTS "${prefix}.max.http.header.size" "-D${prefix}.max.http.header.size=${HADOOP_CATALINA_MAX_HTTP_HEADER_SIZE}"
- hadoop_add_param CATALINA_OPTS "${prefix}.ssl.keystore.file" "-D${prefix}.ssl.keystore.file=${HADOOP_CATALINA_SSL_KEYSTORE_FILE}"
-}
-
## @description Finish all the remaining environment settings prior
## @description to executing Java. This is a wrapper that calls
## @description the other `finalize` routines.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9e24a13/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize_catalina_opts.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize_catalina_opts.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize_catalina_opts.bats
deleted file mode 100644
index d91223e..0000000
--- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize_catalina_opts.bats
+++ /dev/null
@@ -1,56 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-load hadoop-functions_test_helper
-
-@test "hadoop_finalize_catalina_opts (raw)" {
- local j
-
- HADOOP_IS_CYGWIN=false
- HADOOP_CATALINA_PREFIX=test
- CATALINA_OPTS=""
- hadoop_finalize_catalina_opts
- for j in test.home.dir \
- test.config.dir \
- test.log.dir \
- test.admin.port \
- test.http.port \
- test.max.threads \
- test.ssl.keystore.file; do
- [ "${CATALINA_OPTS#*${j}}" != "${CATALINA_OPTS}" ]
- done
-}
-
-@test "hadoop_finalize_catalina_opts (cygwin)" {
- local j
-
- skip "catalina commands not supported under cygwin yet"
-
- HADOOP_IS_CYGWIN=true
- HADOOP_CATALINA_PREFIX=test
- CATALINA_OPTS=""
-
- catalina_translate_cygwin_path () {
- eval ${1}="foobarbaz"
- }
-
- hadoop_finalize_catalina_opts
- for j in test.home.dir \
- test.config.dir \
- test.log.dir \
- test.ssl.keystore.file; do
- [ "${CATALINA_OPTS#*${j}=foobarbaz}" != "${CATALINA_OPTS}" ]
- done
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] hadoop git commit: HADOOP-14427. Avoid reloading of
Configuration in ViewFileSystem creation. Contributed by Vinayakumar B
Posted by wa...@apache.org.
HADOOP-14427. Avoid reloading of Configuration in ViewFileSystem creation. Contributed by Vinayakumar B
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb7791b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb7791b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb7791b4
Branch: refs/heads/YARN-5734
Commit: eb7791b4743c593ac7d0f0d08962132723ffe3c6
Parents: d4aa9e3
Author: Mingliang Liu <li...@apache.org>
Authored: Wed May 17 13:12:14 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed May 17 13:12:14 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb7791b4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index e16da64..8265d89 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -195,7 +195,7 @@ public class ViewFileSystem extends FileSystem {
protected
FileSystem getTargetFileSystem(final INodeDir<FileSystem> dir)
throws URISyntaxException {
- return new InternalDirOfViewFs(dir, creationTime, ugi, myUri);
+ return new InternalDirOfViewFs(dir, creationTime, ugi, myUri, config);
}
@Override
@@ -905,11 +905,11 @@ public class ViewFileSystem extends FileSystem {
final URI myUri;
public InternalDirOfViewFs(final InodeTree.INodeDir<FileSystem> dir,
- final long cTime, final UserGroupInformation ugi, URI uri)
- throws URISyntaxException {
+ final long cTime, final UserGroupInformation ugi, URI uri,
+ Configuration config) throws URISyntaxException {
myUri = uri;
try {
- initialize(myUri, new Configuration());
+ initialize(myUri, config);
} catch (IOException e) {
throw new RuntimeException("Cannot occur");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] hadoop git commit: MAPREDUCE-6459. Native task crashes when
merging spilled file on ppc64. Contributed by Tao Jie and Ayappan.
Posted by wa...@apache.org.
MAPREDUCE-6459. Native task crashes when merging spilled file on ppc64. Contributed by Tao Jie and Ayappan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/035d4683
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/035d4683
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/035d4683
Branch: refs/heads/YARN-5734
Commit: 035d46836be9463a1e5d8237274150e63dc5190a
Parents: e6f6682
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed May 17 07:50:29 2017 -0400
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed May 17 07:50:29 2017 -0400
----------------------------------------------------------------------
.../hadoop-mapreduce-client-nativetask/src/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/035d4683/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index d9cec1a..bbeece9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -26,7 +26,7 @@ set(GTEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../../../../hadoop-common-project/hadoop-c
# Add extra compiler and linker flags.
# -Wno-sign-compare
-hadoop_add_compiler_flags("-DNDEBUG -DSIMPLE_MEMCPY -fno-strict-aliasing")
+hadoop_add_compiler_flags("-DNDEBUG -DSIMPLE_MEMCPY -fno-strict-aliasing -fsigned-char")
# Source location.
set(SRC main/native)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] hadoop git commit: HDFS-11842.
TestDataNodeOutlierDetectionViaMetrics UT fails intermittently. Controbuted
by Hanisha Koneru.
Posted by wa...@apache.org.
HDFS-11842. TestDataNodeOutlierDetectionViaMetrics UT fails intermittently. Controbuted by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44e9ef2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44e9ef2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44e9ef2e
Branch: refs/heads/YARN-5734
Commit: 44e9ef2e906bed85d6d660a53980d0b4202506ab
Parents: ef9e536
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed May 17 19:33:37 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed May 17 19:33:37 2017 -0700
----------------------------------------------------------------------
.../metrics/TestDataNodeOutlierDetectionViaMetrics.java | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e9ef2e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
index eb7769e..2cade4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.datanode.metrics;
+import com.google.common.base.Supplier;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.Before;
@@ -81,6 +82,13 @@ public class TestDataNodeOutlierDetectionViaMetrics {
// Trigger a snapshot.
peerMetrics.dumpSendPacketDownstreamAvgInfoAsJson();
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return peerMetrics.getOutliers().size() > 0;
+ }
+ }, 500, 100_000);
+
final Map<String, Double> outliers = peerMetrics.getOutliers();
LOG.info("Got back outlier nodes: {}", outliers);
assertThat(outliers.size(), is(1));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] hadoop git commit: HDFS-11833. HDFS architecture
documentation descibes outdated placement policy. Contributed by Chen Liang.
Posted by wa...@apache.org.
HDFS-11833. HDFS architecture documentation descibes outdated placement policy. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d1c52b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d1c52b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d1c52b4
Branch: refs/heads/YARN-5734
Commit: 1d1c52b42feae5a4271ef4b771d0d8de43e83c15
Parents: 489f859
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue May 16 11:52:33 2017 -0400
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue May 16 11:52:55 2017 -0400
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c52b4/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
index bda53a9..4bf1897 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
@@ -102,7 +102,7 @@ Large HDFS instances run on a cluster of computers that commonly spread across m
The NameNode determines the rack id each DataNode belongs to via the process outlined in [Hadoop Rack Awareness](../hadoop-common/RackAwareness.html).
A simple but non-optimal policy is to place replicas on unique racks. This prevents losing data when an entire rack fails and allows use of bandwidth from multiple racks when reading data. This policy evenly distributes replicas in the cluster which makes it easy to balance load on component failure. However, this policy increases the cost of writes because a write needs to transfer blocks to multiple racks.
-For the common case, when the replication factor is three, HDFS’s placement policy is to put one replica on one node in the local rack, another on a different node in the local rack, and the last on a different node in a different rack. This policy cuts the inter-rack write traffic which generally improves write performance. The chance of rack failure is far less than that of node failure; this policy does not impact data reliability and availability guarantees. However, it does reduce the aggregate network bandwidth used when reading data since a block is placed in only two unique racks rather than three. With this policy, the replicas of a file do not evenly distribute across the racks. One third of replicas are on one node, two thirds of replicas are on one rack, and the other third are evenly distributed across the remaining racks. This policy improves write performance without compromising data reliability or read performance.
+For the common case, when the replication factor is three, HDFS’s placement policy is to put one replica on the local machine if the writer is on a datanode, otherwise on a random datanode, another replica on a node in a different (remote) rack, and the last on a different node in the same remote rack. This policy cuts the inter-rack write traffic which generally improves write performance. The chance of rack failure is far less than that of node failure; this policy does not impact data reliability and availability guarantees. However, it does reduce the aggregate network bandwidth used when reading data since a block is placed in only two unique racks rather than three. With this policy, the replicas of a file do not evenly distribute across the racks. One third of replicas are on one node, two thirds of replicas are on one rack, and the other third are evenly distributed across the remaining racks. This policy improves write performance without compromising data reliability or
read performance.
If the replication factor is greater than 3,
the placement of the 4th and following replicas are determined randomly
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] hadoop git commit: HDFS-11757. Query StreamCapabilities when
creating balancer's lock file. Contributed by SammiChen.
Posted by wa...@apache.org.
HDFS-11757. Query StreamCapabilities when creating balancer's lock file. Contributed by SammiChen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83dd14aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83dd14aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83dd14aa
Branch: refs/heads/YARN-5734
Commit: 83dd14aa84ad697ad32c51007ac31ad39feb4288
Parents: 90cb5b4
Author: Andrew Wang <wa...@apache.org>
Authored: Thu May 11 14:09:44 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu May 11 14:09:49 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/balancer/NameNodeConnector.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83dd14aa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
index 88e40ee..e1b1005 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities.StreamCapability;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
@@ -250,8 +251,10 @@ public class NameNodeConnector implements Closeable {
.setFlags(EnumSet.of(CreateFlag.CREATE))
.build();
- Preconditions.checkState(!fs.getFileStatus(idPath).isErasureCoded(),
- "Id File should be a replicate file");
+ Preconditions.checkState(
+ fsout.hasCapability(StreamCapability.HFLUSH.getValue())
+ && fsout.hasCapability(StreamCapability.HSYNC.getValue()),
+ "Id lock file should support hflush and hsync");
// mark balancer idPath to be deleted during filesystem closure
fs.deleteOnExit(idPath);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] hadoop git commit: HADOOP-14416. Path starting with
'wasb:///' not resolved correctly while authorizing with WASB-Ranger.
Contributed by Sivaguru Sankaridurg
Posted by wa...@apache.org.
HADOOP-14416. Path starting with 'wasb:///' not resolved correctly while authorizing with WASB-Ranger. Contributed by Sivaguru Sankaridurg
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b415c6fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b415c6fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b415c6fe
Branch: refs/heads/YARN-5734
Commit: b415c6fe743242acf1d1d3eb7ea7091d90d2c0d4
Parents: 8236130
Author: Mingliang Liu <li...@apache.org>
Authored: Tue May 16 11:22:11 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue May 16 11:22:32 2017 -0700
----------------------------------------------------------------------
.../hadoop/fs/azure/NativeAzureFileSystem.java | 47 ++---
.../hadoop/fs/azure/AbstractWasbTestBase.java | 7 +-
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 11 ++
.../TestNativeAzureFileSystemAuthorization.java | 170 ++++++++-----------
.../fs/azure/TestWasbRemoteCallHelper.java | 16 +-
5 files changed, 116 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b415c6fe/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 03a2782..0ba47ef 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -1433,13 +1433,19 @@ public class NativeAzureFileSystem extends FileSystem {
* @param operation - A string describing the operation being performed ("delete", "create" etc.).
* @param originalPath - The originalPath that was being accessed
*/
- private void performAuthCheck(String requestingAccessForPath, WasbAuthorizationOperations accessType,
- String operation, String originalPath) throws WasbAuthorizationException, IOException {
+ private void performAuthCheck(Path requestingAccessForPath, WasbAuthorizationOperations accessType,
+ String operation, Path originalPath) throws WasbAuthorizationException, IOException {
+
+ if (azureAuthorization && this.authorizer != null) {
+
+ requestingAccessForPath = requestingAccessForPath.makeQualified(getUri(), getWorkingDirectory());
+ originalPath = originalPath.makeQualified(getUri(), getWorkingDirectory());
+
+ if (!this.authorizer.authorize(requestingAccessForPath.toString(), accessType.toString())) {
+ throw new WasbAuthorizationException(operation
+ + " operation for Path : " + originalPath.toString() + " not allowed");
+ }
- if (azureAuthorization && this.authorizer != null &&
- !this.authorizer.authorize(requestingAccessForPath, accessType.toString())) {
- throw new WasbAuthorizationException(operation
- + " operation for Path : " + originalPath + " not allowed");
}
}
@@ -1466,7 +1472,7 @@ public class NativeAzureFileSystem extends FileSystem {
Path absolutePath = makeAbsolute(f);
- performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.WRITE, "append", absolutePath.toString());
+ performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "append", absolutePath);
String key = pathToKey(absolutePath);
FileMetadata meta = null;
@@ -1671,7 +1677,7 @@ public class NativeAzureFileSystem extends FileSystem {
Path absolutePath = makeAbsolute(f);
Path ancestor = getAncestor(absolutePath);
- performAuthCheck(ancestor.toString(), WasbAuthorizationOperations.WRITE, "create", absolutePath.toString());
+ performAuthCheck(ancestor, WasbAuthorizationOperations.WRITE, "create", absolutePath);
String key = pathToKey(absolutePath);
@@ -1685,7 +1691,7 @@ public class NativeAzureFileSystem extends FileSystem {
throw new FileAlreadyExistsException("File already exists:" + f);
}
else {
- performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.WRITE, "create", absolutePath.toString());
+ performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "create", absolutePath);
}
}
@@ -1800,7 +1806,7 @@ public class NativeAzureFileSystem extends FileSystem {
Path absolutePath = makeAbsolute(f);
Path parentPath = absolutePath.getParent();
- performAuthCheck(parentPath.toString(), WasbAuthorizationOperations.WRITE, "delete", absolutePath.toString());
+ performAuthCheck(parentPath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
String key = pathToKey(absolutePath);
@@ -2002,14 +2008,12 @@ public class NativeAzureFileSystem extends FileSystem {
// NOTE: Ideally the subtree needs read-write-execute access check.
// But we will simplify it to write-access check.
if (metaFile.isDir()) { // the absolute-path
- performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.WRITE, "delete",
- absolutePath.toString());
+ performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
}
for (FileMetadata meta : contents) {
if (meta.isDir()) {
Path subTreeDir = keyToPath(meta.getKey());
- performAuthCheck(subTreeDir.toString(), WasbAuthorizationOperations.WRITE, "delete",
- absolutePath.toString());
+ performAuthCheck(subTreeDir, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
}
}
}
@@ -2090,8 +2094,7 @@ public class NativeAzureFileSystem extends FileSystem {
// Capture the absolute path and the path to key.
Path absolutePath = makeAbsolute(f);
- performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.READ, "getFileStatus",
- absolutePath.toString());
+ performAuthCheck(absolutePath, WasbAuthorizationOperations.READ, "getFileStatus", absolutePath);
String key = pathToKey(absolutePath);
if (key.length() == 0) { // root always exists
@@ -2192,7 +2195,7 @@ public class NativeAzureFileSystem extends FileSystem {
Path absolutePath = makeAbsolute(f);
- performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.READ, "liststatus", absolutePath.toString());
+ performAuthCheck(absolutePath, WasbAuthorizationOperations.READ, "liststatus", absolutePath);
String key = pathToKey(absolutePath);
Set<FileStatus> status = new TreeSet<FileStatus>();
@@ -2436,7 +2439,7 @@ public class NativeAzureFileSystem extends FileSystem {
Path absolutePath = makeAbsolute(f);
Path ancestor = getAncestor(absolutePath);
- performAuthCheck(ancestor.toString(), WasbAuthorizationOperations.WRITE, "mkdirs", absolutePath.toString());
+ performAuthCheck(ancestor, WasbAuthorizationOperations.WRITE, "mkdirs", absolutePath);
PermissionStatus permissionStatus = null;
if(noUmask) {
@@ -2482,7 +2485,7 @@ public class NativeAzureFileSystem extends FileSystem {
Path absolutePath = makeAbsolute(f);
- performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.READ, "read", absolutePath.toString());
+ performAuthCheck(absolutePath, WasbAuthorizationOperations.READ, "read", absolutePath);
String key = pathToKey(absolutePath);
FileMetadata meta = null;
@@ -2548,8 +2551,7 @@ public class NativeAzureFileSystem extends FileSystem {
return false;
}
- performAuthCheck(srcParentFolder.toString(), WasbAuthorizationOperations.WRITE, "rename",
- absoluteSrcPath.toString());
+ performAuthCheck(srcParentFolder, WasbAuthorizationOperations.WRITE, "rename", absoluteSrcPath);
String srcKey = pathToKey(absoluteSrcPath);
@@ -2562,8 +2564,7 @@ public class NativeAzureFileSystem extends FileSystem {
Path absoluteDstPath = makeAbsolute(dst);
Path dstParentFolder = absoluteDstPath.getParent();
- performAuthCheck(dstParentFolder.toString(), WasbAuthorizationOperations.WRITE, "rename",
- absoluteDstPath.toString());
+ performAuthCheck(dstParentFolder, WasbAuthorizationOperations.WRITE, "rename", absoluteDstPath);
String dstKey = pathToKey(absoluteDstPath);
FileMetadata dstMetadata = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b415c6fe/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
index 58d278a..6ae18fe 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
@@ -20,11 +20,9 @@ package org.apache.hadoop.fs.azure;
import static org.junit.Assume.assumeNotNull;
-import org.apache.hadoop.fs.FileSystem;
-
+import com.google.common.annotations.VisibleForTesting;
import org.junit.After;
import org.junit.Before;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,7 +38,8 @@ public abstract class AbstractWasbTestBase {
protected static final Logger LOG =
LoggerFactory.getLogger(AbstractWasbTestBase.class);
- protected FileSystem fs;
+ @VisibleForTesting
+ protected NativeAzureFileSystem fs;
private AzureBlobStorageTestAccount testAccount;
@Before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b415c6fe/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
index 445bfd8..0b3422c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
/**
* A mock wasb authorizer implementation.
@@ -32,6 +33,14 @@ public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface {
private Map<AuthorizationComponent, Boolean> authRules;
+ // The full qualified URL to the root directory
+ private String qualifiedPrefixUrl;
+
+ public MockWasbAuthorizerImpl(NativeAzureFileSystem fs) {
+ qualifiedPrefixUrl = new Path("/").makeQualified(fs.getUri(), fs.getWorkingDirectory())
+ .toString().replaceAll("/$", "");
+ }
+
@Override
public void init(Configuration conf) {
authRules = new HashMap<AuthorizationComponent, Boolean>();
@@ -40,6 +49,8 @@ public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface {
public void addAuthRule(String wasbAbsolutePath,
String accessType, boolean access) {
+ wasbAbsolutePath = qualifiedPrefixUrl + wasbAbsolutePath;
+
AuthorizationComponent component = wasbAbsolutePath.endsWith("*")
? new AuthorizationComponent("^" + wasbAbsolutePath.replace("*", ".*"), accessType)
: new AuthorizationComponent(wasbAbsolutePath, accessType);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b415c6fe/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
index 4e49622..f7a2eb7 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
@@ -78,6 +78,15 @@ public class TestNativeAzureFileSystemAuthorization
}
/**
+ * Setup the expected exception class, and exception message that the test is supposed to fail with
+ */
+ private void setExpectedFailureMessage(String operation, Path path) {
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage(String.format("%s operation for Path : %s not allowed",
+ operation, path.makeQualified(fs.getUri(), fs.getWorkingDirectory())));
+ }
+
+ /**
* Positive test to verify Create access check
* The file is created directly under an existing folder.
* No intermediate folders need to be created.
@@ -86,13 +95,10 @@ public class TestNativeAzureFileSystemAuthorization
@Test
public void testCreateAccessWithoutCreateIntermediateFoldersCheckPositive() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
@@ -117,13 +123,10 @@ public class TestNativeAzureFileSystemAuthorization
@Test
public void testCreateAccessWithCreateIntermediateFoldersCheckPositive() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/testCreateAccessCheckPositive/1/2/3");
Path testPath = new Path(parentDir, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
@@ -148,16 +151,12 @@ public class TestNativeAzureFileSystemAuthorization
@Test // (expected=WasbAuthorizationException.class)
public void testCreateAccessWithOverwriteCheckNegative() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
- expectedEx.expectMessage("create operation for Path : /test.dat not allowed");
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ setExpectedFailureMessage("create", testPath);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
@@ -184,13 +183,10 @@ public class TestNativeAzureFileSystemAuthorization
@Test
public void testCreateAccessWithOverwriteCheckPositive() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
@@ -218,16 +214,12 @@ public class TestNativeAzureFileSystemAuthorization
@Test // (expected=WasbAuthorizationException.class)
public void testCreateAccessCheckNegative() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
- expectedEx.expectMessage("create operation for Path : /testCreateAccessCheckNegative/test.dat not allowed");
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/testCreateAccessCheckNegative");
Path testPath = new Path(parentDir, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ setExpectedFailureMessage("create", testPath);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false);
fs.updateWasbAuthorizer(authorizer);
@@ -249,14 +241,11 @@ public class TestNativeAzureFileSystemAuthorization
@Test
public void testListAccessCheckPositive() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/testListAccessCheckPositive");
Path intermediateFolders = new Path(parentDir, "1/2/3/");
Path testPath = new Path(intermediateFolders, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
@@ -280,16 +269,12 @@ public class TestNativeAzureFileSystemAuthorization
@Test //(expected=WasbAuthorizationException.class)
public void testListAccessCheckNegative() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
- expectedEx.expectMessage("liststatus operation for Path : /testListAccessCheckNegative/test.dat not allowed");
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/testListAccessCheckNegative");
Path testPath = new Path(parentDir, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ setExpectedFailureMessage("liststatus", testPath);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false);
@@ -312,14 +297,11 @@ public class TestNativeAzureFileSystemAuthorization
@Test
public void testRenameAccessCheckPositive() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/testRenameAccessCheckPositive");
Path srcPath = new Path(parentDir, "test1.dat");
Path dstPath = new Path(parentDir, "test2.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parentDir */
authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); /* for rename */
@@ -347,16 +329,13 @@ public class TestNativeAzureFileSystemAuthorization
@Test //(expected=WasbAuthorizationException.class)
public void testRenameAccessCheckNegative() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
- expectedEx.expectMessage("rename operation for Path : /testRenameAccessCheckNegative/test1.dat not allowed");
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
Path parentDir = new Path("/testRenameAccessCheckNegative");
Path srcPath = new Path(parentDir, "test1.dat");
Path dstPath = new Path(parentDir, "test2.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ setExpectedFailureMessage("rename", srcPath);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dir */
authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), false);
@@ -384,17 +363,14 @@ public class TestNativeAzureFileSystemAuthorization
@Test //(expected=WasbAuthorizationException.class)
public void testRenameAccessCheckNegativeOnDstFolder() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
- expectedEx.expectMessage("rename operation for Path : /testRenameAccessCheckNegativeDst/test2.dat not allowed");
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
Path parentSrcDir = new Path("/testRenameAccessCheckNegativeSrc");
Path srcPath = new Path(parentSrcDir, "test1.dat");
Path parentDstDir = new Path("/testRenameAccessCheckNegativeDst");
Path dstPath = new Path(parentDstDir, "test2.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ setExpectedFailureMessage("rename", dstPath);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dir */
authorizer.addAuthRule(parentSrcDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
@@ -419,17 +395,15 @@ public class TestNativeAzureFileSystemAuthorization
* Positive test to verify rename access check - the dstFolder allows rename
* @throws Throwable
*/
- @Test //(expected=WasbAuthorizationException.class)
+ @Test
public void testRenameAccessCheckPositiveOnDstFolder() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
Path parentSrcDir = new Path("/testRenameAccessCheckPositiveSrc");
Path srcPath = new Path(parentSrcDir, "test1.dat");
Path parentDstDir = new Path("/testRenameAccessCheckPositiveDst");
Path dstPath = new Path(parentDstDir, "test2.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dirs */
authorizer.addAuthRule(parentSrcDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
@@ -461,12 +435,10 @@ public class TestNativeAzureFileSystemAuthorization
@Test
public void testReadAccessCheckPositive() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
Path parentDir = new Path("/testReadAccessCheckPositive");
Path testPath = new Path(parentDir, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
@@ -504,15 +476,12 @@ public class TestNativeAzureFileSystemAuthorization
@Test //(expected=WasbAuthorizationException.class)
public void testReadAccessCheckNegative() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
- expectedEx.expectMessage("read operation for Path : /testReadAccessCheckNegative/test.dat not allowed");
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
Path parentDir = new Path("/testReadAccessCheckNegative");
Path testPath = new Path(parentDir, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ setExpectedFailureMessage("read", testPath);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false);
@@ -548,13 +517,10 @@ public class TestNativeAzureFileSystemAuthorization
@Test
public void testFileDeleteAccessCheckPositive() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
@@ -576,16 +542,12 @@ public class TestNativeAzureFileSystemAuthorization
@Test //(expected=WasbAuthorizationException.class)
public void testFileDeleteAccessCheckNegative() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
- expectedEx.expectMessage("delete operation for Path : /test.dat not allowed");
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ setExpectedFailureMessage("delete", testPath);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
@@ -622,13 +584,10 @@ public class TestNativeAzureFileSystemAuthorization
@Test
public void testFileDeleteAccessWithIntermediateFoldersCheckPositive() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path parentDir = new Path("/testDeleteIntermediateFolder");
Path testPath = new Path(parentDir, "1/2/test.dat");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); // for create and delete
authorizer.addAuthRule("/testDeleteIntermediateFolder*",
@@ -655,12 +614,9 @@ public class TestNativeAzureFileSystemAuthorization
@Test
public void testGetFileStatusPositive() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path testPath = new Path("/");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.READ.toString(), true);
fs.updateWasbAuthorizer(authorizer);
@@ -675,15 +631,11 @@ public class TestNativeAzureFileSystemAuthorization
@Test //(expected=WasbAuthorizationException.class)
public void testGetFileStatusNegative() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
- expectedEx.expectMessage("getFileStatus operation for Path : / not allowed");
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path testPath = new Path("/");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ setExpectedFailureMessage("getFileStatus", testPath);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.READ.toString(), false);
fs.updateWasbAuthorizer(authorizer);
@@ -698,12 +650,9 @@ public class TestNativeAzureFileSystemAuthorization
@Test
public void testMkdirsCheckPositive() throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path testPath = new Path("/testMkdirsAccessCheckPositive/1/2/3");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
@@ -726,15 +675,11 @@ public class TestNativeAzureFileSystemAuthorization
@Test //(expected=WasbAuthorizationException.class)
public void testMkdirsCheckNegative() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
- expectedEx.expectMessage("mkdirs operation for Path : /testMkdirsAccessCheckNegative/1/2/3 not allowed");
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
Path testPath = new Path("/testMkdirsAccessCheckNegative/1/2/3");
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ setExpectedFailureMessage("mkdirs", testPath);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
authorizer.init(null);
authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false);
authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
@@ -749,4 +694,23 @@ public class TestNativeAzureFileSystemAuthorization
fs.delete(new Path("/testMkdirsAccessCheckNegative"), true);
}
}
+
+
+ /**
+ * Positive test triple slash format (wasb:///) access check
+ * @throws Throwable
+ */
+ @Test
+ public void testListStatusWithTripleSlashCheckPositive() throws Throwable {
+
+ Path testPath = new Path("/");
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs);
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ Path testPathWithTripleSlash = new Path("wasb:///" + testPath);
+ fs.listStatus(testPathWithTripleSlash);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b415c6fe/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java
index d7c40b9..77be1b8 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java
@@ -32,6 +32,8 @@ import org.junit.rules.ExpectedException;
import org.mockito.Mockito;
import java.io.ByteArrayInputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE;
@@ -261,16 +263,20 @@ public class TestWasbRemoteCallHelper
performop(mockHttpClient);
}
- private void setupExpectations() {
+ private void setupExpectations() throws UnsupportedEncodingException {
+
+ String path = new Path("/").makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString();
+ String pathEncoded = URLEncoder.encode(path, "UTF-8");
+
+ String requestURI = String.format("http://localhost/CHECK_AUTHORIZATION?wasb_absolute_path=%s&operation_type=write", pathEncoded);
expectedEx.expect(WasbAuthorizationException.class);
expectedEx.expectMessage("org.apache.hadoop.fs.azure.WasbRemoteCallException: "
- + "http://localhost/CHECK_AUTHORIZATION?wasb_absolute_path=%2F&"
- + "operation_type=write:Encountered IOException while making remote call");
+ + requestURI
+ + ":Encountered IOException while making remote call"
+ );
}
private void performop(HttpClient mockHttpClient) throws Throwable {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
Path testPath = new Path("/", "test.dat");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] hadoop git commit: HADOOP-14434. Use MoveFileEx to allow
renaming a file when the destination exists. Contributed by Lukas Majercak
Posted by wa...@apache.org.
HADOOP-14434. Use MoveFileEx to allow renaming a file when the destination exists. Contributed by Lukas Majercak
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef9e536a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef9e536a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef9e536a
Branch: refs/heads/YARN-5734
Commit: ef9e536a7137d209985d25a4769712c2db2c52bf
Parents: eb7791b
Author: Chris Douglas <cd...@apache.org>
Authored: Wed May 17 17:21:03 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Wed May 17 17:21:03 2017 -0700
----------------------------------------------------------------------
.../main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c | 2 +-
.../test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java | 6 ++++--
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef9e536a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
index a7d4b55..242a456 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
@@ -1204,7 +1204,7 @@ done:
if (!src) goto done; // exception was thrown
dst = (LPCWSTR) (*env)->GetStringChars(env, jdst, NULL);
if (!dst) goto done; // exception was thrown
- if (!MoveFile(src, dst)) {
+ if (!MoveFileEx(src, dst, MOVEFILE_REPLACE_EXISTING)) {
throw_ioe(env, GetLastError());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef9e536a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
index b2445a3..436f10a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
@@ -510,7 +510,7 @@ public class TestNativeIO {
Assert.assertEquals(Errno.ENOENT, e.getErrno());
}
}
-
+
// Test renaming a file to itself. It should succeed and do nothing.
File sourceFile = new File(TEST_DIR, "source");
Assert.assertTrue(sourceFile.createNewFile());
@@ -536,7 +536,9 @@ public class TestNativeIO {
}
}
- FileUtils.deleteQuietly(TEST_DIR);
+ // Test renaming to an existing file
+ assertTrue(targetFile.exists());
+ NativeIO.renameTo(sourceFile, targetFile);
}
@Test(timeout=10000)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] hadoop git commit: YARN-6587. Refactor of
ResourceManager#startWebApp in a Util class. (Giovanni Matteo Fumarola via
curino).
Posted by wa...@apache.org.
YARN-6587. Refactor of ResourceManager#startWebApp in a Util class. (Giovanni Matteo Fumarola via curino).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6600abbb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6600abbb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6600abbb
Branch: refs/heads/YARN-5734
Commit: 6600abbb5c23a83e3a9ef48a945bc8fe19c8178a
Parents: 2397a26
Author: Carlo Curino <cu...@apache.org>
Authored: Fri May 12 17:48:55 2017 -0700
Committer: Carlo Curino <cu...@apache.org>
Committed: Fri May 12 17:48:55 2017 -0700
----------------------------------------------------------------------
.../server/resourcemanager/ResourceManager.java | 93 +-----------
.../resourcemanager/webapp/RMWebAppUtil.java | 149 +++++++++++++++++++
2 files changed, 152 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6600abbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 75d6df2..1f5e8cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -29,17 +29,13 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.http.lib.StaticUserWebFilter;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.Groups;
-import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
@@ -103,9 +99,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRen
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilter;
-import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer;
import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxy;
@@ -1038,92 +1033,10 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected void startWepApp() {
- // Use the customized yarn filter instead of the standard kerberos filter to
- // allow users to authenticate using delegation tokens
- // 4 conditions need to be satisfied -
- // 1. security is enabled
- // 2. http auth type is set to kerberos
- // 3. "yarn.resourcemanager.webapp.use-yarn-filter" override is set to true
- // 4. hadoop.http.filter.initializers container AuthenticationFilterInitializer
-
Configuration conf = getConfig();
- boolean enableCorsFilter =
- conf.getBoolean(YarnConfiguration.RM_WEBAPP_ENABLE_CORS_FILTER,
- YarnConfiguration.DEFAULT_RM_WEBAPP_ENABLE_CORS_FILTER);
- boolean useYarnAuthenticationFilter =
- conf.getBoolean(
- YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER,
- YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER);
- String authPrefix = "hadoop.http.authentication.";
- String authTypeKey = authPrefix + "type";
- String filterInitializerConfKey = "hadoop.http.filter.initializers";
- String actualInitializers = "";
- Class<?>[] initializersClasses =
- conf.getClasses(filterInitializerConfKey);
-
- // setup CORS
- if (enableCorsFilter) {
- conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX
- + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true);
- }
-
- boolean hasHadoopAuthFilterInitializer = false;
- boolean hasRMAuthFilterInitializer = false;
- if (initializersClasses != null) {
- for (Class<?> initializer : initializersClasses) {
- if (initializer.getName().equals(
- AuthenticationFilterInitializer.class.getName())) {
- hasHadoopAuthFilterInitializer = true;
- }
- if (initializer.getName().equals(
- RMAuthenticationFilterInitializer.class.getName())) {
- hasRMAuthFilterInitializer = true;
- }
- }
- if (UserGroupInformation.isSecurityEnabled()
- && useYarnAuthenticationFilter
- && hasHadoopAuthFilterInitializer
- && conf.get(authTypeKey, "").equals(
- KerberosAuthenticationHandler.TYPE)) {
- ArrayList<String> target = new ArrayList<String>();
- for (Class<?> filterInitializer : initializersClasses) {
- if (filterInitializer.getName().equals(
- AuthenticationFilterInitializer.class.getName())) {
- if (hasRMAuthFilterInitializer == false) {
- target.add(RMAuthenticationFilterInitializer.class.getName());
- }
- continue;
- }
- target.add(filterInitializer.getName());
- }
- actualInitializers = StringUtils.join(",", target);
- LOG.info("Using RM authentication filter(kerberos/delegation-token)"
- + " for RM webapp authentication");
- RMAuthenticationFilter
- .setDelegationTokenSecretManager(getClientRMService().rmDTSecretManager);
- conf.set(filterInitializerConfKey, actualInitializers);
- }
- }
-
- // if security is not enabled and the default filter initializer has not
- // been set, set the initializer to include the
- // RMAuthenticationFilterInitializer which in turn will set up the simple
- // auth filter.
-
- String initializers = conf.get(filterInitializerConfKey);
- if (!UserGroupInformation.isSecurityEnabled()) {
- if (initializersClasses == null || initializersClasses.length == 0) {
- conf.set(filterInitializerConfKey,
- RMAuthenticationFilterInitializer.class.getName());
- conf.set(authTypeKey, "simple");
- } else if (initializers.equals(StaticUserWebFilter.class.getName())) {
- conf.set(filterInitializerConfKey,
- RMAuthenticationFilterInitializer.class.getName() + ","
- + initializers);
- conf.set(authTypeKey, "simple");
- }
- }
+ RMWebAppUtil.setupSecurityAndFilters(conf,
+ getClientRMService().rmDTSecretManager);
Builder<ApplicationMasterService> builder =
WebApps
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6600abbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java
new file mode 100644
index 0000000..263828b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java
@@ -0,0 +1,149 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.lib.StaticUserWebFilter;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
+import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilter;
+import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer;
+
+/**
+ * Util class for ResourceManager WebApp.
+ */
+public final class RMWebAppUtil {
+
+ private static final Log LOG = LogFactory.getLog(RMWebAppUtil.class);
+
+ /**
+ * Private constructor.
+ */
+ private RMWebAppUtil() {
+ // not called
+ }
+
+ /**
+ * Helper method to setup filters and authentication for ResourceManager
+ * WebServices.
+ *
+ * Use the customized yarn filter instead of the standard kerberos filter to
+ * allow users to authenticate using delegation tokens 4 conditions need to be
+ * satisfied:
+ *
+ * 1. security is enabled.
+ *
+ * 2. http auth type is set to kerberos.
+ *
+ * 3. "yarn.resourcemanager.webapp.use-yarn-filter" override is set to true.
+ *
+ * 4. hadoop.http.filter.initializers container
+ * AuthenticationFilterInitializer.
+ *
+ * @param conf RM configuration.
+ * @param rmDTSecretManager RM specific delegation token secret manager.
+ **/
+ public static void setupSecurityAndFilters(Configuration conf,
+ RMDelegationTokenSecretManager rmDTSecretManager) {
+
+ boolean enableCorsFilter =
+ conf.getBoolean(YarnConfiguration.RM_WEBAPP_ENABLE_CORS_FILTER,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ENABLE_CORS_FILTER);
+ boolean useYarnAuthenticationFilter = conf.getBoolean(
+ YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER);
+ String authPrefix = "hadoop.http.authentication.";
+ String authTypeKey = authPrefix + "type";
+ String filterInitializerConfKey = "hadoop.http.filter.initializers";
+ String actualInitializers = "";
+ Class<?>[] initializersClasses = conf.getClasses(filterInitializerConfKey);
+
+ // setup CORS
+ if (enableCorsFilter) {
+ conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX
+ + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true);
+ }
+
+ boolean hasHadoopAuthFilterInitializer = false;
+ boolean hasRMAuthFilterInitializer = false;
+ if (initializersClasses != null) {
+ for (Class<?> initializer : initializersClasses) {
+ if (initializer.getName()
+ .equals(AuthenticationFilterInitializer.class.getName())) {
+ hasHadoopAuthFilterInitializer = true;
+ }
+ if (initializer.getName()
+ .equals(RMAuthenticationFilterInitializer.class.getName())) {
+ hasRMAuthFilterInitializer = true;
+ }
+ }
+ if (UserGroupInformation.isSecurityEnabled()
+ && useYarnAuthenticationFilter && hasHadoopAuthFilterInitializer
+ && conf.get(authTypeKey, "")
+ .equals(KerberosAuthenticationHandler.TYPE)) {
+ ArrayList<String> target = new ArrayList<String>();
+ for (Class<?> filterInitializer : initializersClasses) {
+ if (filterInitializer.getName()
+ .equals(AuthenticationFilterInitializer.class.getName())) {
+ if (!hasRMAuthFilterInitializer) {
+ target.add(RMAuthenticationFilterInitializer.class.getName());
+ }
+ continue;
+ }
+ target.add(filterInitializer.getName());
+ }
+ actualInitializers = StringUtils.join(",", target);
+
+ LOG.info("Using RM authentication filter(kerberos/delegation-token)"
+ + " for RM webapp authentication");
+ RMAuthenticationFilter
+ .setDelegationTokenSecretManager(rmDTSecretManager);
+ conf.set(filterInitializerConfKey, actualInitializers);
+ }
+ }
+
+ // if security is not enabled and the default filter initializer has not
+ // been set, set the initializer to include the
+ // RMAuthenticationFilterInitializer which in turn will set up the simple
+ // auth filter.
+
+ String initializers = conf.get(filterInitializerConfKey);
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ if (initializersClasses == null || initializersClasses.length == 0) {
+ conf.set(filterInitializerConfKey,
+ RMAuthenticationFilterInitializer.class.getName());
+ conf.set(authTypeKey, "simple");
+ } else if (initializers.equals(StaticUserWebFilter.class.getName())) {
+ conf.set(filterInitializerConfKey,
+ RMAuthenticationFilterInitializer.class.getName() + ","
+ + initializers);
+ conf.set(authTypeKey, "simple");
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] hadoop git commit: YARN-6535. Program needs to exit when SLS
finishes. (yufeigu via rkanter)
Posted by wa...@apache.org.
YARN-6535. Program needs to exit when SLS finishes. (yufeigu via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/101852ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/101852ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/101852ca
Branch: refs/heads/YARN-5734
Commit: 101852ca11ed4a9c4d4664c6c797fa7173dc59ae
Parents: b415c6f
Author: Robert Kanter <rk...@apache.org>
Authored: Tue May 16 17:52:17 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue May 16 17:52:17 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/101852ca/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 03281a5..f66d2d0 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -124,6 +124,8 @@ public class SLSRunner extends Configured implements Tool {
private final static int DEFAULT_MAPPER_PRIORITY = 20;
private final static int DEFAULT_REDUCER_PRIORITY = 10;
+ private static boolean exitAtTheFinish = false;
+
/**
* The type of trace in input.
*/
@@ -761,6 +763,9 @@ public class SLSRunner extends Configured implements Tool {
if (remainingApps == 0) {
LOG.info("SLSRunner tears down.");
+ if (exitAtTheFinish) {
+ System.exit(0);
+ }
}
}
@@ -857,6 +862,7 @@ public class SLSRunner extends Configured implements Tool {
}
public static void main(String[] argv) throws Exception {
+ exitAtTheFinish = true;
ToolRunner.run(new Configuration(), new SLSRunner(), argv);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] hadoop git commit: HDFS-11818.
TestBlockManager.testSufficientlyReplBlocksUsesNewRack fails intermittently.
Contributed by Nathan Roberts
Posted by wa...@apache.org.
HDFS-11818. TestBlockManager.testSufficientlyReplBlocksUsesNewRack fails intermittently. Contributed by Nathan Roberts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2397a262
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2397a262
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2397a262
Branch: refs/heads/YARN-5734
Commit: 2397a2626e22d002174f4a36891d713a7e1f1b20
Parents: 7bc2172
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Fri May 12 17:42:47 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Fri May 12 17:42:47 2017 -0500
----------------------------------------------------------------------
.../hadoop/hdfs/server/blockmanagement/TestBlockManager.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2397a262/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index beaef4a..3088b7b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -159,7 +159,6 @@ public class TestBlockManager {
Mockito.when(haState.shouldPopulateReplQueues()).thenReturn(true);
Mockito.when(fsn.getHAContext()).thenReturn(haContext);
bm = new BlockManager(fsn, false, conf);
- bm.setInitializedReplQueues(true);
CacheManager cm = Mockito.mock(CacheManager.class);
Mockito.doReturn(cm).when(fsn).getCacheManager();
GSet<CachedBlock, CachedBlock> cb =
@@ -967,10 +966,12 @@ public class TestBlockManager {
StorageReceivedDeletedBlocks srdb =
new StorageReceivedDeletedBlocks(new DatanodeStorage(ds.getStorageID()),
rdbiList.toArray(new ReceivedDeletedBlockInfo[rdbiList.size()]));
+ bm.setInitializedReplQueues(true);
bm.processIncrementalBlockReport(node, srdb);
// Needed replications should still be 0.
assertEquals("UC block was incorrectly added to needed Replications",
0, bm.neededReconstruction.size());
+ bm.setInitializedReplQueues(false);
}
private BlockInfo addBlockToBM(long blkId) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] hadoop git commit: YARN-6602. Impersonation does not work if
standby RM is contacted first (rkanter)
Posted by wa...@apache.org.
YARN-6602. Impersonation does not work if standby RM is contacted first (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9855225a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9855225a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9855225a
Branch: refs/heads/YARN-5734
Commit: 9855225a790e66835c79816712f432ffa8c599b4
Parents: 66bba8c
Author: Robert Kanter <rk...@apache.org>
Authored: Fri May 19 15:26:51 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Fri May 19 15:43:22 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/client/ClientRMProxy.java | 4 +-
.../ConfiguredRMFailoverProxyProvider.java | 2 +-
.../org/apache/hadoop/yarn/client/RMProxy.java | 45 +++-----
.../RequestHedgingRMFailoverProxyProvider.java | 2 +-
.../hadoop/yarn/client/TestClientRMProxy.java | 109 +++++++++++++++++++
.../hadoop/yarn/server/api/ServerRMProxy.java | 4 +-
6 files changed, 129 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9855225a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
index b29263e..0232deb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
@@ -48,7 +48,6 @@ import com.google.common.base.Preconditions;
@InterfaceStability.Stable
public class ClientRMProxy<T> extends RMProxy<T> {
private static final Log LOG = LogFactory.getLog(ClientRMProxy.class);
- private static final ClientRMProxy INSTANCE = new ClientRMProxy();
private interface ClientRMProtocols extends ApplicationClientProtocol,
ApplicationMasterProtocol, ResourceManagerAdministrationProtocol {
@@ -69,7 +68,8 @@ public class ClientRMProxy<T> extends RMProxy<T> {
*/
public static <T> T createRMProxy(final Configuration configuration,
final Class<T> protocol) throws IOException {
- return createRMProxy(configuration, protocol, INSTANCE);
+ ClientRMProxy<T> clientRMProxy = new ClientRMProxy<>();
+ return createRMProxy(configuration, protocol, clientRMProxy);
}
private static void setAMRMTokenService(final Configuration conf)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9855225a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java
index 8676db2..d6b6cce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java
@@ -74,7 +74,7 @@ public class ConfiguredRMFailoverProxyProvider<T>
protected T getProxyInternal() {
try {
final InetSocketAddress rmAddress = rmProxy.getRMAddress(conf, protocol);
- return RMProxy.getProxy(conf, protocol, rmAddress);
+ return rmProxy.getProxy(conf, protocol, rmAddress);
} catch (IOException ioe) {
LOG.error("Unable to create proxy to the ResourceManager " +
rmServiceIds[currentProxyIndex], ioe);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9855225a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index 3ab06bd..8aa4107 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -57,8 +57,15 @@ import com.google.common.annotations.VisibleForTesting;
public class RMProxy<T> {
private static final Log LOG = LogFactory.getLog(RMProxy.class);
+ private UserGroupInformation user;
- protected RMProxy() {}
+ protected RMProxy() {
+ try {
+ this.user = UserGroupInformation.getCurrentUser();
+ } catch (IOException ioe) {
+ throw new YarnRuntimeException("Unable to determine user", ioe);
+ }
+ }
/**
* Verify the passed protocol is supported.
@@ -86,7 +93,7 @@ public class RMProxy<T> {
*/
@Private
protected static <T> T createRMProxy(final Configuration configuration,
- final Class<T> protocol, RMProxy instance) throws IOException {
+ final Class<T> protocol, RMProxy<T> instance) throws IOException {
YarnConfiguration conf = (configuration instanceof YarnConfiguration)
? (YarnConfiguration) configuration
: new YarnConfiguration(configuration);
@@ -103,7 +110,7 @@ public class RMProxy<T> {
*/
@Private
protected static <T> T createRMProxy(final Configuration configuration,
- final Class<T> protocol, RMProxy instance, final long retryTime,
+ final Class<T> protocol, RMProxy<T> instance, final long retryTime,
final long retryInterval) throws IOException {
YarnConfiguration conf = (configuration instanceof YarnConfiguration)
? (YarnConfiguration) configuration
@@ -114,7 +121,7 @@ public class RMProxy<T> {
}
private static <T> T newProxyInstance(final YarnConfiguration conf,
- final Class<T> protocol, RMProxy instance, RetryPolicy retryPolicy)
+ final Class<T> protocol, RMProxy<T> instance, RetryPolicy retryPolicy)
throws IOException{
if (HAUtil.isHAEnabled(conf)) {
RMFailoverProxyProvider<T> provider =
@@ -123,44 +130,20 @@ public class RMProxy<T> {
} else {
InetSocketAddress rmAddress = instance.getRMAddress(conf, protocol);
LOG.info("Connecting to ResourceManager at " + rmAddress);
- T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
+ T proxy = instance.getProxy(conf, protocol, rmAddress);
return (T) RetryProxy.create(protocol, proxy, retryPolicy);
}
}
/**
- * @deprecated
- * This method is deprecated and is not used by YARN internally any more.
- * To create a proxy to the RM, use ClientRMProxy#createRMProxy or
- * ServerRMProxy#createRMProxy.
- *
- * Create a proxy to the ResourceManager at the specified address.
- *
- * @param conf Configuration to generate retry policy
- * @param protocol Protocol for the proxy
- * @param rmAddress Address of the ResourceManager
- * @param <T> Type information of the proxy
- * @return Proxy to the RM
- * @throws IOException
- */
- @Deprecated
- public static <T> T createRMProxy(final Configuration conf,
- final Class<T> protocol, InetSocketAddress rmAddress) throws IOException {
- RetryPolicy retryPolicy = createRetryPolicy(conf, HAUtil.isHAEnabled(conf));
- T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
- LOG.info("Connecting to ResourceManager at " + rmAddress);
- return (T) RetryProxy.create(protocol, proxy, retryPolicy);
- }
-
- /**
* Get a proxy to the RM at the specified address. To be used to create a
* RetryProxy.
*/
@Private
- static <T> T getProxy(final Configuration conf,
+ <T> T getProxy(final Configuration conf,
final Class<T> protocol, final InetSocketAddress rmAddress)
throws IOException {
- return UserGroupInformation.getCurrentUser().doAs(
+ return user.doAs(
new PrivilegedAction<T>() {
@Override
public T run() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9855225a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RequestHedgingRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RequestHedgingRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RequestHedgingRMFailoverProxyProvider.java
index 13c02af..4c16225 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RequestHedgingRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RequestHedgingRMFailoverProxyProvider.java
@@ -95,7 +95,7 @@ public class RequestHedgingRMFailoverProxyProvider<T>
// Create proxy that can retry exceptions properly.
RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf, false);
InetSocketAddress rmAddress = rmProxy.getRMAddress(conf, protocol);
- T proxy = RMProxy.<T> getProxy(conf, protocol, rmAddress);
+ T proxy = rmProxy.getProxy(conf, protocol, rmAddress);
return (T) RetryProxy.create(protocol, proxy, retryPolicy);
} catch (IOException ioe) {
LOG.error("Unable to create proxy to the ResourceManager "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9855225a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/TestClientRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/TestClientRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/TestClientRMProxy.java
index 700a37f..6c31fea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/TestClientRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/TestClientRMProxy.java
@@ -18,12 +18,26 @@
package org.apache.hadoop.yarn.client;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.Records;
+import org.junit.Assert;
import org.junit.Test;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public class TestClientRMProxy {
@@ -86,4 +100,99 @@ public class TestClientRMProxy {
service.contains(defaultRMAddress));
}
}
+
+ /**
+ * Verify that the RPC layer is always created using the correct UGI from the
+ * RMProxy. It should always use the UGI from creation in subsequent uses,
+ * even outside of a doAs.
+ *
+ * @throws Exception an Exception occurred
+ */
+ @Test
+ public void testProxyUserCorrectUGI() throws Exception {
+ final YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
+ conf.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, "rm1"),
+ "0.0.0.0");
+ conf.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, "rm2"),
+ "0.0.0.0");
+ conf.setLong(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, 2);
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 2);
+ conf.setLong(
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 2);
+
+ // Replace the RPC implementation with one that will capture the current UGI
+ conf.setClass(YarnConfiguration.IPC_RPC_IMPL,
+ UGICapturingHadoopYarnProtoRPC.class, YarnRPC.class);
+
+ UserGroupInformation realUser = UserGroupInformation.getCurrentUser();
+ UserGroupInformation proxyUser =
+ UserGroupInformation.createProxyUserForTesting("proxy", realUser,
+ new String[] {"group1"});
+
+ // Create the RMProxy using the proxyUser
+ ApplicationClientProtocol rmProxy = proxyUser.doAs(
+ new PrivilegedExceptionAction<ApplicationClientProtocol>() {
+ @Override
+ public ApplicationClientProtocol run() throws Exception {
+ return ClientRMProxy.createRMProxy(conf,
+ ApplicationClientProtocol.class);
+ }
+ });
+
+ // It was in a doAs, so the UGI should be correct
+ assertUGI();
+
+ // Try to use the RMProxy, which should trigger the RPC again
+ GetNewApplicationRequest request =
+ Records.newRecord(GetNewApplicationRequest.class);
+ UGICapturingHadoopYarnProtoRPC.lastCurrentUser = null;
+ try {
+ rmProxy.getNewApplication(request);
+ } catch (IOException ioe) {
+ // ignore - RMs are not running so this is expected to fail
+ }
+
+ // This time it was outside a doAs, but make sure the UGI was still correct
+ assertUGI();
+ }
+
+ private void assertUGI() throws IOException {
+ UserGroupInformation lastCurrentUser =
+ UGICapturingHadoopYarnProtoRPC.lastCurrentUser;
+ assertNotNull(lastCurrentUser);
+ assertEquals("proxy", lastCurrentUser.getShortUserName());
+ Assert.assertEquals(UserGroupInformation.AuthenticationMethod.PROXY,
+ lastCurrentUser.getAuthenticationMethod());
+ assertEquals(UserGroupInformation.getCurrentUser(),
+ lastCurrentUser.getRealUser());
+ // Reset UGICapturingHadoopYarnProtoRPC
+ UGICapturingHadoopYarnProtoRPC.lastCurrentUser = null;
+ }
+
+ /**
+ * Subclass of {@link HadoopYarnProtoRPC} which captures the current UGI in
+ * a static variable. Used by {@link #testProxyUserCorrectUGI()}.
+ */
+ public static class UGICapturingHadoopYarnProtoRPC
+ extends HadoopYarnProtoRPC {
+
+ static UserGroupInformation lastCurrentUser = null;
+
+ @Override
+ public Object getProxy(Class protocol, InetSocketAddress addr,
+ Configuration conf) {
+ UserGroupInformation currentUser = null;
+ try {
+ currentUser = UserGroupInformation.getCurrentUser();
+ } catch (IOException ioe) {
+ Assert.fail("Unable to get current user\n"
+ + StringUtils.stringifyException(ioe));
+ }
+ lastCurrentUser = currentUser;
+
+ return super.getProxy(protocol, addr, conf);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9855225a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
index 8555fc3..3012be3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
@@ -32,7 +32,6 @@ import com.google.common.base.Preconditions;
public class ServerRMProxy<T> extends RMProxy<T> {
private static final Log LOG = LogFactory.getLog(ServerRMProxy.class);
- private static final ServerRMProxy INSTANCE = new ServerRMProxy();
private ServerRMProxy() {
super();
@@ -65,7 +64,8 @@ public class ServerRMProxy<T> extends RMProxy<T> {
configuration.getLong(
YarnConfiguration.NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
rmRetryInterval);
- return createRMProxy(configuration, protocol, INSTANCE,
+ ServerRMProxy<T> serverRMProxy = new ServerRMProxy<>();
+ return createRMProxy(configuration, protocol, serverRMProxy,
nmRmConnectWait, nmRmRetryInterval);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] hadoop git commit: HADOOP-14407. DistCp - Introduce a
configurable copy buffer size. (Omkar Aradhya K S via Yongjun Zhang)
Posted by wa...@apache.org.
HADOOP-14407. DistCp - Introduce a configurable copy buffer size. (Omkar Aradhya K S via Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4adc839
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4adc839
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4adc839
Branch: refs/heads/YARN-5734
Commit: b4adc8392c1314d6d6fbdd00f2afb306ef20a650
Parents: 40e6a85
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Thu May 18 15:35:22 2017 -0700
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Thu May 18 15:35:22 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/tools/DistCpConstants.java | 6 +++++
.../org/apache/hadoop/tools/DistCpContext.java | 4 ++++
.../apache/hadoop/tools/DistCpOptionSwitch.java | 8 +++++++
.../org/apache/hadoop/tools/DistCpOptions.java | 23 +++++++++++++++++++-
.../org/apache/hadoop/tools/OptionsParser.java | 12 ++++++++++
.../tools/mapred/RetriableFileCopyCommand.java | 11 ++++++----
.../src/site/markdown/DistCp.md.vm | 1 +
.../apache/hadoop/tools/TestDistCpOptions.java | 19 +++++++++++++++-
8 files changed, 78 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4adc839/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
index ff16e44..7e8e67d 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
@@ -109,6 +109,10 @@ public class DistCpConstants {
/* DistCp CopyListing class override param */
public static final String CONF_LABEL_COPY_LISTING_CLASS = "distcp.copy.listing.class";
+ /* DistCp Copy Buffer Size */
+ public static final String CONF_LABEL_COPY_BUFFER_SIZE =
+ "distcp.copy.buffer.size";
+
/**
* Constants for DistCp return code to shell / consumer of ToolRunner's run
*/
@@ -141,4 +145,6 @@ public class DistCpConstants {
public static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw";
static final String HDFS_DISTCP_DIFF_DIRECTORY_NAME = ".distcp.diff.tmp";
+
+ public static final int COPY_BUFFER_SIZE_DEFAULT = 8 * 1024;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4adc839/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java
index c34005e..fc047ca 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java
@@ -175,6 +175,10 @@ public class DistCpContext {
return options.getBlocksPerChunk() > 0;
}
+ public int getCopyBufferSize() {
+ return options.getCopyBufferSize();
+ }
+
public void setTargetPathExists(boolean targetPathExists) {
this.targetPathExists = targetPathExists;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4adc839/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index 81abb7d..016172e0 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -180,6 +180,14 @@ public enum DistCpOptionSwitch {
+ "system implements concat method")),
/**
+ * Configurable copy buffer size.
+ */
+ COPY_BUFFER_SIZE(DistCpConstants.CONF_LABEL_COPY_BUFFER_SIZE,
+ new Option("copybuffersize", true, "Size of the copy buffer to use. "
+ + "By default <copybuffersize> is "
+ + DistCpConstants.COPY_BUFFER_SIZE_DEFAULT + "B.")),
+
+ /**
* Specify bandwidth per map in MB, accepts bandwidth as a fraction
*/
BANDWIDTH(DistCpConstants.CONF_LABEL_BANDWIDTH_MB,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4adc839/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index 97ae0c4..af6cb8b 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -143,6 +143,8 @@ public final class DistCpOptions {
// to copy in parallel. Default is 0 and file are not splitted.
private final int blocksPerChunk;
+ private final int copyBufferSize;
+
/**
* File attributes for preserve.
*
@@ -200,6 +202,8 @@ public final class DistCpOptions {
this.preserveStatus = builder.preserveStatus;
this.blocksPerChunk = builder.blocksPerChunk;
+
+ this.copyBufferSize = builder.copyBufferSize;
}
public Path getSourceFileListing() {
@@ -302,7 +306,7 @@ public final class DistCpOptions {
}
/**
- * Checks if the input attribute should be preserved or not
+ * Checks if the input attribute should be preserved or not.
*
* @param attribute - Attribute to check
* @return True if attribute should be preserved, false otherwise
@@ -315,6 +319,10 @@ public final class DistCpOptions {
return blocksPerChunk;
}
+ public int getCopyBufferSize() {
+ return copyBufferSize;
+ }
+
/**
* Add options to configuration. These will be used in the Mapper/committer
*
@@ -351,6 +359,8 @@ public final class DistCpOptions {
}
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BLOCKS_PER_CHUNK,
String.valueOf(blocksPerChunk));
+ DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.COPY_BUFFER_SIZE,
+ String.valueOf(copyBufferSize));
}
/**
@@ -385,6 +395,7 @@ public final class DistCpOptions {
", targetPath=" + targetPath +
", filtersFile='" + filtersFile + '\'' +
", blocksPerChunk=" + blocksPerChunk +
+ ", copyBufferSize=" + copyBufferSize +
'}';
}
@@ -429,6 +440,9 @@ public final class DistCpOptions {
private int blocksPerChunk = 0;
+ private int copyBufferSize =
+ DistCpConstants.COPY_BUFFER_SIZE_DEFAULT;
+
public Builder(List<Path> sourcePaths, Path targetPath) {
Preconditions.checkArgument(sourcePaths != null && !sourcePaths.isEmpty(),
"Source paths should not be null or empty!");
@@ -664,6 +678,13 @@ public final class DistCpOptions {
this.blocksPerChunk = newBlocksPerChunk;
return this;
}
+
+ public Builder withCopyBufferSize(int newCopyBufferSize) {
+ this.copyBufferSize =
+ newCopyBufferSize > 0 ? newCopyBufferSize
+ : DistCpConstants.COPY_BUFFER_SIZE_DEFAULT;
+ return this;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4adc839/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
index 21ff0f8..96fb1d9 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
@@ -213,6 +213,18 @@ public class OptionsParser {
}
}
+ if (command.hasOption(DistCpOptionSwitch.COPY_BUFFER_SIZE.getSwitch())) {
+ final String copyBufferSizeStr = getVal(command,
+ DistCpOptionSwitch.COPY_BUFFER_SIZE.getSwitch().trim());
+ try {
+ int copyBufferSize = Integer.parseInt(copyBufferSizeStr);
+ builder.withCopyBufferSize(copyBufferSize);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("copyBufferSize is invalid: "
+ + copyBufferSizeStr, e);
+ }
+ }
+
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4adc839/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
index 2c17fef..21f621a 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.tools.CopyListingFileStatus;
import org.apache.hadoop.tools.DistCpConstants;
+import org.apache.hadoop.tools.DistCpOptionSwitch;
import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.tools.mapred.CopyMapper.FileAction;
import org.apache.hadoop.tools.util.DistCpUtils;
@@ -53,7 +54,6 @@ import com.google.common.annotations.VisibleForTesting;
public class RetriableFileCopyCommand extends RetriableCommand {
private static Log LOG = LogFactory.getLog(RetriableFileCopyCommand.class);
- private static int BUFFER_SIZE = 8 * 1024;
private boolean skipCrc = false;
private FileAction action;
@@ -169,6 +169,9 @@ public class RetriableFileCopyCommand extends RetriableCommand {
throws IOException {
FsPermission permission = FsPermission.getFileDefault().applyUMask(
FsPermission.getUMask(targetFS.getConf()));
+ int copyBufferSize = context.getConfiguration().getInt(
+ DistCpOptionSwitch.COPY_BUFFER_SIZE.getConfigLabel(),
+ DistCpConstants.COPY_BUFFER_SIZE_DEFAULT);
final OutputStream outStream;
if (action == FileAction.OVERWRITE) {
// If there is an erasure coding policy set on the target directory,
@@ -180,14 +183,14 @@ public class RetriableFileCopyCommand extends RetriableCommand {
targetFS, targetPath);
FSDataOutputStream out = targetFS.create(targetPath, permission,
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
- BUFFER_SIZE, repl, blockSize, context,
+ copyBufferSize, repl, blockSize, context,
getChecksumOpt(fileAttributes, sourceChecksum));
outStream = new BufferedOutputStream(out);
} else {
outStream = new BufferedOutputStream(targetFS.append(targetPath,
- BUFFER_SIZE));
+ copyBufferSize));
}
- return copyBytes(source, sourceOffset, outStream, BUFFER_SIZE,
+ return copyBytes(source, sourceOffset, outStream, copyBufferSize,
context);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4adc839/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
index a77deb2..ee0a93e 100644
--- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
+++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
@@ -238,6 +238,7 @@ Flag | Description | Notes
`-numListstatusThreads` | Number of threads to use for building file listing | At most 40 threads.
`-skipcrccheck` | Whether to skip CRC checks between source and target paths. |
`-blocksperchunk <blocksperchunk>` | Number of blocks per chunk. When specified, split files into chunks to copy in parallel | If set to a positive value, files with more blocks than this value will be split into chunks of `<blocksperchunk>` blocks to be transferred in parallel, and reassembled on the destination. By default, `<blocksperchunk>` is 0 and the files will be transmitted in their entirety without splitting. This switch is only applicable when the source file system implements getBlockLocations method and the target file system implements concat method. |
+`-copybuffersize <copybuffersize>` | Size of the copy buffer to use. By default, `<copybuffersize>` is set to 8192B |
Architecture of DistCp
----------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4adc839/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
index 3525194..6b59b97 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
@@ -287,7 +287,7 @@ public class TestDistCpOptions {
"mapBandwidth=0.0, copyStrategy='uniformsize', preserveStatus=[], " +
"atomicWorkPath=null, logPath=null, sourceFileListing=abc, " +
"sourcePaths=null, targetPath=xyz, filtersFile='null'," +
- " blocksPerChunk=0}";
+ " blocksPerChunk=0, copyBufferSize=8192}";
String optionString = option.toString();
Assert.assertEquals(val, optionString);
Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
@@ -497,4 +497,21 @@ public class TestDistCpOptions {
Assert.assertFalse(builder.build().shouldAppend());
}
+ @Test
+ public void testSetCopyBufferSize() {
+ final DistCpOptions.Builder builder = new DistCpOptions.Builder(
+ Collections.singletonList(new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+
+ Assert.assertEquals(DistCpConstants.COPY_BUFFER_SIZE_DEFAULT,
+ builder.build().getCopyBufferSize());
+
+ builder.withCopyBufferSize(4194304);
+ Assert.assertEquals(4194304,
+ builder.build().getCopyBufferSize());
+
+ builder.withCopyBufferSize(-1);
+ Assert.assertEquals(DistCpConstants.COPY_BUFFER_SIZE_DEFAULT,
+ builder.build().getCopyBufferSize());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] hadoop git commit: YARN-6603. NPE in RMAppsBlock. Contributed
by Jason Lowe
Posted by wa...@apache.org.
YARN-6603. NPE in RMAppsBlock. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/489f8593
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/489f8593
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/489f8593
Branch: refs/heads/YARN-5734
Commit: 489f85933c508bc26de607b921e56e23b979fce8
Parents: c48f297
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Tue May 16 09:26:44 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Tue May 16 09:26:44 2017 -0500
----------------------------------------------------------------------
.../server/resourcemanager/webapp/RMAppsBlock.java | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/489f8593/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index 305f1d5..6a18296 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.webapp.AppsBlock;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
@@ -96,13 +97,15 @@ public class RMAppsBlock extends AppsBlock {
}
String blacklistedNodesCount = "N/A";
- RMAppAttempt appAttempt =
- rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
- .getAppAttempts().get(appAttemptId);
- Set<String> nodes =
- null == appAttempt ? null : appAttempt.getBlacklistedNodes();
- if (nodes != null) {
- blacklistedNodesCount = String.valueOf(nodes.size());
+ RMApp rmApp = rm.getRMContext().getRMApps()
+ .get(appAttemptId.getApplicationId());
+ if (rmApp != null) {
+ RMAppAttempt appAttempt = rmApp.getRMAppAttempt(appAttemptId);
+ Set<String> nodes =
+ null == appAttempt ? null : appAttempt.getBlacklistedNodes();
+ if (nodes != null) {
+ blacklistedNodesCount = String.valueOf(nodes.size());
+ }
}
String percent = StringUtils.format("%.1f", app.getProgress());
appsTableData
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] hadoop git commit: YARN-6598. History server
getApplicationReport NPE when fetching report for pre-2.8 job (Jason Lowe via
jeagles)
Posted by wa...@apache.org.
YARN-6598. History server getApplicationReport NPE when fetching report for pre-2.8 job (Jason Lowe via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c48f2976
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c48f2976
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c48f2976
Branch: refs/heads/YARN-5734
Commit: c48f2976a3de60b95c4a5ada4f0131c4cdde177a
Parents: 6600abb
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Mon May 15 10:32:01 2017 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Mon May 15 10:32:01 2017 -0500
----------------------------------------------------------------------
...pplicationHistoryManagerOnTimelineStore.java | 35 ++++++++++++--------
...pplicationHistoryManagerOnTimelineStore.java | 32 ++++++++++++++----
2 files changed, 48 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c48f2976/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 54893ef..d18f3dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -330,20 +330,19 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
- long vcoreSeconds=Long.parseLong(entityInfo.get(
- ApplicationMetricsConstants.APP_CPU_METRICS).toString());
- long memorySeconds=Long.parseLong(entityInfo.get(
- ApplicationMetricsConstants.APP_MEM_METRICS).toString());
- long preemptedMemorySeconds = Long.parseLong(entityInfo.get(
- ApplicationMetricsConstants
- .APP_MEM_PREEMPT_METRICS).toString());
- long preemptedVcoreSeconds = Long.parseLong(entityInfo.get(
- ApplicationMetricsConstants
- .APP_CPU_PREEMPT_METRICS).toString());
- appResources = ApplicationResourceUsageReport
- .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds, 0,
- 0, preemptedMemorySeconds, preemptedVcoreSeconds);
+ long vcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_CPU_METRICS);
+ long memorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_MEM_METRICS);
+ long preemptedMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
+ long preemptedVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
+ appResources = ApplicationResourceUsageReport.newInstance(0, 0, null,
+ null, null, memorySeconds, vcoreSeconds, 0, 0,
+ preemptedMemorySeconds, preemptedVcoreSeconds);
}
+
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
appTags = new HashSet<String>();
Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
@@ -445,6 +444,16 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
amNodeLabelExpression), appViewACLs);
}
+ private static long parseLong(Map<String, Object> entityInfo,
+ String infoKey) {
+ long result = 0;
+ Object infoValue = entityInfo.get(infoKey);
+ if (infoValue != null) {
+ result = Long.parseLong(infoValue.toString());
+ }
+ return result;
+ }
+
private static boolean isFinalState(YarnApplicationState state) {
return state == YarnApplicationState.FINISHED
|| state == YarnApplicationState.FAILED
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c48f2976/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 3d4e91b..9600251 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -143,6 +143,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
if (i == 2) {
entities.addEntity(createApplicationTimelineEntity(
appId, true, false, false, true, YarnApplicationState.FINISHED));
+ } else if (i == 3) {
+ entities.addEntity(createApplicationTimelineEntity(
+ appId, false, false, false, false, YarnApplicationState.FINISHED,
+ true));
} else {
entities.addEntity(createApplicationTimelineEntity(
appId, false, false, false, false, YarnApplicationState.FINISHED));
@@ -176,7 +180,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@Test
public void testGetApplicationReport() throws Exception {
- for (int i = 1; i <= 2; ++i) {
+ for (int i = 1; i <= 3; ++i) {
final ApplicationId appId = ApplicationId.newInstance(0, i);
ApplicationReport app;
if (callerUGI == null) {
@@ -214,7 +218,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
- if ((i == 1 && callerUGI != null &&
+ if ((i != 2 && callerUGI != null &&
callerUGI.getShortUserName().equals("user3")) ||
(i == 2 && callerUGI != null &&
(callerUGI.getShortUserName().equals("user2") ||
@@ -245,10 +249,16 @@ public class TestApplicationHistoryManagerOnTimelineStore {
applicationResourceUsageReport.getMemorySeconds());
Assert
.assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
- Assert.assertEquals(456,
+ long expectedPreemptMemSecs = 456;
+ long expectedPreemptVcoreSecs = 789;
+ if (i == 3) {
+ expectedPreemptMemSecs = 0;
+ expectedPreemptVcoreSecs = 0;
+ }
+ Assert.assertEquals(expectedPreemptMemSecs,
applicationResourceUsageReport.getPreemptedMemorySeconds());
Assert
- .assertEquals(789, applicationResourceUsageReport
+ .assertEquals(expectedPreemptVcoreSecs, applicationResourceUsageReport
.getPreemptedVcoreSeconds());
Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
app.getFinalApplicationStatus());
@@ -486,6 +496,14 @@ public class TestApplicationHistoryManagerOnTimelineStore {
ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
boolean wrongAppId, boolean enableUpdateEvent,
YarnApplicationState state) {
+ return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId,
+ wrongAppId, enableUpdateEvent, state, false);
+ }
+
+ private static TimelineEntity createApplicationTimelineEntity(
+ ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
+ boolean wrongAppId, boolean enableUpdateEvent,
+ YarnApplicationState state, boolean missingPreemptMetrics) {
TimelineEntity entity = new TimelineEntity();
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
if (wrongAppId) {
@@ -510,8 +528,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Integer.MAX_VALUE + 1L);
entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
- entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,456);
- entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,789);
+ if (!missingPreemptMetrics) {
+ entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456);
+ entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789);
+ }
if (emptyACLs) {
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "");
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] hadoop git commit: HDFS-11696. Fix warnings from Spotbugs in
hadoop-hdfs. Contributed by Yiqun Lin.
Posted by wa...@apache.org.
HDFS-11696. Fix warnings from Spotbugs in hadoop-hdfs. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89a8edc0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89a8edc0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89a8edc0
Branch: refs/heads/YARN-5734
Commit: 89a8edc0149e3f31a5ade9a0927c4b6332cf6b1a
Parents: 9b90e52
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue May 16 12:41:59 2017 -0400
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue May 16 12:41:59 2017 -0400
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 7 +++--
.../hdfs/server/protocol/SlowDiskReports.java | 5 ++--
.../dev-support/findbugsExcludeFile.xml | 5 ++++
.../hdfs/qjournal/server/JournalNode.java | 16 +++++++-----
.../hdfs/server/common/HdfsServerConstants.java | 7 ++++-
.../hdfs/server/datanode/DataStorage.java | 12 ++++++---
.../namenode/NNStorageRetentionManager.java | 27 +++++++++++---------
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 6 ++---
.../offlineImageViewer/ImageLoaderCurrent.java | 10 +++++---
9 files changed, 62 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a8edc0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index d21b9b4..7de8b71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2857,9 +2857,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
synchronized (DFSClient.class) {
if (STRIPED_READ_THREAD_POOL == null) {
- STRIPED_READ_THREAD_POOL = DFSUtilClient.getThreadPoolExecutor(1,
+ // Only after thread pool is fully constructed then save it to
+ // volatile field.
+ ThreadPoolExecutor threadPool = DFSUtilClient.getThreadPoolExecutor(1,
numThreads, 60, "StripedRead-", true);
- STRIPED_READ_THREAD_POOL.allowCoreThreadTimeOut(true);
+ threadPool.allowCoreThreadTimeOut(true);
+ STRIPED_READ_THREAD_POOL = threadPool;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a8edc0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java
index 8095c2a..496389a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java
@@ -101,8 +101,9 @@ public final class SlowDiskReports {
}
boolean areEqual;
- for (String disk : this.slowDisks.keySet()) {
- if (!this.slowDisks.get(disk).equals(that.slowDisks.get(disk))) {
+ for (Map.Entry<String, Map<DiskOp, Double>> entry : this.slowDisks
+ .entrySet()) {
+ if (!entry.getValue().equals(that.slowDisks.get(entry.getKey()))) {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a8edc0/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index be54efb..9270990 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -252,4 +252,9 @@
<Class name="org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture" />
<Bug pattern="NS_DANGEROUS_NON_SHORT_CIRCUIT" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.server.namenode.NNUpgradeUtil$1" />
+ <Method name="visitFile" />
+ <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a8edc0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
index 42e9be7..ab724d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
@@ -297,14 +297,18 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
return file.isDirectory();
}
});
- for (File journalDir : journalDirs) {
- String jid = journalDir.getName();
- if (!status.containsKey(jid)) {
- Map<String, String> jMap = new HashMap<String, String>();
- jMap.put("Formatted", "true");
- status.put(jid, jMap);
+
+ if (journalDirs != null) {
+ for (File journalDir : journalDirs) {
+ String jid = journalDir.getName();
+ if (!status.containsKey(jid)) {
+ Map<String, String> jMap = new HashMap<String, String>();
+ jMap.put("Formatted", "true");
+ status.put(jid, jMap);
+ }
}
}
+
return JSON.toString(status);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a8edc0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index c3098f3..ab50eb4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -188,8 +188,10 @@ public interface HdfsServerConstants {
return NamenodeRole.NAMENODE;
}
}
-
+
public void setClusterId(String cid) {
+ Preconditions.checkState(this == UPGRADE || this == UPGRADEONLY
+ || this == FORMAT);
clusterId = cid;
}
@@ -214,6 +216,7 @@ public interface HdfsServerConstants {
}
public void setForce(int force) {
+ Preconditions.checkState(this == RECOVER);
this.force = force;
}
@@ -226,6 +229,7 @@ public interface HdfsServerConstants {
}
public void setForceFormat(boolean force) {
+ Preconditions.checkState(this == FORMAT);
isForceFormat = force;
}
@@ -234,6 +238,7 @@ public interface HdfsServerConstants {
}
public void setInteractiveFormat(boolean interactive) {
+ Preconditions.checkState(this == FORMAT);
isInteractiveFormat = interactive;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a8edc0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 9a71081..6d6e96a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -1336,10 +1336,14 @@ public class DataStorage extends Storage {
return name.startsWith(BLOCK_SUBDIR_PREFIX);
}
});
- for(int i = 0; i < otherNames.length; i++)
- linkBlocksHelper(new File(from, otherNames[i]),
- new File(to, otherNames[i]), oldLV, hl, upgradeToIdBasedLayout,
- blockRoot, idBasedLayoutSingleLinks);
+
+ if (otherNames != null) {
+ for (int i = 0; i < otherNames.length; i++) {
+ linkBlocksHelper(new File(from, otherNames[i]),
+ new File(to, otherNames[i]), oldLV, hl, upgradeToIdBasedLayout,
+ blockRoot, idBasedLayoutSingleLinks);
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a8edc0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
index 98b7e9a..2a83541 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
@@ -255,24 +255,27 @@ public class NNStorageRetentionManager {
});
// Check whether there is any work to do.
- if (filesInStorage.length <= numCheckpointsToRetain) {
+ if (filesInStorage != null
+ && filesInStorage.length <= numCheckpointsToRetain) {
return;
}
// Create a sorted list of txids from the file names.
TreeSet<Long> sortedTxIds = new TreeSet<Long>();
- for (String fName : filesInStorage) {
- // Extract the transaction id from the file name.
- long fTxId;
- try {
- fTxId = Long.parseLong(fName.substring(oivImagePrefix.length() + 1));
- } catch (NumberFormatException nfe) {
- // This should not happen since we have already filtered it.
- // Log and continue.
- LOG.warn("Invalid file name. Skipping " + fName);
- continue;
+ if (filesInStorage != null) {
+ for (String fName : filesInStorage) {
+ // Extract the transaction id from the file name.
+ long fTxId;
+ try {
+ fTxId = Long.parseLong(fName.substring(oivImagePrefix.length() + 1));
+ } catch (NumberFormatException nfe) {
+ // This should not happen since we have already filtered it.
+ // Log and continue.
+ LOG.warn("Invalid file name. Skipping " + fName);
+ continue;
+ }
+ sortedTxIds.add(Long.valueOf(fTxId));
}
- sortedTxIds.add(Long.valueOf(fTxId));
}
int numFilesToDelete = sortedTxIds.size() - numCheckpointsToRetain;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a8edc0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index 6d67089..5375cd9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -1917,7 +1917,7 @@ public class DFSAdmin extends FsShell {
return exitCode;
}
} else if ("-report".equals(cmd)) {
- if (argv.length < 1) {
+ if (argv.length > 4) {
printUsage(cmd);
return exitCode;
}
@@ -1947,7 +1947,7 @@ public class DFSAdmin extends FsShell {
return exitCode;
}
} else if (RollingUpgradeCommand.matches(cmd)) {
- if (argv.length < 1 || argv.length > 2) {
+ if (argv.length > 2) {
printUsage(cmd);
return exitCode;
}
@@ -2022,7 +2022,7 @@ public class DFSAdmin extends FsShell {
return exitCode;
}
} else if ("-triggerBlockReport".equals(cmd)) {
- if (argv.length < 1) {
+ if ((argv.length != 2) && (argv.length != 3)) {
printUsage(cmd);
return exitCode;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a8edc0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
index f2c7427..2e2eaf4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
@@ -722,9 +722,13 @@ class ImageLoaderCurrent implements ImageLoader {
if (supportSnapshot && supportInodeId) {
dirNodeMap.put(inodeId, pathName);
}
- v.visit(ImageElement.NS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
- if (NameNodeLayoutVersion.supports(Feature.DISKSPACE_QUOTA, imageVersion))
- v.visit(ImageElement.DS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
+
+ v.visit(ImageElement.NS_QUOTA, in.readLong());
+ if (NameNodeLayoutVersion.supports(Feature.DISKSPACE_QUOTA,
+ imageVersion)) {
+ v.visit(ImageElement.DS_QUOTA, in.readLong());
+ }
+
if (supportSnapshot) {
boolean snapshottable = in.readBoolean();
if (!snapshottable) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] hadoop git commit: HDFS-11827. NPE is thrown when log level
changed in BlockPlacementPolicyDefault#chooseRandom() method. Contributed by
xupeng.
Posted by wa...@apache.org.
HDFS-11827. NPE is thrown when log level changed in BlockPlacementPolicyDefault#chooseRandom() method. Contributed by xupeng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec21ce42
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec21ce42
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec21ce42
Branch: refs/heads/YARN-5734
Commit: ec21ce425f4e5637be716406f9d0e038921550d7
Parents: 18c494a
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue May 16 19:51:04 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue May 16 19:51:04 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec21ce42/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index a245f0c..a479397 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -735,7 +735,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
Preconditions.checkState(excludedNodes.add(chosenNode), "chosenNode "
+ chosenNode + " is already in excludedNodes " + excludedNodes);
- if (LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled() && builder != null) {
builder.append("\nNode ").append(NodeBase.getPath(chosenNode))
.append(" [");
}
@@ -771,7 +771,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
}
- if (LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled() && builder != null) {
builder.append("\n]");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] hadoop git commit: YARN-5951. Changes to allow
CapacityScheduler to use configuration store
Posted by wa...@apache.org.
YARN-5951. Changes to allow CapacityScheduler to use configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e485218
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e485218
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e485218
Branch: refs/heads/YARN-5734
Commit: 8e4852187cfe0f78fc1f5951617b8aa1acd72449
Parents: e135c0b
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Mon Jan 30 19:03:48 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Sat May 20 08:22:43 2017 -0700
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 36 +++++------
.../CapacitySchedulerConfiguration.java | 10 +++
.../capacity/conf/CSConfigurationProvider.java | 46 ++++++++++++++
.../conf/FileBasedCSConfigurationProvider.java | 67 ++++++++++++++++++++
.../scheduler/capacity/conf/package-info.java | 29 +++++++++
.../capacity/TestCapacityScheduler.java | 4 +-
6 files changed, 170 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e485218/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index d3186da..a925904 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
@@ -104,6 +103,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation;
@@ -162,6 +163,7 @@ public class CapacityScheduler extends
private int offswitchPerHeartbeatLimit;
+ private CSConfigurationProvider csConfProvider;
@Override
public void setConf(Configuration conf) {
@@ -285,7 +287,18 @@ public class CapacityScheduler extends
IOException {
try {
writeLock.lock();
- this.conf = loadCapacitySchedulerConfiguration(configuration);
+ String confProviderStr = configuration.get(
+ CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
+ if (confProviderStr.equals(
+ CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) {
+ this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext);
+ } else {
+ throw new IOException("Invalid CS configuration provider: " +
+ confProviderStr);
+ }
+ this.csConfProvider.init(configuration);
+ this.conf = this.csConfProvider.loadConfiguration(configuration);
validateConf(this.conf);
this.minimumAllocation = this.conf.getMinimumAllocation();
initMaximumResourceCapability(this.conf.getMaximumAllocation());
@@ -392,7 +405,7 @@ public class CapacityScheduler extends
writeLock.lock();
Configuration configuration = new Configuration(newConf);
CapacitySchedulerConfiguration oldConf = this.conf;
- this.conf = loadCapacitySchedulerConfiguration(configuration);
+ this.conf = csConfProvider.loadConfiguration(configuration);
validateConf(this.conf);
try {
LOG.info("Re-initializing queues...");
@@ -1754,23 +1767,6 @@ public class CapacityScheduler extends
return true;
}
- private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration(
- Configuration configuration) throws IOException {
- try {
- InputStream CSInputStream =
- this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(configuration,
- YarnConfiguration.CS_CONFIGURATION_FILE);
- if (CSInputStream != null) {
- configuration.addResource(CSInputStream);
- return new CapacitySchedulerConfiguration(configuration, false);
- }
- return new CapacitySchedulerConfiguration(configuration, true);
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
private String getDefaultReservationQueueName(String planQueueName) {
return planQueueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e485218/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 9fb92ec..c53a588 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -292,6 +292,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final boolean DEFAULT_LAZY_PREEMPTION_ENABLED = false;
+ @Private
+ public static final String CS_CONF_PROVIDER = PREFIX
+ + "configuration.provider";
+
+ @Private
+ public static final String FILE_CS_CONF_PROVIDER = "file";
+
+ @Private
+ public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
+
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
public CapacitySchedulerConfiguration() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e485218/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
new file mode 100644
index 0000000..c9984ac
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import java.io.IOException;
+
+/**
+ * Configuration provider for {@link CapacityScheduler}.
+ */
+public interface CSConfigurationProvider {
+
+ /**
+ * Initialize the configuration provider with given conf.
+ * @param conf configuration to initialize with
+ */
+ void init(Configuration conf);
+
+ /**
+ * Loads capacity scheduler configuration object.
+ * @param conf initial bootstrap configuration
+ * @return CS configuration
+ * @throws IOException if fail to retrieve configuration
+ */
+ CapacitySchedulerConfiguration loadConfiguration(Configuration conf)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e485218/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
new file mode 100644
index 0000000..51c64fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * {@link CapacityScheduler} configuration provider based on local
+ * {@code capacity-scheduler.xml} file.
+ */
+public class FileBasedCSConfigurationProvider implements
+ CSConfigurationProvider {
+
+ private RMContext rmContext;
+
+ /**
+ * Construct file based CS configuration provider with given context.
+ * @param rmContext the RM context
+ */
+ public FileBasedCSConfigurationProvider(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void init(Configuration conf) {}
+
+ @Override
+ public CapacitySchedulerConfiguration loadConfiguration(Configuration conf)
+ throws IOException {
+ try {
+ InputStream csInputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.CS_CONFIGURATION_FILE);
+ if (csInputStream != null) {
+ conf.addResource(csInputStream);
+ return new CapacitySchedulerConfiguration(conf, false);
+ }
+ return new CapacitySchedulerConfiguration(conf, true);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e485218/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
new file mode 100644
index 0000000..08d0522
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package
+ * org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf
+ * contains classes related to capacity scheduler configuration management.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e485218/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index bf1f6eb..5de6c6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -231,13 +231,13 @@ public class TestCapacityScheduler {
@Test (timeout = 30000)
public void testConfValidation() throws Exception {
- ResourceScheduler scheduler = new CapacityScheduler();
+ CapacityScheduler scheduler = new CapacityScheduler();
scheduler.setRMContext(resourceManager.getRMContext());
Configuration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
try {
- scheduler.reinitialize(conf, mockContext);
+ scheduler.init(conf);
fail("Exception is expected because the min memory allocation is" +
" larger than the max memory allocation.");
} catch (YarnRuntimeException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] hadoop git commit: YARN-5946: Create YarnConfigurationStore
interface and InMemoryConfigurationStore class. Contributed by Jonathan Hung
Posted by wa...@apache.org.
YARN-5946: Create YarnConfigurationStore interface and
InMemoryConfigurationStore class. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9cf1c93c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9cf1c93c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9cf1c93c
Branch: refs/heads/YARN-5734
Commit: 9cf1c93cc8d6033cf012c7c94a9cb2fbd5658f43
Parents: 8e48521
Author: Xuan <xg...@apache.org>
Authored: Fri Feb 24 15:58:12 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Sat May 20 08:22:43 2017 -0700
----------------------------------------------------------------------
.../conf/InMemoryConfigurationStore.java | 86 +++++++++++
.../capacity/conf/YarnConfigurationStore.java | 154 +++++++++++++++++++
.../conf/TestYarnConfigurationStore.java | 70 +++++++++
3 files changed, 310 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9cf1c93c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
new file mode 100644
index 0000000..a208fb9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A default implementation of {@link YarnConfigurationStore}. Doesn't offer
+ * persistent configuration storage, just stores the configuration in memory.
+ */
+public class InMemoryConfigurationStore implements YarnConfigurationStore {
+
+ private Configuration schedConf;
+ private LinkedList<LogMutation> pendingMutations;
+ private long pendingId;
+
+ @Override
+ public void initialize(Configuration conf, Configuration schedConf) {
+ this.schedConf = schedConf;
+ this.pendingMutations = new LinkedList<>();
+ this.pendingId = 0;
+ }
+
+ @Override
+ public synchronized long logMutation(LogMutation logMutation) {
+ logMutation.setId(++pendingId);
+ pendingMutations.add(logMutation);
+ return pendingId;
+ }
+
+ @Override
+ public synchronized boolean confirmMutation(long id, boolean isValid) {
+ LogMutation mutation = pendingMutations.poll();
+ // If confirmMutation is called out of order, discard mutations until id
+ // is reached.
+ while (mutation != null) {
+ if (mutation.getId() == id) {
+ if (isValid) {
+ Map<String, String> mutations = mutation.getUpdates();
+ for (Map.Entry<String, String> kv : mutations.entrySet()) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ }
+ return true;
+ }
+ mutation = pendingMutations.poll();
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized Configuration retrieve() {
+ return schedConf;
+ }
+
+ @Override
+ public synchronized List<LogMutation> getPendingMutations() {
+ return pendingMutations;
+ }
+
+ @Override
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ // Unimplemented.
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9cf1c93c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
new file mode 100644
index 0000000..22c0ef8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * YarnConfigurationStore exposes the methods needed for retrieving and
+ * persisting {@link CapacityScheduler} configuration via key-value
+ * using write-ahead logging. When configuration mutation is requested, caller
+ * should first log it with {@code logMutation}, which persists this pending
+ * mutation. This mutation is merged to the persisted configuration only after
+ * {@code confirmMutation} is called.
+ *
+ * On startup/recovery, caller should call {@code retrieve} to get all
+ * confirmed mutations, then get pending mutations which were not confirmed via
+ * {@code getPendingMutations}, and replay/confirm them via
+ * {@code confirmMutation} as in the normal case.
+ */
+public interface YarnConfigurationStore {
+
+ /**
+ * LogMutation encapsulates the fields needed for configuration mutation
+ * audit logging and recovery.
+ */
+ class LogMutation {
+ private Map<String, String> updates;
+ private String user;
+ private long id;
+
+ /**
+ * Create log mutation prior to logging.
+ * @param updates key-value configuration updates
+ * @param user user who requested configuration change
+ */
+ public LogMutation(Map<String, String> updates, String user) {
+ this(updates, user, 0);
+ }
+
+ /**
+ * Create log mutation for recovery.
+ * @param updates key-value configuration updates
+ * @param user user who requested configuration change
+ * @param id transaction id of configuration change
+ */
+ LogMutation(Map<String, String> updates, String user, long id) {
+ this.updates = updates;
+ this.user = user;
+ this.id = id;
+ }
+
+ /**
+ * Get key-value configuration updates.
+ * @return map of configuration updates
+ */
+ public Map<String, String> getUpdates() {
+ return updates;
+ }
+
+ /**
+ * Get user who requested configuration change.
+ * @return user who requested configuration change
+ */
+ public String getUser() {
+ return user;
+ }
+
+ /**
+ * Get transaction id of this configuration change.
+ * @return transaction id
+ */
+ public long getId() {
+ return id;
+ }
+
+ /**
+ * Set transaction id of this configuration change.
+ * @param id transaction id
+ */
+ public void setId(long id) {
+ this.id = id;
+ }
+ }
+
+ /**
+ * Initialize the configuration store.
+ * @param conf configuration to initialize store with
+ * @param schedConf Initial key-value configuration to persist
+ */
+ void initialize(Configuration conf, Configuration schedConf);
+
+ /**
+ * Logs the configuration change to backing store. Generates an id associated
+ * with this mutation, sets it in {@code logMutation}, and returns it.
+ * @param logMutation configuration change to be persisted in write ahead log
+ * @return id which configuration store associates with this mutation
+ */
+ long logMutation(LogMutation logMutation);
+
+ /**
+ * Should be called after {@code logMutation}. Gets the pending mutation
+ * associated with {@code id} and marks the mutation as persisted (no longer
+ * pending). If isValid is true, merge the mutation with the persisted
+ * configuration.
+ *
+ * If {@code confirmMutation} is called with ids in a different order than
+ * was returned by {@code logMutation}, the result is implementation
+ * dependent.
+ * @param id id of mutation to be confirmed
+ * @param isValid if true, update persisted configuration with mutation
+ * associated with {@code id}.
+ * @return true on success
+ */
+ boolean confirmMutation(long id, boolean isValid);
+
+ /**
+ * Retrieve the persisted configuration.
+ * @return configuration as key-value
+ */
+ Configuration retrieve();
+
+ /**
+ * Get the list of pending mutations, in the order they were logged.
+ * @return list of mutations
+ */
+ List<LogMutation> getPendingMutations();
+
+ /**
+ * Get a list of confirmed configuration mutations starting from a given id.
+ * @param fromId id from which to start getting mutations, inclusive
+ * @return list of configuration mutations
+ */
+ List<LogMutation> getConfirmedConfHistory(long fromId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9cf1c93c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
new file mode 100644
index 0000000..dff4e77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class TestYarnConfigurationStore {
+
+ private YarnConfigurationStore confStore;
+ private Configuration schedConf;
+
+ private static final String testUser = "testUser";
+
+ @Before
+ public void setUp() {
+ schedConf = new Configuration(false);
+ schedConf.set("key1", "val1");
+ }
+
+ @Test
+ public void testInMemoryConfigurationStore() {
+ confStore = new InMemoryConfigurationStore();
+ confStore.initialize(new Configuration(), schedConf);
+ assertEquals("val1", confStore.retrieve().get("key1"));
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("keyUpdate1", "valUpdate1");
+ LogMutation mutation1 = new LogMutation(update1, testUser);
+ long id = confStore.logMutation(mutation1);
+ assertEquals(1, confStore.getPendingMutations().size());
+ confStore.confirmMutation(id, true);
+ assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1"));
+ assertEquals(0, confStore.getPendingMutations().size());
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("keyUpdate2", "valUpdate2");
+ LogMutation mutation2 = new LogMutation(update2, testUser);
+ id = confStore.logMutation(mutation2);
+ assertEquals(1, confStore.getPendingMutations().size());
+ confStore.confirmMutation(id, false);
+ assertNull("Configuration should not be updated",
+ confStore.retrieve().get("keyUpdate2"));
+ assertEquals(0, confStore.getPendingMutations().size());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] hadoop git commit: HADOOP-14419. Remove findbugs report from
docs profile. Contributed by Andras Bokor.
Posted by wa...@apache.org.
HADOOP-14419. Remove findbugs report from docs profile. Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6f6682a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6f6682a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6f6682a
Branch: refs/heads/YARN-5734
Commit: e6f6682aa864635b9f70b387872701c064455ad5
Parents: d87a63a
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed May 17 07:35:41 2017 -0400
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed May 17 07:35:41 2017 -0400
----------------------------------------------------------------------
BUILDING.txt | 2 --
hadoop-project-dist/pom.xml | 20 --------------------
2 files changed, 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6f6682a/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 57dad32..f9cc842 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -6,7 +6,6 @@ Requirements:
* Unix System
* JDK 1.8+
* Maven 3.3 or later
-* Findbugs 1.3.9 (if running findbugs)
* ProtocolBuffer 2.5.0
* CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac
* Zlib devel (if compiling native code)
@@ -345,7 +344,6 @@ Requirements:
* Windows System
* JDK 1.8+
* Maven 3.0 or later
-* Findbugs 1.3.9 (if running findbugs)
* ProtocolBuffer 2.5.0
* CMake 2.6 or newer
* Windows SDK 7.1 or Visual Studio 2010 Professional
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6f6682a/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 988d369..8f386e7 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -167,21 +167,6 @@
</executions>
</plugin>
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>findbugs</goal>
- </goals>
- <phase>prepare-package</phase>
- </execution>
- </executions>
- <configuration>
- <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
- </configuration>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
@@ -298,11 +283,6 @@
<path refid="maven.compile.classpath"/>
</classpath>
</javadoc>
-
- <xslt style="${env.FINDBUGS_HOME}/src/xsl/default.xsl"
- in="${project.build.directory}/findbugsXml.xml"
- out="${project.build.directory}/site/findbugs.html"/>
-
</target>
</configuration>
</execution>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] hadoop git commit: YARN-6380. FSAppAttempt keeps redundant
copy of the queue
Posted by wa...@apache.org.
YARN-6380. FSAppAttempt keeps redundant copy of the queue
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90cb5b46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90cb5b46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90cb5b46
Branch: refs/heads/YARN-5734
Commit: 90cb5b4635b7a0849912458afad052f569131a59
Parents: 2ada100
Author: Daniel Templeton <te...@apache.org>
Authored: Thu May 11 13:47:02 2017 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Thu May 11 13:47:02 2017 -0700
----------------------------------------------------------------------
.../scheduler/fair/FSAppAttempt.java | 21 ++++++++------------
1 file changed, 8 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90cb5b46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index a1c4b4b..4f7e164 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFini
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
@@ -79,7 +78,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
private final ResourceWeights resourceWeights;
private Resource demand = Resources.createResource(0);
private final FairScheduler scheduler;
- private FSQueue fsQueue;
private Resource fairShare = Resources.createResource(0, 0);
// Preemption related variables
@@ -118,7 +116,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
this.scheduler = scheduler;
- this.fsQueue = queue;
this.startTime = scheduler.getClock().getTime();
this.lastTimeAtFairShare = this.startTime;
this.appPriority = Priority.newInstance(1);
@@ -229,11 +226,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
*/
@Override
public Resource getHeadroom() {
- final FSQueue queue = (FSQueue) this.queue;
- SchedulingPolicy policy = queue.getPolicy();
+ final FSQueue fsQueue = getQueue();
+ SchedulingPolicy policy = fsQueue.getPolicy();
- Resource queueFairShare = queue.getFairShare();
- Resource queueUsage = queue.getResourceUsage();
+ Resource queueFairShare = fsQueue.getFairShare();
+ Resource queueUsage = fsQueue.getResourceUsage();
Resource clusterResource = this.scheduler.getClusterResource();
Resource clusterUsage = this.scheduler.getRootQueueMetrics()
.getAllocatedResources();
@@ -243,7 +240,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
subtractResourcesOnBlacklistedNodes(clusterAvailableResources);
Resource queueMaxAvailableResources =
- Resources.subtract(queue.getMaxShare(), queueUsage);
+ Resources.subtract(fsQueue.getMaxShare(), queueUsage);
Resource maxAvailableResource = Resources.componentwiseMin(
clusterAvailableResources, queueMaxAvailableResources);
@@ -514,8 +511,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
@Override
public FSLeafQueue getQueue() {
- Queue queue = super.getQueue();
- assert queue instanceof FSLeafQueue;
return (FSLeafQueue) queue;
}
@@ -1102,7 +1097,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Resource fairShareStarvation() {
long now = scheduler.getClock().getTime();
Resource threshold = Resources.multiply(
- getFairShare(), fsQueue.getFairSharePreemptionThreshold());
+ getFairShare(), getQueue().getFairSharePreemptionThreshold());
Resource fairDemand = Resources.componentwiseMin(threshold, demand);
// Check if the queue is starved for fairshare
@@ -1113,7 +1108,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
if (!starved ||
- now - lastTimeAtFairShare < fsQueue.getFairSharePreemptionTimeout()) {
+ now - lastTimeAtFairShare < getQueue().getFairSharePreemptionTimeout()) {
fairshareStarvation = Resources.none();
} else {
// The app has been starved for longer than preemption-timeout.
@@ -1128,7 +1123,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* {@code share}.
*/
private boolean isUsageBelowShare(Resource usage, Resource share) {
- return fsQueue.getPolicy().getResourceCalculator().compare(
+ return getQueue().getPolicy().getResourceCalculator().compare(
scheduler.getClusterResource(), usage, share, true) < 0;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] hadoop git commit: YARN-6580. Incorrect logger for
FairSharePolicy. (Vrushali C via Haibo Chen)
Posted by wa...@apache.org.
YARN-6580. Incorrect logger for FairSharePolicy. (Vrushali C via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c35001b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c35001b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c35001b
Branch: refs/heads/YARN-5734
Commit: 6c35001b9f93fd85a02c3465e87bfd1612f4cce9
Parents: a9e24a1
Author: Haibo Chen <ha...@cloudera.com>
Authored: Fri May 12 13:05:40 2017 -0700
Committer: Haibo Chen <ha...@cloudera.com>
Committed: Fri May 12 13:05:40 2017 -0700
----------------------------------------------------------------------
.../resourcemanager/scheduler/fair/policies/FairSharePolicy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c35001b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index f8cdb45..c3ec47a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -42,7 +42,7 @@ import com.google.common.annotations.VisibleForTesting;
@Private
@Unstable
public class FairSharePolicy extends SchedulingPolicy {
- private static final Log LOG = LogFactory.getLog(FifoPolicy.class);
+ private static final Log LOG = LogFactory.getLog(FairSharePolicy.class);
@VisibleForTesting
public static final String NAME = "fair";
private static final DefaultResourceCalculator RESOURCE_CALCULATOR =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] hadoop git commit: HADOOP-14413. Add Javadoc comment for
jitter parameter on CachingGetSpaceUsed (Contributed by Erik Krogen via
Daniel Templeton)
Posted by wa...@apache.org.
HADOOP-14413. Add Javadoc comment for jitter parameter on CachingGetSpaceUsed
(Contributed by Erik Krogen via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09f28da2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09f28da2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09f28da2
Branch: refs/heads/YARN-5734
Commit: 09f28da2d2aae1487bae85ff110794f8346f92a4
Parents: 4aae2d4
Author: Daniel Templeton <te...@apache.org>
Authored: Thu May 11 11:02:20 2017 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Thu May 11 12:06:06 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09f28da2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
index a2b6980..92476d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
@@ -65,6 +65,9 @@ public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed {
*
* @param path the path to check disk usage in
* @param interval refresh the disk usage at this interval
+ * @param jitter randomize the refresh interval timing by this amount;
+ * the actual interval will be chosen uniformly between
+ * {@code interval-jitter} and {@code interval+jitter}
* @param initialUsed use this value until next refresh
* @throws IOException if we fail to refresh the disk usage
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] hadoop git commit: YARN-6627. Use deployed webapp folder to
launch new YARN UI. Contributed by Sunil G
Posted by wa...@apache.org.
YARN-6627. Use deployed webapp folder to launch new YARN UI. Contributed by Sunil G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e135c0b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e135c0b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e135c0b2
Branch: refs/heads/YARN-5734
Commit: e135c0b20794705dabf86a296d934eb8df02a774
Parents: 9855225
Author: Jian He <ji...@apache.org>
Authored: Fri May 19 20:12:06 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Fri May 19 20:14:03 2017 -0700
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 5 ++-
.../server/resourcemanager/ResourceManager.java | 34 ++++++++++++++------
2 files changed, 29 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e135c0b2/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 05e1a93..74ce9bc 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -180,7 +180,7 @@
</fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-ui/target/hadoop-yarn-ui-${project.version}</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/webapps/rm</outputDirectory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/webapps/ui2</outputDirectory>
<includes>
<include>**/*</include>
</includes>
@@ -188,6 +188,9 @@
</fileSets>
<moduleSets>
<moduleSet>
+ <excludes>
+ <exclude>org.apache.hadoop:hadoop-yarn-ui</exclude>
+ </excludes>
<binaries>
<outputDirectory>share/hadoop/${hadoop.component}</outputDirectory>
<includeDependencies>false</includeDependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e135c0b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 1f5e8cd..8f2c121 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -113,6 +113,8 @@ import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
import org.eclipse.jetty.webapp.WebAppContext;
import com.google.common.annotations.VisibleForTesting;
+
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
@@ -1068,33 +1070,47 @@ public class ResourceManager extends CompositeService implements Recoverable {
WebAppContext uiWebAppContext = null;
if (getConfig().getBoolean(YarnConfiguration.YARN_WEBAPP_UI2_ENABLE,
YarnConfiguration.DEFAULT_YARN_WEBAPP_UI2_ENABLE)) {
- String webPath = UI2_WEBAPP_NAME;
String onDiskPath = getConfig()
.get(YarnConfiguration.YARN_WEBAPP_UI2_WARFILE_PATH);
+ uiWebAppContext = new WebAppContext();
+ uiWebAppContext.setContextPath(UI2_WEBAPP_NAME);
+
if (null == onDiskPath) {
String war = "hadoop-yarn-ui-" + VersionInfo.getVersion() + ".war";
URLClassLoader cl = (URLClassLoader) ClassLoader.getSystemClassLoader();
URL url = cl.findResource(war);
if (null == url) {
- onDiskPath = "";
+ onDiskPath = getWebAppsPath("ui2");
} else {
onDiskPath = url.getFile();
}
-
- LOG.info(
- "New web UI war file name:" + war + ", and path:" + onDiskPath);
}
-
- uiWebAppContext = new WebAppContext();
- uiWebAppContext.setContextPath(webPath);
- uiWebAppContext.setWar(onDiskPath);
+ if (onDiskPath == null || onDiskPath.isEmpty()) {
+ LOG.error("No war file or webapps found for ui2 !");
+ } else {
+ if (onDiskPath.endsWith(".war")) {
+ uiWebAppContext.setWar(onDiskPath);
+ LOG.info("Using war file at: " + onDiskPath);
+ } else {
+ uiWebAppContext.setResourceBase(onDiskPath);
+ LOG.info("Using webapps at: " + onDiskPath);
+ }
+ }
}
webApp = builder.start(new RMWebApp(this), uiWebAppContext);
}
+ private String getWebAppsPath(String appName) {
+ URL url = getClass().getClassLoader().getResource("webapps/" + appName);
+ if (url == null) {
+ return "";
+ }
+ return url.toString();
+ }
+
/**
* Helper method to create and init {@link #activeServices}. This creates an
* instance of {@link RMActiveServices} and initializes it.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] hadoop git commit: YARN-6306. NMClient API change for
container upgrade. Contributed by Arun Suresh
Posted by wa...@apache.org.
YARN-6306. NMClient API change for container upgrade. Contributed by Arun Suresh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8236130b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8236130b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8236130b
Branch: refs/heads/YARN-5734
Commit: 8236130b2c61ab0ee9b8ed747ce8cf96af7f17aa
Parents: 89a8edc
Author: Jian He <ji...@apache.org>
Authored: Tue May 16 10:48:46 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Tue May 16 10:48:46 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/api/NMClient.java | 86 ++++-
.../yarn/client/api/async/NMClientAsync.java | 98 ++++-
.../api/async/impl/NMClientAsyncImpl.java | 260 +++++++++++++-
.../yarn/client/api/impl/NMClientImpl.java | 91 +++++
.../api/async/impl/TestNMClientAsync.java | 359 +++++++++++++++----
.../yarn/client/api/impl/TestNMClient.java | 141 ++++++++
.../container/ContainerImpl.java | 9 +-
7 files changed, 966 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8236130b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
index 47270f5..c1447ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
@@ -58,6 +58,10 @@ public abstract class NMClient extends AbstractService {
return client;
}
+ protected enum UpgradeOp {
+ REINIT, RESTART, COMMIT, ROLLBACK
+ }
+
private NMTokenCache nmTokenCache = NMTokenCache.getSingleton();
@Private
@@ -79,8 +83,8 @@ public abstract class NMClient extends AbstractService {
* <code>NodeManager</code> to launch the
* container
* @return a map between the auxiliary service names and their outputs
- * @throws YarnException
- * @throws IOException
+ * @throws YarnException YarnException.
+ * @throws IOException IOException.
*/
public abstract Map<String, ByteBuffer> startContainer(Container container,
ContainerLaunchContext containerLaunchContext)
@@ -95,9 +99,10 @@ public abstract class NMClient extends AbstractService {
* {@link Container}.
* </p>
*
- * @param container the container with updated token
- * @throws YarnException
- * @throws IOException
+ * @param container the container with updated token.
+ *
+ * @throws YarnException YarnException.
+ * @throws IOException IOException.
*/
public abstract void increaseContainerResource(Container container)
throws YarnException, IOException;
@@ -107,9 +112,9 @@ public abstract class NMClient extends AbstractService {
*
* @param containerId the Id of the started container
* @param nodeId the Id of the <code>NodeManager</code>
- *
- * @throws YarnException
- * @throws IOException
+ *
+ * @throws YarnException YarnException.
+ * @throws IOException IOException.
*/
public abstract void stopContainer(ContainerId containerId, NodeId nodeId)
throws YarnException, IOException;
@@ -120,14 +125,62 @@ public abstract class NMClient extends AbstractService {
* @param containerId the Id of the started container
* @param nodeId the Id of the <code>NodeManager</code>
*
- * @return the status of a container
- * @throws YarnException
- * @throws IOException
+ * @return the status of a container.
+ *
+ * @throws YarnException YarnException.
+ * @throws IOException IOException.
*/
public abstract ContainerStatus getContainerStatus(ContainerId containerId,
NodeId nodeId) throws YarnException, IOException;
/**
+ * <p>Re-Initialize the Container.</p>
+ *
+ * @param containerId the Id of the container to Re-Initialize.
+ * @param containerLaunchContex the updated ContainerLaunchContext.
+ * @param autoCommit commit re-initialization automatically ?
+ *
+ * @throws YarnException YarnException.
+ * @throws IOException IOException.
+ */
+ public abstract void reInitializeContainer(ContainerId containerId,
+ ContainerLaunchContext containerLaunchContex, boolean autoCommit)
+ throws YarnException, IOException;
+
+ /**
+ * <p>Restart the specified container.</p>
+ *
+ * @param containerId the Id of the container to restart.
+ *
+ * @throws YarnException YarnException.
+ * @throws IOException IOException.
+ */
+ public abstract void restartContainer(ContainerId containerId)
+ throws YarnException, IOException;
+
+ /**
+ * <p>Rollback last reInitialization of the specified container.</p>
+ *
+ * @param containerId the Id of the container to restart.
+ *
+ * @throws YarnException YarnException.
+ * @throws IOException IOException.
+ */
+ public abstract void rollbackLastReInitialization(ContainerId containerId)
+ throws YarnException, IOException;
+
+ /**
+ * <p>Commit last reInitialization of the specified container.</p>
+ *
+ * @param containerId the Id of the container to commit reInitialize.
+ *
+ * @throws YarnException YarnException.
+ * @throws IOException IOException.
+ */
+ public abstract void commitLastReInitialization(ContainerId containerId)
+ throws YarnException, IOException;
+
+ /**
* <p>Set whether the containers that are started by this client, and are
* still running should be stopped when the client stops. By default, the
* feature should be enabled.</p> However, containers will be stopped only
@@ -165,4 +218,15 @@ public abstract class NMClient extends AbstractService {
return nmTokenCache;
}
+ /**
+ * Get the NodeId of the node on which container is running. It returns
+ * null if the container if container is not found or if it is not running.
+ *
+ * @param containerId Container Id of the container.
+ * @return NodeId of the container on which it is running.
+ */
+ public NodeId getNodeIdOfStartedContainer(ContainerId containerId) {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8236130b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
index 8e90564..c94942a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.client.api.async;
import java.nio.ByteBuffer;
import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -32,7 +31,6 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.client.api.NMClient;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.client.api.impl.NMClientImpl;
@@ -181,6 +179,38 @@ public abstract class NMClientAsync extends AbstractService {
public abstract void increaseContainerResourceAsync(Container container);
+ /**
+ * <p>Re-Initialize the Container.</p>
+ *
+ * @param containerId the Id of the container to Re-Initialize.
+ * @param containerLaunchContex the updated ContainerLaunchContext.
+ * @param autoCommit commit re-initialization automatically ?
+ */
+ public abstract void reInitializeContainerAsync(ContainerId containerId,
+ ContainerLaunchContext containerLaunchContex, boolean autoCommit);
+
+ /**
+ * <p>Restart the specified container.</p>
+ *
+ * @param containerId the Id of the container to restart.
+ */
+ public abstract void restartContainerAsync(ContainerId containerId);
+
+ /**
+ * <p>Rollback last reInitialization of the specified container.</p>
+ *
+ * @param containerId the Id of the container to restart.
+ */
+ public abstract void rollbackLastReInitializationAsync(
+ ContainerId containerId);
+
+ /**
+ * <p>Commit last reInitialization of the specified container.</p>
+ *
+ * @param containerId the Id of the container to commit reInitialize.
+ */
+ public abstract void commitLastReInitializationAsync(ContainerId containerId);
+
public abstract void stopContainerAsync(
ContainerId containerId, NodeId nodeId);
@@ -303,6 +333,70 @@ public abstract class NMClientAsync extends AbstractService {
*/
public abstract void onStopContainerError(
ContainerId containerId, Throwable t);
+
+ /**
+ * Callback for container re-initialization request.
+ *
+ * @param containerId the Id of the container to be Re-Initialized.
+ */
+ public void onContainerReInitialize(ContainerId containerId) {}
+
+ /**
+ * Callback for container restart.
+ *
+ * @param containerId the Id of the container to restart.
+ */
+ public void onContainerRestart(ContainerId containerId) {}
+
+ /**
+ * Callback for rollback of last re-initialization.
+ *
+ * @param containerId the Id of the container to restart.
+ */
+ public void onRollbackLastReInitialization(ContainerId containerId) {}
+
+ /**
+ * Callback for commit of last re-initialization.
+ *
+ * @param containerId the Id of the container to commit reInitialize.
+ */
+ public void onCommitLastReInitialization(ContainerId containerId) {}
+
+ /**
+ * Error Callback for container re-initialization request.
+ *
+ * @param containerId the Id of the container to be Re-Initialized.
+ * @param t a Throwable.
+ */
+ public void onContainerReInitializeError(ContainerId containerId,
+ Throwable t) {}
+
+ /**
+ * Error Callback for container restart.
+ *
+ * @param containerId the Id of the container to restart.
+ * @param t a Throwable.
+ *
+ */
+ public void onContainerRestartError(ContainerId containerId, Throwable t) {}
+
+ /**
+ * Error Callback for rollback of last re-initialization.
+ *
+ * @param containerId the Id of the container to restart.
+ * @param t a Throwable.
+ */
+ public void onRollbackLastReInitializationError(ContainerId containerId,
+ Throwable t) {}
+
+ /**
+ * Error Callback for commit of last re-initialization.
+ *
+ * @param containerId the Id of the container to commit reInitialize.
+ * @param t a Throwable.
+ */
+ public void onCommitLastReInitializationError(ContainerId containerId,
+ Throwable t) {}
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8236130b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
index 575ce13..515a8e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
@@ -282,6 +282,103 @@ public class NMClientAsyncImpl extends NMClientAsync {
}
}
+ @Override
+ public void reInitializeContainerAsync(ContainerId containerId,
+ ContainerLaunchContext containerLaunchContex, boolean autoCommit){
+ if (!(callbackHandler instanceof AbstractCallbackHandler)) {
+ LOG.error("Callback handler does not implement container re-initialize "
+ + "callback methods");
+ return;
+ }
+ AbstractCallbackHandler handler = (AbstractCallbackHandler) callbackHandler;
+ if (containers.get(containerId) == null) {
+ handler.onContainerReInitializeError(
+ containerId, RPCUtil.getRemoteException(
+ "Container " + containerId + " is not started"));
+ }
+ try {
+ events.put(new ReInitializeContainerEvevnt(containerId,
+ client.getNodeIdOfStartedContainer(containerId),
+ containerLaunchContex, autoCommit));
+ } catch (InterruptedException e) {
+ LOG.warn("Exception when scheduling the event of re-initializing of "
+ + "Container " + containerId);
+ handler.onContainerReInitializeError(containerId, e);
+ }
+ }
+
+ @Override
+ public void restartContainerAsync(ContainerId containerId){
+ if (!(callbackHandler instanceof AbstractCallbackHandler)) {
+ LOG.error("Callback handler does not implement container restart "
+ + "callback methods");
+ return;
+ }
+ AbstractCallbackHandler handler = (AbstractCallbackHandler) callbackHandler;
+ if (containers.get(containerId) == null) {
+ handler.onContainerRestartError(
+ containerId, RPCUtil.getRemoteException(
+ "Container " + containerId + " is not started"));
+ }
+ try {
+ events.put(new ContainerEvent(containerId,
+ client.getNodeIdOfStartedContainer(containerId),
+ null, ContainerEventType.RESTART_CONTAINER));
+ } catch (InterruptedException e) {
+ LOG.warn("Exception when scheduling the event of restart of "
+ + "Container " + containerId);
+ handler.onContainerRestartError(containerId, e);
+ }
+ }
+
+ @Override
+ public void rollbackLastReInitializationAsync(ContainerId containerId){
+ if (!(callbackHandler instanceof AbstractCallbackHandler)) {
+ LOG.error("Callback handler does not implement container rollback "
+ + "callback methods");
+ return;
+ }
+ AbstractCallbackHandler handler = (AbstractCallbackHandler) callbackHandler;
+ if (containers.get(containerId) == null) {
+ handler.onRollbackLastReInitializationError(
+ containerId, RPCUtil.getRemoteException(
+ "Container " + containerId + " is not started"));
+ }
+ try {
+ events.put(new ContainerEvent(containerId,
+ client.getNodeIdOfStartedContainer(containerId),
+ null, ContainerEventType.ROLLBACK_LAST_REINIT));
+ } catch (InterruptedException e) {
+ LOG.warn("Exception when scheduling the event Rollback re-initialization"
+ + " of Container " + containerId);
+ handler.onRollbackLastReInitializationError(containerId, e);
+ }
+ }
+
+ @Override
+ public void commitLastReInitializationAsync(ContainerId containerId){
+ if (!(callbackHandler instanceof AbstractCallbackHandler)) {
+ LOG.error("Callback handler does not implement container commit last " +
+ "re-initialization callback methods");
+ return;
+ }
+ AbstractCallbackHandler handler = (AbstractCallbackHandler) callbackHandler;
+ if (containers.get(containerId) == null) {
+ handler.onCommitLastReInitializationError(
+ containerId, RPCUtil.getRemoteException(
+ "Container " + containerId + " is not started"));
+ }
+ try {
+ events.put(new ContainerEvent(containerId,
+ client.getNodeIdOfStartedContainer(containerId),
+ null, ContainerEventType.COMMIT_LAST_REINT));
+ } catch (InterruptedException e) {
+ LOG.warn("Exception when scheduling the event Commit re-initialization"
+ + " of Container " + containerId);
+ handler.onCommitLastReInitializationError(containerId, e);
+ }
+ }
+
public void stopContainerAsync(ContainerId containerId, NodeId nodeId) {
if (containers.get(containerId) == null) {
callbackHandler.onStopContainerError(containerId,
@@ -330,7 +427,11 @@ public class NMClientAsyncImpl extends NMClientAsync {
START_CONTAINER,
STOP_CONTAINER,
QUERY_CONTAINER,
- INCREASE_CONTAINER_RESOURCE
+ INCREASE_CONTAINER_RESOURCE,
+ REINITIALIZE_CONTAINER,
+ RESTART_CONTAINER,
+ ROLLBACK_LAST_REINIT,
+ COMMIT_LAST_REINT
}
protected static class ContainerEvent
@@ -381,6 +482,27 @@ public class NMClientAsyncImpl extends NMClientAsync {
}
}
+ protected static class ReInitializeContainerEvevnt extends ContainerEvent {
+ private ContainerLaunchContext containerLaunchContext;
+ private boolean autoCommit;
+
+ public ReInitializeContainerEvevnt(ContainerId containerId, NodeId nodeId,
+ ContainerLaunchContext containerLaunchContext, boolean autoCommit) {
+ super(containerId, nodeId, null,
+ ContainerEventType.REINITIALIZE_CONTAINER);
+ this.containerLaunchContext = containerLaunchContext;
+ this.autoCommit = autoCommit;
+ }
+
+ public ContainerLaunchContext getContainerLaunchContext() {
+ return containerLaunchContext;
+ }
+
+ public boolean isAutoCommit() {
+ return autoCommit;
+ }
+ }
+
protected static class IncreaseContainerResourceEvent extends ContainerEvent {
private Container container;
@@ -416,6 +538,25 @@ public class NMClientAsyncImpl extends NMClientAsync {
.addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
ContainerEventType.INCREASE_CONTAINER_RESOURCE,
new IncreaseContainerResourceTransition())
+
+ // Transitions for Container Upgrade
+ .addTransition(ContainerState.RUNNING,
+ EnumSet.of(ContainerState.RUNNING, ContainerState.FAILED),
+ ContainerEventType.REINITIALIZE_CONTAINER,
+ new ReInitializeContainerTransition())
+ .addTransition(ContainerState.RUNNING,
+ EnumSet.of(ContainerState.RUNNING, ContainerState.FAILED),
+ ContainerEventType.RESTART_CONTAINER,
+ new ReInitializeContainerTransition())
+ .addTransition(ContainerState.RUNNING,
+ EnumSet.of(ContainerState.RUNNING, ContainerState.FAILED),
+ ContainerEventType.ROLLBACK_LAST_REINIT,
+ new ReInitializeContainerTransition())
+ .addTransition(ContainerState.RUNNING,
+ EnumSet.of(ContainerState.RUNNING, ContainerState.FAILED),
+ ContainerEventType.COMMIT_LAST_REINT,
+ new ReInitializeContainerTransition())
+
.addTransition(ContainerState.RUNNING,
EnumSet.of(ContainerState.DONE, ContainerState.FAILED),
ContainerEventType.STOP_CONTAINER,
@@ -431,6 +572,10 @@ public class NMClientAsyncImpl extends NMClientAsync {
.addTransition(ContainerState.FAILED, ContainerState.FAILED,
EnumSet.of(ContainerEventType.START_CONTAINER,
ContainerEventType.STOP_CONTAINER,
+ ContainerEventType.REINITIALIZE_CONTAINER,
+ ContainerEventType.RESTART_CONTAINER,
+ ContainerEventType.COMMIT_LAST_REINT,
+ ContainerEventType.ROLLBACK_LAST_REINIT,
ContainerEventType.INCREASE_CONTAINER_RESOURCE));
protected static class StartContainerTransition implements
@@ -529,6 +674,119 @@ public class NMClientAsyncImpl extends NMClientAsync {
}
}
+ protected static class ReInitializeContainerTransition implements
+ MultipleArcTransition<StatefulContainer, ContainerEvent,
+ ContainerState> {
+
+ @Override
+ public ContainerState transition(StatefulContainer container,
+ ContainerEvent containerEvent) {
+ ContainerId containerId = containerEvent.getContainerId();
+ AbstractCallbackHandler handler = (AbstractCallbackHandler) container
+ .nmClientAsync.getCallbackHandler();
+ Throwable handlerError = null;
+ try {
+ switch(containerEvent.getType()) {
+ case REINITIALIZE_CONTAINER:
+ if (!(containerEvent instanceof ReInitializeContainerEvevnt)) {
+ LOG.error("Unexpected Event.. [" +containerEvent.getType() + "]");
+ return ContainerState.FAILED;
+ }
+ ReInitializeContainerEvevnt rEvent =
+ (ReInitializeContainerEvevnt)containerEvent;
+ container.nmClientAsync.getClient().reInitializeContainer(
+ containerId, rEvent.getContainerLaunchContext(),
+ rEvent.isAutoCommit());
+ try {
+ handler.onContainerReInitialize(containerId);
+ } catch (Throwable tr) {
+ handlerError = tr;
+ }
+ break;
+ case RESTART_CONTAINER:
+ container.nmClientAsync.getClient().restartContainer(containerId);
+ try {
+ handler.onContainerRestart(containerId);
+ } catch (Throwable tr) {
+ handlerError = tr;
+ }
+ break;
+ case ROLLBACK_LAST_REINIT:
+ container.nmClientAsync.getClient()
+ .rollbackLastReInitialization(containerId);
+ try {
+ handler.onRollbackLastReInitialization(containerId);
+ } catch (Throwable tr) {
+ handlerError = tr;
+ }
+ break;
+ case COMMIT_LAST_REINT:
+ container.nmClientAsync.getClient()
+ .commitLastReInitialization(containerId);
+ try {
+ handler.onCommitLastReInitialization(containerId);
+ } catch (Throwable tr) {
+ handlerError = tr;
+ }
+ break;
+ default:
+ LOG.warn("Event of type [" + containerEvent.getType() + "] not" +
+ " expected here..");
+ break;
+ }
+ if (handlerError != null) {
+ LOG.info("Unchecked exception is thrown in handler for event ["
+ + containerEvent.getType() + "] for Container "
+ + containerId, handlerError);
+ }
+
+ return ContainerState.RUNNING;
+ } catch (Throwable t) {
+ switch(containerEvent.getType()) {
+ case REINITIALIZE_CONTAINER:
+ try {
+ handler.onContainerReInitializeError(containerId, t);
+ } catch (Throwable tr) {
+ handlerError = tr;
+ }
+ break;
+ case RESTART_CONTAINER:
+ try {
+ handler.onContainerRestartError(containerId, t);
+ } catch (Throwable tr) {
+ handlerError = tr;
+ }
+ break;
+ case ROLLBACK_LAST_REINIT:
+ try {
+ handler.onRollbackLastReInitializationError(containerId, t);
+ } catch (Throwable tr) {
+ handlerError = tr;
+ }
+ break;
+ case COMMIT_LAST_REINT:
+ try {
+ handler.onCommitLastReInitializationError(containerId, t);
+ } catch (Throwable tr) {
+ handlerError = tr;
+ }
+ break;
+ default:
+ LOG.warn("Event of type [" + containerEvent.getType() + "] not" +
+ " expected here..");
+ break;
+ }
+ if (handlerError != null) {
+ LOG.info("Unchecked exception is thrown in handler for event ["
+ + containerEvent.getType() + "] for Container "
+ + containerId, handlerError);
+ }
+ }
+
+ return ContainerState.FAILED;
+ }
+ }
+
protected static class StopContainerTransition implements
MultipleArcTransition<StatefulContainer, ContainerEvent,
ContainerState> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8236130b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
index dc92cda..c81d448 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
@@ -33,10 +33,13 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
@@ -306,6 +309,84 @@ public class NMClientImpl extends NMClient {
}
}
+ @Override
+ public void reInitializeContainer(ContainerId containerId,
+ ContainerLaunchContext containerLaunchContex, boolean autoCommit)
+ throws YarnException, IOException {
+ ContainerManagementProtocolProxyData proxy = null;
+ StartedContainer container = startedContainers.get(containerId);
+ if (container != null) {
+ synchronized (container) {
+ proxy = cmProxy.getProxy(container.getNodeId().toString(), containerId);
+ try {
+ proxy.getContainerManagementProtocol().reInitializeContainer(
+ ReInitializeContainerRequest.newInstance(
+ containerId, containerLaunchContex, autoCommit));
+ } finally {
+ if (proxy != null) {
+ cmProxy.mayBeCloseProxy(proxy);
+ }
+ }
+ }
+ } else {
+ throw new YarnException("Unknown container [" + containerId + "]");
+ }
+ }
+
+ @Override
+ public void restartContainer(ContainerId containerId)
+ throws YarnException, IOException {
+ restartCommitOrRollbackContainer(containerId, UpgradeOp.RESTART);
+ }
+
+ @Override
+ public void rollbackLastReInitialization(ContainerId containerId)
+ throws YarnException, IOException {
+ restartCommitOrRollbackContainer(containerId, UpgradeOp.ROLLBACK);
+ }
+
+ @Override
+ public void commitLastReInitialization(ContainerId containerId)
+ throws YarnException, IOException {
+ restartCommitOrRollbackContainer(containerId, UpgradeOp.COMMIT);
+ }
+
+
+ private void restartCommitOrRollbackContainer(ContainerId containerId,
+ UpgradeOp upgradeOp) throws YarnException, IOException {
+ ContainerManagementProtocolProxyData proxy = null;
+ StartedContainer container = startedContainers.get(containerId);
+ if (container != null) {
+ synchronized (container) {
+ proxy = cmProxy.getProxy(container.getNodeId().toString(), containerId);
+ ContainerManagementProtocol cmp =
+ proxy.getContainerManagementProtocol();
+ try {
+ switch (upgradeOp) {
+ case RESTART:
+ cmp.restartContainer(containerId);
+ break;
+ case COMMIT:
+ cmp.commitLastReInitialization(containerId);
+ break;
+ case ROLLBACK:
+ cmp.rollbackLastReInitialization(containerId);
+ break;
+ default:
+ // Should not happen..
+ break;
+ }
+ } finally {
+ if (proxy != null) {
+ cmProxy.mayBeCloseProxy(proxy);
+ }
+ }
+ }
+ } else {
+ throw new YarnException("Unknown container [" + containerId + "]");
+ }
+ }
+
private void stopContainerInternal(ContainerId containerId, NodeId nodeId)
throws IOException, YarnException {
ContainerManagementProtocolProxyData proxy = null;
@@ -343,4 +424,14 @@ public class NMClientImpl extends NMClient {
throw (IOException) t;
}
}
+
+ @Override
+ public NodeId getNodeIdOfStartedContainer(ContainerId containerId) {
+ StartedContainer container = startedContainers.get(containerId);
+ if (container != null) {
+ return container.getNodeId();
+ }
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8236130b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
index 48f3431..dda3eec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.client.api.async.impl;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -27,6 +28,8 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -37,6 +40,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerArray;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
@@ -69,6 +73,22 @@ public class TestNMClientAsync {
private NodeId nodeId;
private Token containerToken;
+ enum OpsToTest {
+ START, QUERY, STOP, INCR, REINIT, RESTART, ROLLBACK, COMMIT
+ }
+
+ final static class TestData {
+ AtomicInteger success = new AtomicInteger(0);
+ AtomicInteger failure = new AtomicInteger(0);
+ final AtomicIntegerArray successArray;
+ final AtomicIntegerArray failureArray;
+
+ private TestData(int expectedSuccess, int expectedFailure) {
+ this.successArray = new AtomicIntegerArray(expectedSuccess);
+ this.failureArray = new AtomicIntegerArray(expectedFailure);
+ }
+ }
+
@After
public void teardown() {
ServiceOperations.stop(asyncClient);
@@ -194,25 +214,7 @@ public class TestNMClientAsync {
private int expectedSuccess;
private int expectedFailure;
- private AtomicInteger actualStartSuccess = new AtomicInteger(0);
- private AtomicInteger actualStartFailure = new AtomicInteger(0);
- private AtomicInteger actualQuerySuccess = new AtomicInteger(0);
- private AtomicInteger actualQueryFailure = new AtomicInteger(0);
- private AtomicInteger actualStopSuccess = new AtomicInteger(0);
- private AtomicInteger actualStopFailure = new AtomicInteger(0);
- private AtomicInteger actualIncreaseResourceSuccess =
- new AtomicInteger(0);
- private AtomicInteger actualIncreaseResourceFailure =
- new AtomicInteger(0);
-
- private AtomicIntegerArray actualStartSuccessArray;
- private AtomicIntegerArray actualStartFailureArray;
- private AtomicIntegerArray actualQuerySuccessArray;
- private AtomicIntegerArray actualQueryFailureArray;
- private AtomicIntegerArray actualStopSuccessArray;
- private AtomicIntegerArray actualStopFailureArray;
- private AtomicIntegerArray actualIncreaseResourceSuccessArray;
- private AtomicIntegerArray actualIncreaseResourceFailureArray;
+ private final Map<OpsToTest, TestData> testMap = new HashMap<>();
private Set<String> errorMsgs =
Collections.synchronizedSet(new HashSet<String>());
@@ -221,16 +223,9 @@ public class TestNMClientAsync {
this.expectedSuccess = expectedSuccess;
this.expectedFailure = expectedFailure;
- actualStartSuccessArray = new AtomicIntegerArray(expectedSuccess);
- actualStartFailureArray = new AtomicIntegerArray(expectedFailure);
- actualQuerySuccessArray = new AtomicIntegerArray(expectedSuccess);
- actualQueryFailureArray = new AtomicIntegerArray(expectedFailure);
- actualStopSuccessArray = new AtomicIntegerArray(expectedSuccess);
- actualStopFailureArray = new AtomicIntegerArray(expectedFailure);
- actualIncreaseResourceSuccessArray =
- new AtomicIntegerArray(expectedSuccess);
- actualIncreaseResourceFailureArray =
- new AtomicIntegerArray(expectedFailure);
+ for (OpsToTest op : OpsToTest.values()) {
+ testMap.put(op, new TestData(expectedSuccess, expectedFailure));
+ }
}
@SuppressWarnings("deprecation")
@@ -243,8 +238,9 @@ public class TestNMClientAsync {
" should throw the exception onContainerStarted");
return;
}
- actualStartSuccess.addAndGet(1);
- actualStartSuccessArray.set(containerId.getId(), 1);
+ TestData td = testMap.get(OpsToTest.START);
+ td.success.addAndGet(1);
+ td.successArray.set(containerId.getId(), 1);
// move on to the following success tests
asyncClient.getContainerStatusAsync(containerId, nodeId);
@@ -254,7 +250,28 @@ public class TestNMClientAsync {
// containerId
Container container = Container.newInstance(
containerId, nodeId, null, null, null, containerToken);
- asyncClient.increaseContainerResourceAsync(container);
+ int t = containerId.getId() % 5;
+ switch (t) {
+ case 0:
+ asyncClient.increaseContainerResourceAsync(container);
+ break;
+ case 1:
+ asyncClient.reInitializeContainerAsync(containerId,
+ recordFactory.newRecordInstance(ContainerLaunchContext.class),
+ true);
+ break;
+ case 2:
+ asyncClient.restartContainerAsync(containerId);
+ break;
+ case 3:
+ asyncClient.rollbackLastReInitializationAsync(containerId);
+ break;
+ case 4:
+ asyncClient.commitLastReInitializationAsync(containerId);
+ break;
+ default:
+ break;
+ }
}
// Shouldn't crash the test thread
@@ -270,8 +287,9 @@ public class TestNMClientAsync {
" should throw the exception onContainerStatusReceived");
return;
}
- actualQuerySuccess.addAndGet(1);
- actualQuerySuccessArray.set(containerId.getId(), 1);
+ TestData td = testMap.get(OpsToTest.QUERY);
+ td.success.addAndGet(1);
+ td.successArray.set(containerId.getId(), 1);
// move on to the following success tests
// make sure we pass in the container with the same
// containerId
@@ -292,8 +310,78 @@ public class TestNMClientAsync {
" should throw the exception onContainerResourceIncreased");
return;
}
- actualIncreaseResourceSuccess.addAndGet(1);
- actualIncreaseResourceSuccessArray.set(containerId.getId(), 1);
+ TestData td = testMap.get(OpsToTest.INCR);
+ td.success.addAndGet(1);
+ td.successArray.set(containerId.getId(), 1);
+ // move on to the following success tests
+ asyncClient.reInitializeContainerAsync(containerId,
+ Records.newRecord(ContainerLaunchContext.class), true);
+ // throw a fake user exception, and shouldn't crash the test
+ throw new RuntimeException("Ignorable Exception");
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void onContainerReInitialize(ContainerId containerId) {
+ if (containerId.getId() >= expectedSuccess) {
+ errorMsgs.add("Container " + containerId +
+ " should throw the exception onContainerReInitialize");
+ return;
+ }
+ TestData td = testMap.get(OpsToTest.REINIT);
+ td.success.addAndGet(1);
+ td.successArray.set(containerId.getId(), 1);
+ // move on to the following success tests
+ asyncClient.restartContainerAsync(containerId);
+ // throw a fake user exception, and shouldn't crash the test
+ throw new RuntimeException("Ignorable Exception");
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void onContainerRestart(ContainerId containerId) {
+ if (containerId.getId() >= expectedSuccess) {
+ errorMsgs.add("Container " + containerId +
+ " should throw the exception onContainerReInitialize");
+ return;
+ }
+ TestData td = testMap.get(OpsToTest.RESTART);
+ td.success.addAndGet(1);
+ td.successArray.set(containerId.getId(), 1);
+ // move on to the following success tests
+ asyncClient.rollbackLastReInitializationAsync(containerId);
+ // throw a fake user exception, and shouldn't crash the test
+ throw new RuntimeException("Ignorable Exception");
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void onRollbackLastReInitialization(ContainerId containerId) {
+ if (containerId.getId() >= expectedSuccess) {
+ errorMsgs.add("Container " + containerId +
+ " should throw the exception onContainerReInitialize");
+ return;
+ }
+ TestData td = testMap.get(OpsToTest.ROLLBACK);
+ td.success.addAndGet(1);
+ td.successArray.set(containerId.getId(), 1);
+ // move on to the following success tests
+ asyncClient.commitLastReInitializationAsync(containerId);
+ // throw a fake user exception, and shouldn't crash the test
+ throw new RuntimeException("Ignorable Exception");
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void onCommitLastReInitialization(ContainerId containerId) {
+ if (containerId.getId() >= expectedSuccess) {
+ errorMsgs.add("Container " + containerId +
+ " should throw the exception onContainerReInitialize");
+ return;
+ }
+ TestData td = testMap.get(OpsToTest.COMMIT);
+ td.success.addAndGet(1);
+ td.successArray.set(containerId.getId(), 1);
// move on to the following success tests
asyncClient.stopContainerAsync(containerId, nodeId);
// throw a fake user exception, and shouldn't crash the test
@@ -308,8 +396,9 @@ public class TestNMClientAsync {
" should throw the exception onContainerStopped");
return;
}
- actualStopSuccess.addAndGet(1);
- actualStopSuccessArray.set(containerId.getId(), 1);
+ TestData td = testMap.get(OpsToTest.STOP);
+ td.success.addAndGet(1);
+ td.successArray.set(containerId.getId(), 1);
// Shouldn't crash the test thread
throw new RuntimeException("Ignorable Exception");
@@ -330,8 +419,9 @@ public class TestNMClientAsync {
" shouldn't throw the exception onStartContainerError");
return;
}
- actualStartFailure.addAndGet(1);
- actualStartFailureArray.set(containerId.getId() - expectedSuccess, 1);
+ TestData td = testMap.get(OpsToTest.START);
+ td.failure.addAndGet(1);
+ td.failureArray.set(containerId.getId() - expectedSuccess, 1);
// move on to the following failure tests
asyncClient.getContainerStatusAsync(containerId, nodeId);
@@ -348,8 +438,9 @@ public class TestNMClientAsync {
" shouldn't throw the exception onIncreaseContainerResourceError");
return;
}
- actualIncreaseResourceFailure.addAndGet(1);
- actualIncreaseResourceFailureArray.set(
+ TestData td = testMap.get(OpsToTest.INCR);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
containerId.getId() - expectedSuccess - expectedFailure, 1);
// increase container resource error should NOT change the
// the container status to FAILED
@@ -361,6 +452,102 @@ public class TestNMClientAsync {
@SuppressWarnings("deprecation")
@Override
+ public void onContainerReInitializeError(ContainerId containerId,
+ Throwable t) {
+ if (containerId.getId() < expectedSuccess + expectedFailure) {
+ errorMsgs.add("Container " + containerId +
+ " shouldn't throw the exception onContainerReInitializeError");
+ return;
+ }
+ TestData td = testMap.get(OpsToTest.REINIT);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
+ containerId.getId() - expectedSuccess - expectedFailure, 1);
+
+ // increment the stop counters here.. since the container will fail
+ td = testMap.get(OpsToTest.STOP);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
+ containerId.getId() - expectedSuccess - expectedFailure, 1);
+ // reInit container changes the container status to FAILED
+ // Shouldn't crash the test thread
+ throw new RuntimeException("Ignorable Exception");
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void onContainerRestartError(ContainerId containerId, Throwable t) {
+ if (containerId.getId() < expectedSuccess + expectedFailure) {
+ errorMsgs.add("Container " + containerId +
+ " shouldn't throw the exception onContainerRestartError");
+ return;
+ }
+ TestData td = testMap.get(OpsToTest.RESTART);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
+ containerId.getId() - expectedSuccess - expectedFailure, 1);
+
+ // increment the stop counters here.. since the container will fail
+ td = testMap.get(OpsToTest.STOP);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
+ containerId.getId() - expectedSuccess - expectedFailure, 1);
+ // restart container changes the container status to FAILED
+ // Shouldn't crash the test thread
+ throw new RuntimeException("Ignorable Exception");
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void onRollbackLastReInitializationError(ContainerId containerId,
+ Throwable t) {
+ if (containerId.getId() < expectedSuccess + expectedFailure) {
+ errorMsgs.add("Container " + containerId +
+ " shouldn't throw the exception" +
+ " onRollbackLastReInitializationError");
+ return;
+ }
+ TestData td = testMap.get(OpsToTest.ROLLBACK);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
+ containerId.getId() - expectedSuccess - expectedFailure, 1);
+
+ // increment the stop counters here.. since the container will fail
+ td = testMap.get(OpsToTest.STOP);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
+ containerId.getId() - expectedSuccess - expectedFailure, 1);
+ // rollback container changes the container status to FAILED
+ // Shouldn't crash the test thread
+ throw new RuntimeException("Ignorable Exception");
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void onCommitLastReInitializationError(ContainerId containerId,
+ Throwable t) {
+ if (containerId.getId() < expectedSuccess + expectedFailure) {
+ errorMsgs.add("Container " + containerId +
+ " shouldn't throw the exception onCommitLastReInitializationError");
+ return;
+ }
+ TestData td = testMap.get(OpsToTest.COMMIT);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
+ containerId.getId() - expectedSuccess - expectedFailure, 1);
+
+ // increment the stop counters here.. since the container will fail
+ td = testMap.get(OpsToTest.STOP);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
+ containerId.getId() - expectedSuccess - expectedFailure, 1);
+ // commit container changes the container status to FAILED
+ // Shouldn't crash the test thread
+ throw new RuntimeException("Ignorable Exception");
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
public void onStopContainerError(ContainerId containerId, Throwable t) {
if (t instanceof RuntimeException) {
errorMsgs.add("Unexpected throwable from callback functions should be" +
@@ -371,9 +558,9 @@ public class TestNMClientAsync {
" shouldn't throw the exception onStopContainerError");
return;
}
-
- actualStopFailure.addAndGet(1);
- actualStopFailureArray.set(
+ TestData td = testMap.get(OpsToTest.STOP);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
containerId.getId() - expectedSuccess - expectedFailure, 1);
// Shouldn't crash the test thread
@@ -393,8 +580,9 @@ public class TestNMClientAsync {
" shouldn't throw the exception onGetContainerStatusError");
return;
}
- actualQueryFailure.addAndGet(1);
- actualQueryFailureArray.set(containerId.getId() - expectedSuccess, 1);
+ TestData td = testMap.get(OpsToTest.QUERY);
+ td.failure.addAndGet(1);
+ td.failureArray.set(containerId.getId() - expectedSuccess, 1);
// Shouldn't crash the test thread
throw new RuntimeException("Ignorable Exception");
@@ -402,44 +590,67 @@ public class TestNMClientAsync {
public boolean isAllSuccessCallsExecuted() {
boolean isAllSuccessCallsExecuted =
- actualStartSuccess.get() == expectedSuccess &&
- actualQuerySuccess.get() == expectedSuccess &&
- actualIncreaseResourceSuccess.get() == expectedSuccess &&
- actualStopSuccess.get() == expectedSuccess;
+ testMap.get(OpsToTest.START).success.get() == expectedSuccess &&
+ testMap.get(OpsToTest.QUERY).success.get() == expectedSuccess &&
+ testMap.get(OpsToTest.INCR).success.get() == expectedSuccess &&
+ testMap.get(OpsToTest.REINIT).success.get() == expectedSuccess &&
+ testMap.get(OpsToTest.RESTART).success.get() == expectedSuccess &&
+ testMap.get(OpsToTest.ROLLBACK).success.get() ==
+ expectedSuccess &&
+ testMap.get(OpsToTest.COMMIT).success.get() == expectedSuccess &&
+ testMap.get(OpsToTest.STOP).success.get() == expectedSuccess;
if (isAllSuccessCallsExecuted) {
- assertAtomicIntegerArray(actualStartSuccessArray);
- assertAtomicIntegerArray(actualQuerySuccessArray);
- assertAtomicIntegerArray(actualIncreaseResourceSuccessArray);
- assertAtomicIntegerArray(actualStopSuccessArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.START).successArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.QUERY).successArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.INCR).successArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.REINIT).successArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.RESTART).successArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.ROLLBACK).successArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.COMMIT).successArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.STOP).successArray);
}
return isAllSuccessCallsExecuted;
}
public boolean isStartAndQueryFailureCallsExecuted() {
boolean isStartAndQueryFailureCallsExecuted =
- actualStartFailure.get() == expectedFailure &&
- actualQueryFailure.get() == expectedFailure;
+ testMap.get(OpsToTest.START).failure.get() == expectedFailure &&
+ testMap.get(OpsToTest.QUERY).failure.get() == expectedFailure;
if (isStartAndQueryFailureCallsExecuted) {
- assertAtomicIntegerArray(actualStartFailureArray);
- assertAtomicIntegerArray(actualQueryFailureArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.START).failureArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.QUERY).failureArray);
}
return isStartAndQueryFailureCallsExecuted;
}
public boolean isIncreaseResourceFailureCallsExecuted() {
boolean isIncreaseResourceFailureCallsExecuted =
- actualIncreaseResourceFailure.get() == expectedFailure;
+ testMap.get(OpsToTest.INCR).failure.get()
+ + testMap.get(OpsToTest.REINIT).failure.get()
+ + testMap.get(OpsToTest.RESTART).failure.get()
+ + testMap.get(OpsToTest.ROLLBACK).failure.get()
+ + testMap.get(OpsToTest.COMMIT).failure.get()
+ == expectedFailure;
if (isIncreaseResourceFailureCallsExecuted) {
- assertAtomicIntegerArray(actualIncreaseResourceFailureArray);
+ AtomicIntegerArray testArray =
+ new AtomicIntegerArray(
+ testMap.get(OpsToTest.INCR).failureArray.length());
+ for (int i = 0; i < testArray.length(); i++) {
+ for (OpsToTest op : EnumSet.of(OpsToTest.REINIT, OpsToTest.RESTART,
+ OpsToTest.ROLLBACK, OpsToTest.COMMIT, OpsToTest.INCR)) {
+ testArray.addAndGet(i, testMap.get(op).failureArray.get(i));
+ }
+ }
+ assertAtomicIntegerArray(testArray);
}
return isIncreaseResourceFailureCallsExecuted;
}
public boolean isStopFailureCallsExecuted() {
boolean isStopFailureCallsExecuted =
- actualStopFailure.get() == expectedFailure;
+ testMap.get(OpsToTest.STOP).failure.get() == expectedFailure;
if (isStopFailureCallsExecuted) {
- assertAtomicIntegerArray(actualStopFailureArray);
+ assertAtomicIntegerArray(testMap.get(OpsToTest.STOP).failureArray);
}
return isStopFailureCallsExecuted;
}
@@ -464,6 +675,14 @@ public class TestNMClientAsync {
recordFactory.newRecordInstance(ContainerStatus.class));
doNothing().when(client).increaseContainerResource(
any(Container.class));
+ doNothing().when(client).reInitializeContainer(
+ any(ContainerId.class), any(ContainerLaunchContext.class),
+ anyBoolean());
+ doNothing().when(client).restartContainer(any(ContainerId.class));
+ doNothing().when(client).rollbackLastReInitialization(
+ any(ContainerId.class));
+ doNothing().when(client).commitLastReInitialization(
+ any(ContainerId.class));
doNothing().when(client).stopContainer(any(ContainerId.class),
any(NodeId.class));
break;
@@ -485,9 +704,23 @@ public class TestNMClientAsync {
recordFactory.newRecordInstance(ContainerStatus.class));
doThrow(RPCUtil.getRemoteException("Increase Resource Exception"))
.when(client).increaseContainerResource(any(Container.class));
+ doThrow(RPCUtil.getRemoteException("ReInitialize Exception"))
+ .when(client).reInitializeContainer(
+ any(ContainerId.class), any(ContainerLaunchContext.class),
+ anyBoolean());
+ doThrow(RPCUtil.getRemoteException("Restart Exception"))
+ .when(client).restartContainer(any(ContainerId.class));
+ doThrow(RPCUtil.getRemoteException("Rollback upgrade Exception"))
+ .when(client).rollbackLastReInitialization(
+ any(ContainerId.class));
+ doThrow(RPCUtil.getRemoteException("Commit upgrade Exception"))
+ .when(client).commitLastReInitialization(
+ any(ContainerId.class));
doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
.stopContainer(any(ContainerId.class), any(NodeId.class));
}
+ when(client.getNodeIdOfStartedContainer(any(ContainerId.class)))
+ .thenReturn(NodeId.newInstance("localhost", 0));
return client;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8236130b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
index d211d6d..1034f7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -69,6 +70,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -310,6 +312,36 @@ public class TestNMClient {
e.getMessage().contains("is not handled by this NodeManager"));
}
+ // restart shouldn't be called before startContainer,
+ // otherwise, NodeManager cannot find the container
+ try {
+ nmClient.restartContainer(container.getId());
+ fail("Exception is expected");
+ } catch (YarnException e) {
+ assertTrue("The thrown exception is not expected",
+ e.getMessage().contains("Unknown container"));
+ }
+
+ // rollback shouldn't be called before startContainer,
+ // otherwise, NodeManager cannot find the container
+ try {
+ nmClient.rollbackLastReInitialization(container.getId());
+ fail("Exception is expected");
+ } catch (YarnException e) {
+ assertTrue("The thrown exception is not expected",
+ e.getMessage().contains("Unknown container"));
+ }
+
+ // commit shouldn't be called before startContainer,
+ // otherwise, NodeManager cannot find the container
+ try {
+ nmClient.commitLastReInitialization(container.getId());
+ fail("Exception is expected");
+ } catch (YarnException e) {
+ assertTrue("The thrown exception is not expected",
+ e.getMessage().contains("Unknown container"));
+ }
+
// stopContainer shouldn't be called before startContainer,
// otherwise, an exception will be thrown
try {
@@ -353,6 +385,28 @@ public class TestNMClient {
// Test increase container API and make sure requests can reach NM
testIncreaseContainerResource(container);
+ testRestartContainer(container.getId());
+ testGetContainerStatus(container, i, ContainerState.RUNNING,
+ "will be Restarted", Arrays.asList(new Integer[] {-1000}));
+
+ if (i % 2 == 0) {
+ testReInitializeContainer(container.getId(), clc, false);
+ testGetContainerStatus(container, i, ContainerState.RUNNING,
+ "will be Re-initialized", Arrays.asList(new Integer[] {-1000}));
+ testRollbackContainer(container.getId(), false);
+ testGetContainerStatus(container, i, ContainerState.RUNNING,
+ "will be Rolled-back", Arrays.asList(new Integer[] {-1000}));
+ testCommitContainer(container.getId(), true);
+ testReInitializeContainer(container.getId(), clc, false);
+ testCommitContainer(container.getId(), false);
+ } else {
+ testReInitializeContainer(container.getId(), clc, true);
+ testGetContainerStatus(container, i, ContainerState.RUNNING,
+ "will be Re-initialized", Arrays.asList(new Integer[] {-1000}));
+ testRollbackContainer(container.getId(), true);
+ testCommitContainer(container.getId(), true);
+ }
+
try {
nmClient.stopContainer(container.getId(), container.getNodeId());
} catch (YarnException e) {
@@ -432,4 +486,91 @@ public class TestNMClient {
}
}
}
+
+ private void testRestartContainer(ContainerId containerId)
+ throws YarnException, IOException {
+ try {
+ sleep(250);
+ nmClient.restartContainer(containerId);
+ sleep(250);
+ } catch (YarnException e) {
+ // NM container will only be in SCHEDULED state, so expect the increase
+ // action to fail.
+ if (!e.getMessage().contains(
+ "can only be changed when a container is in RUNNING state")) {
+ throw (AssertionError)
+ (new AssertionError("Exception is not expected: " + e)
+ .initCause(e));
+ }
+ }
+ }
+
+ private void testRollbackContainer(ContainerId containerId,
+ boolean notRollbackable) throws YarnException, IOException {
+ try {
+ sleep(250);
+ nmClient.rollbackLastReInitialization(containerId);
+ if (notRollbackable) {
+ fail("Should not be able to rollback..");
+ }
+ sleep(250);
+ } catch (YarnException e) {
+ // NM container will only be in SCHEDULED state, so expect the increase
+ // action to fail.
+ if (notRollbackable) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Nothing to rollback to"));
+ } else {
+ if (!e.getMessage().contains(
+ "can only be changed when a container is in RUNNING state")) {
+ throw (AssertionError)
+ (new AssertionError("Exception is not expected: " + e)
+ .initCause(e));
+ }
+ }
+ }
+ }
+
+ private void testCommitContainer(ContainerId containerId,
+ boolean notCommittable) throws YarnException, IOException {
+ try {
+ nmClient.commitLastReInitialization(containerId);
+ if (notCommittable) {
+ fail("Should not be able to commit..");
+ }
+ } catch (YarnException e) {
+ // NM container will only be in SCHEDULED state, so expect the increase
+ // action to fail.
+ if (notCommittable) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Nothing to Commit"));
+ } else {
+ if (!e.getMessage().contains(
+ "can only be changed when a container is in RUNNING state")) {
+ throw (AssertionError)
+ (new AssertionError("Exception is not expected: " + e)
+ .initCause(e));
+ }
+ }
+ }
+ }
+
+ private void testReInitializeContainer(ContainerId containerId,
+ ContainerLaunchContext clc, boolean autoCommit)
+ throws YarnException, IOException {
+ try {
+ sleep(250);
+ nmClient.reInitializeContainer(containerId, clc, autoCommit);
+ sleep(250);
+ } catch (YarnException e) {
+ // NM container will only be in SCHEDULED state, so expect the increase
+ // action to fail.
+ if (!e.getMessage().contains(
+ "can only be changed when a container is in RUNNING state")) {
+ throw (AssertionError)
+ (new AssertionError("Exception is not expected: " + e)
+ .initCause(e));
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8236130b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 055e12c..46f8fa0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -22,8 +22,10 @@ import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
+import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Date;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -137,6 +139,8 @@ public class ContainerImpl implements Container {
}
}
+ private final SimpleDateFormat dateFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
private final Lock readLock;
private final Lock writeLock;
private final Dispatcher dispatcher;
@@ -767,7 +771,7 @@ public class ContainerImpl implements Container {
private void addDiagnostics(String... diags) {
for (String s : diags) {
- this.diagnostics.append(s);
+ this.diagnostics.append("[" + dateFormat.format(new Date()) + "]" + s);
}
if (diagnostics.length() > diagnosticsMaxSize) {
diagnostics.delete(0, diagnostics.length() - diagnosticsMaxSize);
@@ -991,6 +995,7 @@ public class ContainerImpl implements Container {
// We also need to make sure that if Rollback is possible, the
// rollback state should be retained in the
// oldLaunchContext and oldResourceSet
+ container.addDiagnostics("Container will be Restarted.\n");
return new ReInitializationContext(
container.launchContext, container.resourceSet,
container.canRollback() ?
@@ -998,6 +1003,7 @@ public class ContainerImpl implements Container {
container.canRollback() ?
container.reInitContext.oldResourceSet : null);
} else {
+ container.addDiagnostics("Container will be Re-initialized.\n");
return new ReInitializationContext(
reInitEvent.getReInitLaunchContext(),
reInitEvent.getResourceSet(),
@@ -1018,6 +1024,7 @@ public class ContainerImpl implements Container {
@Override
protected ReInitializationContext createReInitContext(ContainerImpl
container, ContainerEvent event) {
+ container.addDiagnostics("Container upgrade will be Rolled-back.\n");
LOG.warn("Container [" + container.getContainerId() + "]" +
" about to be explicitly Rolledback !!");
return container.reInitContext.createContextForRollback();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] hadoop git commit: HDFS-11681.
DatanodeStorageInfo#getBlockIterator() should return an iterator to an
unmodifiable set Contributed by Virajith Jalaparti
Posted by wa...@apache.org.
HDFS-11681. DatanodeStorageInfo#getBlockIterator() should return an iterator to an unmodifiable set Contributed by Virajith Jalaparti
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51b671ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51b671ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51b671ef
Branch: refs/heads/YARN-5734
Commit: 51b671ef1844069888f976cd16f66c88f9bbc7de
Parents: eed7314
Author: Chris Douglas <cd...@apache.org>
Authored: Wed May 10 22:20:27 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Wed May 10 22:25:28 2017 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 23 +++++++++++---------
.../blockmanagement/DatanodeStorageInfo.java | 7 +++++-
.../org/apache/hadoop/hdfs/TestGetBlocks.java | 7 ++++++
3 files changed, 26 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51b671ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 41662a4..a9592bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1409,12 +1409,15 @@ public class BlockManager implements BlockStatsMXBean {
void removeBlocksAssociatedTo(final DatanodeDescriptor node) {
for (DatanodeStorageInfo storage : node.getStorageInfos()) {
final Iterator<BlockInfo> it = storage.getBlockIterator();
+ //add the BlockInfos to a new collection as the
+ //returned iterator is not modifiable.
+ Collection<BlockInfo> toRemove = new ArrayList<>();
while (it.hasNext()) {
- BlockInfo block = it.next();
- // DatanodeStorageInfo must be removed using the iterator to avoid
- // ConcurrentModificationException in the underlying storage
- it.remove();
- removeStoredBlock(block, node);
+ toRemove.add(it.next());
+ }
+
+ for (BlockInfo b : toRemove) {
+ removeStoredBlock(b, node);
}
}
// Remove all pending DN messages referencing this DN.
@@ -1429,11 +1432,11 @@ public class BlockManager implements BlockStatsMXBean {
assert namesystem.hasWriteLock();
final Iterator<BlockInfo> it = storageInfo.getBlockIterator();
DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
- while(it.hasNext()) {
- BlockInfo block = it.next();
- // DatanodeStorageInfo must be removed using the iterator to avoid
- // ConcurrentModificationException in the underlying storage
- it.remove();
+ Collection<BlockInfo> toRemove = new ArrayList<>();
+ while (it.hasNext()) {
+ toRemove.add(it.next());
+ }
+ for (BlockInfo block : toRemove) {
removeStoredBlock(block, node);
final Block b = getBlockOnStorage(block, storageInfo);
if (b != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51b671ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index ab666b7..8af86d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -270,8 +271,12 @@ public class DatanodeStorageInfo {
return blocks.size();
}
+ /**
+ * @return iterator to an unmodifiable set of blocks
+ * related to this {@link DatanodeStorageInfo}
+ */
Iterator<BlockInfo> getBlockIterator() {
- return blocks.iterator();
+ return Collections.unmodifiableSet(blocks).iterator();
}
void updateState(StorageReport r) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51b671ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
index 32bdd78..7397507 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
@@ -307,6 +307,13 @@ public class TestGetBlocks {
BlockManagerTestUtil.getBlockIterator(s);
while(storageBlockIt.hasNext()) {
allBlocks[idx++] = storageBlockIt.next();
+ try {
+ storageBlockIt.remove();
+ assertTrue(
+ "BlockInfo iterator should have been unmodifiable", false);
+ } catch (UnsupportedOperationException e) {
+ //expected exception
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] hadoop git commit: YARN-5952. Create REST API for changing
YARN scheduler configurations. (Jonathan Hung via wangda)
Posted by wa...@apache.org.
YARN-5952. Create REST API for changing YARN scheduler configurations. (Jonathan Hung via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb7a0e8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb7a0e8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb7a0e8b
Branch: refs/heads/YARN-5734
Commit: bb7a0e8b54a322728017bff11c9c8717f5135d16
Parents: 6139e47
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Apr 3 10:12:01 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Sat May 20 08:22:44 2017 -0700
----------------------------------------------------------------------
.../scheduler/MutableConfScheduler.java | 40 ++
.../scheduler/MutableConfigurationProvider.java | 5 +-
.../scheduler/capacity/CapacityScheduler.java | 16 +-
.../conf/InMemoryConfigurationStore.java | 6 +-
.../conf/MutableCSConfigurationProvider.java | 24 +-
.../resourcemanager/webapp/RMWebServices.java | 169 +++++++
.../webapp/dao/QueueConfigInfo.java | 57 +++
.../webapp/dao/QueueConfigsUpdateInfo.java | 60 +++
.../TestMutableCSConfigurationProvider.java | 6 +-
.../TestRMWebServicesConfigurationMutation.java | 477 +++++++++++++++++++
10 files changed, 849 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb7a0e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
new file mode 100644
index 0000000..35e36e1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.security.UserGroupInformation;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Interface for a scheduler that supports changing configuration at runtime.
+ *
+ */
+public interface MutableConfScheduler extends ResourceScheduler {
+
+ /**
+ * Update the scheduler's configuration.
+ * @param user Caller of this update
+ * @param confUpdate key-value map of the configuration update
+ * @throws IOException if update is invalid
+ */
+ void updateConfiguration(UserGroupInformation user,
+ Map<String, String> confUpdate) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb7a0e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index da30a2b..889c3bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import java.io.IOException;
import java.util.Map;
/**
@@ -29,7 +30,9 @@ public interface MutableConfigurationProvider {
* Update the scheduler configuration with the provided key value pairs.
* @param user User issuing the request
* @param confUpdate Key-value pairs for configurations to be updated.
+ * @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(String user, Map<String, String> confUpdate);
+ void mutateConfiguration(String user, Map<String, String> confUpdate)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb7a0e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 9353df2..85138d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -85,6 +85,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnSched
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
@@ -149,7 +151,7 @@ import com.google.common.util.concurrent.SettableFuture;
public class CapacityScheduler extends
AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> implements
PreemptableResourceScheduler, CapacitySchedulerContext, Configurable,
- ResourceAllocationCommitter {
+ ResourceAllocationCommitter, MutableConfScheduler {
private static final Log LOG = LogFactory.getLog(CapacityScheduler.class);
@@ -2486,4 +2488,16 @@ public class CapacityScheduler extends
writeLock.unlock();
}
}
+
+ @Override
+ public void updateConfiguration(UserGroupInformation user,
+ Map<String, String> confUpdate) throws IOException {
+ if (csConfProvider instanceof MutableConfigurationProvider) {
+ ((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
+ user.getShortUserName(), confUpdate);
+ } else {
+ throw new UnsupportedOperationException("Configured CS configuration " +
+ "provider does not support updating configuration.");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb7a0e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index a208fb9..b97be1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -58,7 +58,11 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore {
if (isValid) {
Map<String, String> mutations = mutation.getUpdates();
for (Map.Entry<String, String> kv : mutations.entrySet()) {
- schedConf.set(kv.getKey(), kv.getValue());
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
}
}
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb7a0e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 267ab6a..ea1b3c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -60,34 +60,44 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
Configuration initialSchedConf = new Configuration(false);
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
- this.schedConf = initialSchedConf;
- confStore.initialize(config, initialSchedConf);
+ this.schedConf = new Configuration(false);
+ // We need to explicitly set the key-values in schedConf, otherwise
+ // these configuration keys cannot be deleted when
+ // configuration is reloaded.
+ for (Map.Entry<String, String> kv : initialSchedConf) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ confStore.initialize(config, schedConf);
this.conf = config;
}
@Override
public CapacitySchedulerConfiguration loadConfiguration(Configuration
configuration) throws IOException {
- Configuration loadedConf = new Configuration(configuration);
- loadedConf.addResource(schedConf);
+ Configuration loadedConf = new Configuration(schedConf);
+ loadedConf.addResource(configuration);
return new CapacitySchedulerConfiguration(loadedConf, false);
}
@Override
public void mutateConfiguration(String user,
- Map<String, String> confUpdate) {
+ Map<String, String> confUpdate) throws IOException {
Configuration oldConf = new Configuration(schedConf);
LogMutation log = new LogMutation(confUpdate, user);
long id = confStore.logMutation(log);
for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
- schedConf.set(kv.getKey(), kv.getValue());
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
}
try {
rmContext.getScheduler().reinitialize(conf, rmContext);
} catch (IOException e) {
schedConf = oldConf;
confStore.confirmMutation(id, false);
- return;
+ throw e;
}
confStore.confirmMutation(id, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb7a0e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index bd0602b..78eb4de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -57,6 +57,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
+import com.google.common.base.Joiner;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -136,11 +137,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
@@ -2614,4 +2617,170 @@ public class RMWebServices extends WebServices {
app.getApplicationTimeouts().get(appTimeout.getTimeoutType()));
return Response.status(Status.OK).entity(timeout).build();
}
+
+ @PUT
+ @Path("/queues")
+ @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response updateSchedulerConfiguration(QueueConfigsUpdateInfo
+ mutationInfo, @Context HttpServletRequest hsr)
+ throws AuthorizationException, InterruptedException {
+ init();
+
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ ApplicationACLsManager aclsManager = rm.getApplicationACLsManager();
+ if (aclsManager.areACLsEnabled()) {
+ if (callerUGI == null || !aclsManager.isAdmin(callerUGI)) {
+ String msg = "Only admins can carry out this operation.";
+ throw new ForbiddenException(msg);
+ }
+ }
+
+ ResourceScheduler scheduler = rm.getResourceScheduler();
+ if (scheduler instanceof MutableConfScheduler) {
+ try {
+ callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException, YarnException {
+ Map<String, String> confUpdate =
+ constructKeyValueConfUpdate(mutationInfo);
+ ((CapacityScheduler) scheduler).updateConfiguration(callerUGI,
+ confUpdate);
+ return null;
+ }
+ });
+ } catch (IOException e) {
+ return Response.status(Status.BAD_REQUEST).entity(e.getMessage())
+ .build();
+ }
+ return Response.status(Status.OK).entity("Configuration change " +
+ "successfully applied.").build();
+ } else {
+ return Response.status(Status.BAD_REQUEST)
+ .entity("Configuration change only supported by CapacityScheduler.")
+ .build();
+ }
+ }
+
+ private Map<String, String> constructKeyValueConfUpdate(
+ QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ CapacitySchedulerConfiguration currentConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ CapacitySchedulerConfiguration proposedConf =
+ new CapacitySchedulerConfiguration(currentConf, false);
+ Map<String, String> confUpdate = new HashMap<>();
+ for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
+ removeQueue(queueToRemove, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
+ addQueue(addQueueInfo, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
+ updateQueue(updateQueueInfo, proposedConf, confUpdate);
+ }
+ return confUpdate;
+ }
+
+ private void removeQueue(
+ String queueToRemove, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (queueToRemove == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ String queueName = queueToRemove.substring(
+ queueToRemove.lastIndexOf('.') + 1);
+ CSQueue queue = cs.getQueue(queueName);
+ if (queue == null ||
+ !queue.getQueuePath().equals(queueToRemove)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+ } else if (queueToRemove.lastIndexOf('.') == -1) {
+ throw new IOException("Can't remove queue " + queueToRemove);
+ }
+ String parentQueuePath = queueToRemove.substring(0, queueToRemove
+ .lastIndexOf('.'));
+ String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
+ List<String> newSiblingQueues = new ArrayList<>();
+ for (String siblingQueue : siblingQueues) {
+ if (!siblingQueue.equals(queueName)) {
+ newSiblingQueues.add(siblingQueue);
+ }
+ }
+ proposedConf.setQueues(parentQueuePath, newSiblingQueues
+ .toArray(new String[0]));
+ String queuesConfig = CapacitySchedulerConfiguration.PREFIX +
+ parentQueuePath + CapacitySchedulerConfiguration.DOT +
+ CapacitySchedulerConfiguration.QUEUES;
+ if (newSiblingQueues.size() == 0) {
+ confUpdate.put(queuesConfig, null);
+ } else {
+ confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
+ }
+ for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+ ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
+ .entrySet()) {
+ proposedConf.unset(confRemove.getKey());
+ confUpdate.put(confRemove.getKey(), null);
+ }
+ }
+ }
+
+ private void addQueue(
+ QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (addInfo == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ String queuePath = addInfo.getQueue();
+ String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
+ if (cs.getQueue(queueName) != null) {
+ throw new IOException("Can't add existing queue " + queuePath);
+ } else if (queuePath.lastIndexOf('.') == -1) {
+ throw new IOException("Can't add invalid queue " + queuePath);
+ }
+ String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+ String[] siblings = proposedConf.getQueues(parentQueue);
+ List<String> siblingQueues = siblings == null ? new ArrayList<>() :
+ new ArrayList<>(Arrays.<String>asList(siblings));
+ siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
+ proposedConf.setQueues(parentQueue,
+ siblingQueues.toArray(new String[0]));
+ confUpdate.put(CapacitySchedulerConfiguration.PREFIX +
+ parentQueue + CapacitySchedulerConfiguration.DOT +
+ CapacitySchedulerConfiguration.QUEUES,
+ Joiner.on(',').join(siblingQueues));
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
+ queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
+
+ private void updateQueue(QueueConfigInfo updateInfo,
+ CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) {
+ if (updateInfo == null) {
+ return;
+ } else {
+ String queuePath = updateInfo.getQueue();
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
+ queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb7a0e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
new file mode 100644
index 0000000..b20eda6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for adding or updating a queue to scheduler configuration
+ * for this queue.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueConfigInfo {
+
+ @XmlElement(name = "queueName")
+ private String queue;
+
+ private HashMap<String, String> params = new HashMap<>();
+
+ public QueueConfigInfo() { }
+
+ public QueueConfigInfo(String queue, Map<String, String> params) {
+ this.queue = queue;
+ this.params = new HashMap<>(params);
+ }
+
+ public String getQueue() {
+ return this.queue;
+ }
+
+ public HashMap<String, String> getParams() {
+ return this.params;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb7a0e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
new file mode 100644
index 0000000..644ec90
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for making scheduler configuration changes (supports adding,
+ * removing, or updating a queue).
+ */
+@XmlRootElement(name = "schedConf")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueConfigsUpdateInfo {
+
+ @XmlElement(name = "add")
+ private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "remove")
+ private ArrayList<String> removeQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "update")
+ private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
+
+ public QueueConfigsUpdateInfo() {
+ // JAXB needs this
+ }
+
+ public ArrayList<QueueConfigInfo> getAddQueueInfo() {
+ return addQueueInfo;
+ }
+
+ public ArrayList<String> getRemoveQueueInfo() {
+ return removeQueueInfo;
+ }
+
+ public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
+ return updateQueueInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb7a0e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 3f103b1..254da31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -77,7 +77,11 @@ public class TestMutableCSConfigurationProvider {
assertNull(confProvider.loadConfiguration(conf).get("badKey"));
doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
- confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ try {
+ confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ } catch (IOException e) {
+ // Expected exception.
+ }
assertNull(confProvider.loadConfiguration(conf).get("badKey"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb7a0e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
new file mode 100644
index 0000000..d149055
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -0,0 +1,477 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.Status;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Test scheduler configuration mutation via REST API.
+ */
+public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
+
+ private static final File CONF_FILE = new File(new File("target",
+ "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
+ private static final File OLD_CONF_FILE = new File(new File("target",
+ "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE + ".tmp");
+
+ private static MockRM rm;
+ private static String userName;
+ private static CapacitySchedulerConfiguration csConf;
+ private static YarnConfiguration conf;
+
+ private static class WebServletModule extends ServletModule {
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ try {
+ userName = UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException ioe) {
+ throw new RuntimeException("Unable to get current user name "
+ + ioe.getMessage(), ioe);
+ }
+ csConf = new CapacitySchedulerConfiguration(new Configuration(false),
+ false);
+ setupQueueConfiguration(csConf);
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName);
+ try {
+ if (CONF_FILE.exists()) {
+ if (!CONF_FILE.renameTo(OLD_CONF_FILE)) {
+ throw new RuntimeException("Failed to rename conf file");
+ }
+ }
+ FileOutputStream out = new FileOutputStream(CONF_FILE);
+ csConf.writeXml(out);
+ out.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to write XML file", e);
+ }
+ rm = new MockRM(conf);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ filter("/*").through(TestRMWebServicesAppsModification
+ .TestRMCustomAuthFilter.class);
+ }
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ GuiceServletConfig.setInjector(
+ Guice.createInjector(new WebServletModule()));
+ }
+
+ private static void setupQueueConfiguration(
+ CapacitySchedulerConfiguration config) {
+ config.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{"a", "b", "c"});
+
+ final String a = CapacitySchedulerConfiguration.ROOT + ".a";
+ config.setCapacity(a, 25f);
+ config.setMaximumCapacity(a, 50f);
+
+ final String a1 = a + ".a1";
+ final String a2 = a + ".a2";
+ config.setQueues(a, new String[]{"a1", "a2"});
+ config.setCapacity(a1, 100f);
+ config.setCapacity(a2, 0f);
+
+ final String b = CapacitySchedulerConfiguration.ROOT + ".b";
+ config.setCapacity(b, 75f);
+
+ final String c = CapacitySchedulerConfiguration.ROOT + ".c";
+ config.setCapacity(c, 0f);
+
+ final String c1 = c + ".c1";
+ config.setQueues(c, new String[] {"c1"});
+ config.setCapacity(c1, 0f);
+ }
+
+ public TestRMWebServicesConfigurationMutation() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testAddNestedQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Add parent queue root.d with two children d1 and d2.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> d1Capacity = new HashMap<>();
+ d1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
+ d1Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "25");
+ Map<String, String> nearEmptyCapacity = new HashMap<>();
+ nearEmptyCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "1E-4");
+ nearEmptyCapacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY,
+ "1E-4");
+ Map<String, String> d2Capacity = new HashMap<>();
+ d2Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "75");
+ d2Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "75");
+ QueueConfigInfo d1 = new QueueConfigInfo("root.d.d1", d1Capacity);
+ QueueConfigInfo d2 = new QueueConfigInfo("root.d.d2", d2Capacity);
+ QueueConfigInfo d = new QueueConfigInfo("root.d", nearEmptyCapacity);
+ updateInfo.getAddQueueInfo().add(d1);
+ updateInfo.getAddQueueInfo().add(d2);
+ updateInfo.getAddQueueInfo().add(d);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(4, newCSConf.getQueues("root").length);
+ assertEquals(2, newCSConf.getQueues("root.d").length);
+ assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d1"),
+ 0.01f);
+ assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"),
+ 0.01f);
+ }
+
+ @Test
+ public void testAddWithUpdate() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Add root.d with capacity 25, reducing root.b capacity from 75 to 50.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> dCapacity = new HashMap<>();
+ dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
+ Map<String, String> bCapacity = new HashMap<>();
+ bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "50");
+ QueueConfigInfo d = new QueueConfigInfo("root.d", dCapacity);
+ QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity);
+ updateInfo.getAddQueueInfo().add(d);
+ updateInfo.getUpdateQueueInfo().add(b);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(4, newCSConf.getQueues("root").length);
+ assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d"), 0.01f);
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
+ }
+
+ @Test
+ public void testRemoveQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.a.a2");
+ // Remove root.a.a2
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.a.a2");
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(1, newCSConf.getQueues("root.a").length);
+ assertEquals("a1", newCSConf.getQueues("root.a")[0]);
+ }
+
+ @Test
+ public void testRemoveParentQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.c", "root.c.c1");
+ // Remove root.c (parent queue)
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.c");
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(2, newCSConf.getQueues("root").length);
+ assertNull(newCSConf.getQueues("root.c"));
+ }
+
+ @Test
+ public void testRemoveParentQueueWithCapacity() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.a", "root.a.a1", "root.a.a2");
+ // Remove root.a (parent queue) with capacity 25
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.a");
+
+ // Set root.b capacity to 100
+ Map<String, String> bCapacity = new HashMap<>();
+ bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100");
+ QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity);
+ updateInfo.getUpdateQueueInfo().add(b);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(2, newCSConf.getQueues("root").length);
+ assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity("root.b"),
+ 0.01f);
+ }
+
+ @Test
+ public void testRemoveMultipleQueues() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.b", "root.c", "root.c.c1");
+ // Remove root.b and root.c
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.b");
+ updateInfo.getRemoveQueueInfo().add("root.c");
+ Map<String, String> aCapacity = new HashMap<>();
+ aCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100");
+ aCapacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "100");
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.a", aCapacity);
+ updateInfo.getUpdateQueueInfo().add(configInfo);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(1, newCSConf.getQueues("root").length);
+ }
+
+ private void stopQueue(String... queuePaths) throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Set state of queues to STOPPED.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> stoppedParam = new HashMap<>();
+ stoppedParam.put(CapacitySchedulerConfiguration.STATE,
+ QueueState.STOPPED.toString());
+ for (String queue : queuePaths) {
+ QueueConfigInfo stoppedInfo = new QueueConfigInfo(queue, stoppedParam);
+ updateInfo.getUpdateQueueInfo().add(stoppedInfo);
+ }
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ for (String queue : queuePaths) {
+ assertEquals(QueueState.STOPPED, newCSConf.getState(queue));
+ }
+ }
+
+ @Test
+ public void testUpdateQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Update config value.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
+ "0.2");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
+ cs.getConfiguration()
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"),
+ 0.001f);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
+ assertEquals(0.2f, newCSConf
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"), 0.001f);
+
+ // Remove config. Config value should be reverted to default.
+ updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
+ null);
+ aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ updateInfo.getUpdateQueueInfo().clear();
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ newCSConf = cs.getConfiguration();
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, newCSConf
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"),
+ 0.001f);
+ }
+
+ @Test
+ public void testUpdateQueueCapacity() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Update root.a and root.b capacity to 50.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ QueueConfigInfo bUpdateInfo = new QueueConfigInfo("root.b", updateParam);
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ updateInfo.getUpdateQueueInfo().add(bUpdateInfo);
+
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.a"), 0.01f);
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ if (rm != null) {
+ rm.stop();
+ }
+ CONF_FILE.delete();
+ if (!OLD_CONF_FILE.renameTo(CONF_FILE)) {
+ throw new RuntimeException("Failed to re-copy old configuration file");
+ }
+ super.tearDown();
+ }
+
+ @SuppressWarnings("rawtypes")
+ private String toJson(Object nsli, Class klass) throws Exception {
+ StringWriter sw = new StringWriter();
+ JSONJAXBContext ctx = new JSONJAXBContext(klass);
+ JSONMarshaller jm = ctx.createJSONMarshaller();
+ jm.marshallToJSON(nsli, sw);
+ return sw.toString();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] hadoop git commit: HADOOP-11869. Suppress ParameterNumber
checkstyle violations for overridden methods. Contributed by Jonathan Eagles
Posted by wa...@apache.org.
HADOOP-11869. Suppress ParameterNumber checkstyle violations for overridden methods. Contributed by Jonathan Eagles
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/009b9f3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/009b9f3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/009b9f3e
Branch: refs/heads/YARN-5734
Commit: 009b9f3ebc6e673688a281d1595f640eb8c3e77b
Parents: b4adc83
Author: Chris Douglas <cd...@apache.org>
Authored: Thu May 18 21:27:46 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Thu May 18 21:27:46 2017 -0700
----------------------------------------------------------------------
hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/009b9f3e/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
----------------------------------------------------------------------
diff --git a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
index 1b968ae..2053561 100644
--- a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
+++ b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
@@ -125,7 +125,9 @@
<!-- See http://checkstyle.sf.net/config_sizes.html -->
<module name="LineLength"/>
<module name="MethodLength"/>
- <module name="ParameterNumber"/>
+ <module name="ParameterNumber">
+ <property name="ignoreOverriddenMethods" value="true"/>
+ </module>
<!-- Checks for whitespace -->
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] hadoop git commit: YARN-5948. Implement
MutableConfigurationManager for handling storage into configuration store
Posted by wa...@apache.org.
YARN-5948. Implement MutableConfigurationManager for handling storage into configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6139e47b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6139e47b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6139e47b
Branch: refs/heads/YARN-5734
Commit: 6139e47b7aef000cad0c49b278342a2de7d6e51e
Parents: 9cf1c93c
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Wed Mar 1 16:03:01 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Sat May 20 08:22:43 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 6 ++
.../src/main/resources/yarn-default.xml | 12 +++
.../scheduler/MutableConfigurationProvider.java | 35 ++++++++
.../scheduler/capacity/CapacityScheduler.java | 14 ++-
.../CapacitySchedulerConfiguration.java | 3 +
.../capacity/conf/CSConfigurationProvider.java | 3 +-
.../conf/MutableCSConfigurationProvider.java | 94 ++++++++++++++++++++
.../conf/YarnConfigurationStoreFactory.java | 46 ++++++++++
.../TestMutableCSConfigurationProvider.java | 83 +++++++++++++++++
9 files changed, 291 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6139e47b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 5e4c826..fa9e79d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -607,6 +607,12 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
"org.apache.hadoop.yarn.LocalConfigurationProvider";
+ public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
+ YARN_PREFIX + "scheduler.configuration.store.class";
+ public static final String MEMORY_CONFIGURATION_STORE = "memory";
+ public static final String DEFAULT_CONFIGURATION_STORE =
+ MEMORY_CONFIGURATION_STORE;
+
public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX
+ "authorization-provider";
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6139e47b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e687eef..02284e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3108,4 +3108,16 @@
<value>false</value>
</property>
+ <property>
+ <description>
+ The type of configuration store to use for storing scheduler
+ configurations, if using a mutable configuration provider.
+ Keywords such as "memory" map to certain configuration store
+ implementations. If keyword is not found, try to load this
+ value as a class.
+ </description>
+ <name>yarn.scheduler.configuration.store.class</name>
+ <value>memory</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6139e47b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
new file mode 100644
index 0000000..da30a2b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.Map;
+
+/**
+ * Interface for allowing changing scheduler configurations.
+ */
+public interface MutableConfigurationProvider {
+
+ /**
+ * Update the scheduler configuration with the provided key value pairs.
+ * @param user User issuing the request
+ * @param confUpdate Key-value pairs for configurations to be updated.
+ */
+ void mutateConfiguration(String user, Map<String, String> confUpdate);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6139e47b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index a925904..9353df2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -105,6 +105,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.MutableCSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation;
@@ -290,10 +291,15 @@ public class CapacityScheduler extends
String confProviderStr = configuration.get(
CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
- if (confProviderStr.equals(
- CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) {
- this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext);
- } else {
+ switch (confProviderStr) {
+ case CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER:
+ this.csConfProvider =
+ new FileBasedCSConfigurationProvider(rmContext);
+ break;
+ case CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER:
+ this.csConfProvider = new MutableCSConfigurationProvider(rmContext);
+ break;
+ default:
throw new IOException("Invalid CS configuration provider: " +
confProviderStr);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6139e47b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index c53a588..7f516de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -300,6 +300,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String FILE_CS_CONF_PROVIDER = "file";
@Private
+ public static final String STORE_CS_CONF_PROVIDER = "store";
+
+ @Private
public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6139e47b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
index c9984ac..0d2c8bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
@@ -32,8 +32,9 @@ public interface CSConfigurationProvider {
/**
* Initialize the configuration provider with given conf.
* @param conf configuration to initialize with
+ * @throws IOException if initialization fails due to misconfiguration
*/
- void init(Configuration conf);
+ void init(Configuration conf) throws IOException;
/**
* Loads capacity scheduler configuration object.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6139e47b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
new file mode 100644
index 0000000..267ab6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * CS configuration provider which implements
+ * {@link MutableConfigurationProvider} for modifying capacity scheduler
+ * configuration.
+ */
+public class MutableCSConfigurationProvider implements CSConfigurationProvider,
+ MutableConfigurationProvider {
+
+ private Configuration schedConf;
+ private YarnConfigurationStore confStore;
+ private RMContext rmContext;
+ private Configuration conf;
+
+ public MutableCSConfigurationProvider(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void init(Configuration config) throws IOException {
+ String store = config.get(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
+ switch (store) {
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ this.confStore = new InMemoryConfigurationStore();
+ break;
+ default:
+ this.confStore = YarnConfigurationStoreFactory.getStore(config);
+ break;
+ }
+ Configuration initialSchedConf = new Configuration(false);
+ initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
+ this.schedConf = initialSchedConf;
+ confStore.initialize(config, initialSchedConf);
+ this.conf = config;
+ }
+
+ @Override
+ public CapacitySchedulerConfiguration loadConfiguration(Configuration
+ configuration) throws IOException {
+ Configuration loadedConf = new Configuration(configuration);
+ loadedConf.addResource(schedConf);
+ return new CapacitySchedulerConfiguration(loadedConf, false);
+ }
+
+ @Override
+ public void mutateConfiguration(String user,
+ Map<String, String> confUpdate) {
+ Configuration oldConf = new Configuration(schedConf);
+ LogMutation log = new LogMutation(confUpdate, user);
+ long id = confStore.logMutation(log);
+ for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ try {
+ rmContext.getScheduler().reinitialize(conf, rmContext);
+ } catch (IOException e) {
+ schedConf = oldConf;
+ confStore.confirmMutation(id, false);
+ return;
+ }
+ confStore.confirmMutation(id, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6139e47b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
new file mode 100644
index 0000000..60249c8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Factory class for creating instances of {@link YarnConfigurationStore}.
+ */
+public final class YarnConfigurationStoreFactory {
+
+ private static final Log LOG = LogFactory.getLog(
+ YarnConfigurationStoreFactory.class);
+
+ private YarnConfigurationStoreFactory() {
+ // Unused.
+ }
+
+ public static YarnConfigurationStore getStore(Configuration conf) {
+ Class<? extends YarnConfigurationStore> storeClass =
+ conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ InMemoryConfigurationStore.class, YarnConfigurationStore.class);
+ LOG.info("Using YarnConfigurationStore implementation - " + storeClass);
+ return ReflectionUtils.newInstance(storeClass, conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6139e47b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
new file mode 100644
index 0000000..3f103b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests {@link MutableCSConfigurationProvider}.
+ */
+public class TestMutableCSConfigurationProvider {
+
+ private MutableCSConfigurationProvider confProvider;
+ private RMContext rmContext;
+ private Map<String, String> goodUpdate;
+ private Map<String, String> badUpdate;
+ private CapacityScheduler cs;
+
+ private static final String TEST_USER = "testUser";
+
+ @Before
+ public void setUp() {
+ cs = mock(CapacityScheduler.class);
+ rmContext = mock(RMContext.class);
+ when(rmContext.getScheduler()).thenReturn(cs);
+ confProvider = new MutableCSConfigurationProvider(rmContext);
+ goodUpdate = new HashMap<>();
+ goodUpdate.put("goodKey", "goodVal");
+ badUpdate = new HashMap<>();
+ badUpdate.put("badKey", "badVal");
+ }
+
+ @Test
+ public void testInMemoryBackedProvider() throws IOException {
+ Configuration conf = new Configuration();
+ confProvider.init(conf);
+ assertNull(confProvider.loadConfiguration(conf)
+ .get("goodKey"));
+
+ doNothing().when(cs).reinitialize(any(Configuration.class),
+ any(RMContext.class));
+ confProvider.mutateConfiguration(TEST_USER, goodUpdate);
+ assertEquals("goodVal", confProvider.loadConfiguration(conf)
+ .get("goodKey"));
+
+ assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
+ any(RMContext.class));
+ confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] hadoop git commit: HADOOP-11572. s3a delete() operation fails
during a concurrent delete of child entries. Contributed by Steve Loughran.
Posted by wa...@apache.org.
HADOOP-11572. s3a delete() operation fails during a concurrent delete of child entries.
Contributed by Steve Loughran.
(cherry picked from commit 2ac5aab8d725f761a9f9723471a4426f6b5d78c4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba70225c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba70225c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba70225c
Branch: refs/heads/YARN-5734
Commit: ba70225cf6a1e7dc756f4991881de04f525ff088
Parents: b46cd31
Author: Steve Loughran <st...@apache.org>
Authored: Thu May 18 15:44:39 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu May 18 15:44:39 2017 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 32 ++++++++++--
.../src/site/markdown/tools/hadoop-aws/index.md | 25 ++++++++++
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 51 ++++++++++++++++++++
3 files changed, 103 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba70225c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 4082c8a..d53fd13 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -49,6 +49,7 @@ import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.MultiObjectDeleteException;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PartETag;
@@ -455,7 +456,8 @@ public class S3AFileSystem extends FileSystem {
* @param path input path, may be relative to the working dir
* @return a key excluding the leading "/", or, if it is the root path, ""
*/
- private String pathToKey(Path path) {
+ @VisibleForTesting
+ String pathToKey(Path path) {
if (!path.isAbsolute()) {
path = new Path(workingDir, path);
}
@@ -1011,11 +1013,26 @@ public class S3AFileSystem extends FileSystem {
* Increments the {@code OBJECT_DELETE_REQUESTS} and write
* operation statistics.
* @param deleteRequest keys to delete on the s3-backend
+ * @throws MultiObjectDeleteException one or more of the keys could not
+ * be deleted.
+ * @throws AmazonClientException amazon-layer failure.
*/
- private void deleteObjects(DeleteObjectsRequest deleteRequest) {
+ private void deleteObjects(DeleteObjectsRequest deleteRequest)
+ throws MultiObjectDeleteException, AmazonClientException {
incrementWriteOperations();
incrementStatistic(OBJECT_DELETE_REQUESTS, 1);
- s3.deleteObjects(deleteRequest);
+ try {
+ s3.deleteObjects(deleteRequest);
+ } catch (MultiObjectDeleteException e) {
+ // one or more of the operations failed.
+ List<MultiObjectDeleteException.DeleteError> errors = e.getErrors();
+ LOG.error("Partial failure of delete, {} errors", errors.size(), e);
+ for (MultiObjectDeleteException.DeleteError error : errors) {
+ LOG.error("{}: \"{}\" - {}",
+ error.getKey(), error.getCode(), error.getMessage());
+ }
+ throw e;
+ }
}
/**
@@ -1225,10 +1242,15 @@ public class S3AFileSystem extends FileSystem {
* @param deleteFakeDir indicates whether this is for deleting fake dirs
* @throws InvalidRequestException if the request was rejected due to
* a mistaken attempt to delete the root directory.
+ * @throws MultiObjectDeleteException one or more of the keys could not
+ * be deleted in a multiple object delete operation.
+ * @throws AmazonClientException amazon-layer failure.
*/
- private void removeKeys(List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+ @VisibleForTesting
+ void removeKeys(List<DeleteObjectsRequest.KeyVersion> keysToDelete,
boolean clearKeys, boolean deleteFakeDir)
- throws AmazonClientException, InvalidRequestException {
+ throws MultiObjectDeleteException, AmazonClientException,
+ InvalidRequestException {
if (keysToDelete.isEmpty()) {
// exit fast if there are no keys to delete
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba70225c/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index ceda79e..e5aa431 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -1874,6 +1874,31 @@ Consult [Cleaning up After Incremental Upload Failures](#s3a_multipart_purge) fo
details on how the multipart purge timeout can be set. If multipart uploads
are failing with the message above, it may be a sign that this value is too low.
+### `MultiObjectDeleteException` during delete or rename of files
+
+```
+Exception in thread "main" com.amazonaws.services.s3.model.MultiObjectDeleteException:
+ Status Code: 0, AWS Service: null, AWS Request ID: null, AWS Error Code: null,
+ AWS Error Message: One or more objects could not be deleted, S3 Extended Request ID: null
+ at com.amazonaws.services.s3.AmazonS3Client.deleteObjects(AmazonS3Client.java:1745)
+```
+This happens when trying to delete multiple objects, and one of the objects
+could not be deleted. It *should not occur* just because the object is missing.
+More specifically: at the time this document was written, we could not create
+such a failure.
+
+It will occur if the caller lacks the permission to delete any of the objects.
+
+Consult the log to see the specifics of which objects could not be deleted.
+Do you have permission to do so?
+
+If this operation is failing for reasons other than the caller lacking
+permissions:
+
+1. Try setting `fs.s3a.multiobjectdelete.enable` to `false`.
+1. Consult [HADOOP-11572](https://issues.apache.org/jira/browse/HADOOP-11572)
+for up to date advice.
+
### When writing to S3A, HTTP Exceptions logged at info from `AmazonHttpClient`
```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba70225c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
index 7cd1094..7d03668 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
@@ -18,17 +18,24 @@
package org.apache.hadoop.fs.s3a;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import com.amazonaws.services.s3.model.MultiObjectDeleteException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.InvalidRequestException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assume;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.EOFException;
import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.Callable;
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
@@ -41,6 +48,13 @@ public class ITestS3AFailureHandling extends AbstractS3ATestBase {
private static final Logger LOG =
LoggerFactory.getLogger(ITestS3AFailureHandling.class);
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ conf.setBoolean(Constants.ENABLE_MULTI_DELETE, true);
+ return conf;
+ }
@Test
public void testReadFileChanged() throws Throwable {
describe("overwrite a file with a shorter one during a read, seek");
@@ -127,4 +141,41 @@ public class ITestS3AFailureHandling extends AbstractS3ATestBase {
assertEquals("Expected EOF from "+ operation
+ "; got char " + (char) readResult, -1, readResult);
}
+
+ @Test
+ public void testMultiObjectDeleteNoFile() throws Throwable {
+ describe("Deleting a missing object");
+ removeKeys(getFileSystem(), "ITestS3AFailureHandling/missingFile");
+ }
+
+ private void removeKeys(S3AFileSystem fileSystem, String... keys)
+ throws InvalidRequestException {
+ List<DeleteObjectsRequest.KeyVersion> request = new ArrayList<>(
+ keys.length);
+ for (String key : keys) {
+ request.add(new DeleteObjectsRequest.KeyVersion(key));
+ }
+ fileSystem.removeKeys(request, false, false);
+ }
+
+ @Test
+ public void testMultiObjectDeleteSomeFiles() throws Throwable {
+ Path valid = path("ITestS3AFailureHandling/validFile");
+ touch(getFileSystem(), valid);
+ NanoTimer timer = new NanoTimer();
+ removeKeys(getFileSystem(), getFileSystem().pathToKey(valid),
+ "ITestS3AFailureHandling/missingFile");
+ timer.end("removeKeys");
+ }
+
+ @Test(expected = MultiObjectDeleteException.class)
+ public void testMultiObjectDeleteNoPermissions() throws Throwable {
+ Configuration conf = getConfiguration();
+ String csvFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
+ Assume.assumeTrue("CSV test file is not the default",
+ DEFAULT_CSVTEST_FILE.equals(csvFile));
+ Path testFile = new Path(csvFile);
+ S3AFileSystem fs = (S3AFileSystem)FileSystem.newInstance(testFile.toUri(), conf);
+ removeKeys(fs, fs.pathToKey(testFile));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] hadoop git commit: HADOOP-14410. Correct spelling of
'beginning' and variants. Contributed By Dongtao Zhang
Posted by wa...@apache.org.
HADOOP-14410. Correct spelling of 'beginning' and variants. Contributed By Dongtao Zhang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4aae2d40
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4aae2d40
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4aae2d40
Branch: refs/heads/YARN-5734
Commit: 4aae2d40a3e2e732e09b4b8a82623cacc0dc8861
Parents: 51b671e
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Thu May 11 23:33:45 2017 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu May 11 23:33:45 2017 +0530
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java | 2 +-
.../src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java | 2 +-
.../apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java | 2 +-
.../hadoop/security/ShellBasedUnixGroupsNetgroupMapping.java | 2 +-
.../monitor/capacity/QueuePriorityContainerCandidateSelector.java | 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aae2d40/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index fe2f8a9..8b90f53 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -135,7 +135,7 @@ class ChRootedFileSystem extends FilterFileSystem {
/**
* Strip out the root from the path.
* @param p - fully qualified path p
- * @return - the remaining path without the begining /
+ * @return - the remaining path without the beginning /
* @throws IOException if the p is not prefixed with root
*/
String stripOutRoot(final Path p) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aae2d40/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
index ad1f5b5..6168df0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
@@ -132,7 +132,7 @@ class ChRootedFs extends AbstractFileSystem {
* Strip out the root from the path.
*
* @param p - fully qualified path p
- * @return - the remaining path without the begining /
+ * @return - the remaining path without the beginning /
*/
public String stripOutRoot(final Path p) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aae2d40/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java
index fc3c4a7..ff4ab98 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java
@@ -112,7 +112,7 @@ public class JniBasedUnixGroupsNetgroupMapping
protected synchronized List<String> getUsersForNetgroup(String netgroup) {
String[] users = null;
try {
- // JNI code does not expect '@' at the begining of the group name
+ // JNI code does not expect '@' at the beginning of the group name
users = getUsersForNetgroupJNI(netgroup.substring(1));
} catch (Exception e) {
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aae2d40/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsNetgroupMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsNetgroupMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsNetgroupMapping.java
index 0ee1c60..4aa4e9f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsNetgroupMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsNetgroupMapping.java
@@ -133,7 +133,7 @@ public class ShellBasedUnixGroupsNetgroupMapping
throws IOException {
String result = "";
try {
- // shell command does not expect '@' at the begining of the group name
+ // shell command does not expect '@' at the beginning of the group name
result = Shell.execCommand(
Shell.getUsersForNetgroupCommand(netgroup.substring(1)));
} catch (ExitCodeException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aae2d40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
index 438bce9..c730a2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
@@ -425,7 +425,7 @@ public class QueuePriorityContainerCandidateSelector
long currentTime = System.currentTimeMillis();
- // From the begining of the list
+ // From the beginning of the list
for (RMContainer reservedContainer : reservedContainers) {
// Only try to preempt reserved container after reserved container created
// and cannot be allocated after minTimeout
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] hadoop git commit: HADOOP-14415. Use java.lang.AssertionError
instead of junit.framework.AssertionFailedError. Contributed by Chen Liang.
Posted by wa...@apache.org.
HADOOP-14415. Use java.lang.AssertionError instead of junit.framework.AssertionFailedError. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4aa9e3c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4aa9e3c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4aa9e3c
Branch: refs/heads/YARN-5734
Commit: d4aa9e3c699f6a584ed020851189a81f825ce915
Parents: 035d468
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed May 17 15:38:28 2017 -0400
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed May 17 15:38:28 2017 -0400
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/fs/TestFsShell.java | 3 +--
.../src/test/java/org/apache/hadoop/net/TestNetUtils.java | 10 ++++------
.../hadoop/fs/swift/TestSwiftFileSystemContract.java | 3 +--
3 files changed, 6 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4aa9e3c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
index 162a942..da93196 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.fs;
-import junit.framework.AssertionFailedError;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFactory;
@@ -45,7 +44,7 @@ public class TestFsShell {
}
if (!(th instanceof RuntimeException)) {
- throw new AssertionFailedError("Expected Runtime exception, got: " + th)
+ throw new AssertionError("Expected Runtime exception, got: " + th)
.initCause(th);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4aa9e3c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
index e59ac77..1375d9b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
@@ -37,8 +37,6 @@ import java.util.Enumeration;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import junit.framework.AssertionFailedError;
-
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -328,7 +326,7 @@ public class TestNetUtils {
private void assertInException(Exception e, String text) throws Throwable {
String message = extractExceptionMessage(e);
if (!(message.contains(text))) {
- throw new AssertionFailedError("Wrong text in message "
+ throw new AssertionError("Wrong text in message "
+ "\"" + message + "\""
+ " expected \"" + text + "\"")
.initCause(e);
@@ -339,7 +337,7 @@ public class TestNetUtils {
assertNotNull("Null Exception", e);
String message = e.getMessage();
if (message == null) {
- throw new AssertionFailedError("Empty text in exception " + e)
+ throw new AssertionError("Empty text in exception " + e)
.initCause(e);
}
return message;
@@ -349,7 +347,7 @@ public class TestNetUtils {
throws Throwable{
String message = extractExceptionMessage(e);
if (message.contains(text)) {
- throw new AssertionFailedError("Wrong text in message "
+ throw new AssertionError("Wrong text in message "
+ "\"" + message + "\""
+ " did not expect \"" + text + "\"")
.initCause(e);
@@ -364,7 +362,7 @@ public class TestNetUtils {
"localhost", LOCAL_PORT, e);
LOG.info(wrapped.toString(), wrapped);
if(!(wrapped.getClass().equals(expectedClass))) {
- throw new AssertionFailedError("Wrong exception class; expected "
+ throw new AssertionError("Wrong exception class; expected "
+ expectedClass
+ " got " + wrapped.getClass() + ": " + wrapped).initCause(wrapped);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4aa9e3c/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java
index 46a5f0f..76716b2 100644
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.fs.swift;
-import junit.framework.AssertionFailedError;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -119,7 +118,7 @@ public class TestSwiftFileSystemContract
public void testWriteReadAndDeleteEmptyFile() throws Exception {
try {
super.testWriteReadAndDeleteEmptyFile();
- } catch (AssertionFailedError e) {
+ } catch (AssertionError e) {
SwiftTestUtils.downgrade("empty files get mistaken for directories", e);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] hadoop git commit: HADOOP-14432. S3A copyFromLocalFile to be
robust, tested. Contributed by Steve Loughran
Posted by wa...@apache.org.
HADOOP-14432. S3A copyFromLocalFile to be robust, tested. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6672810e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6672810e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6672810e
Branch: refs/heads/YARN-5734
Commit: 6672810eeac1c94fd764aaf2a709ace9d4b5aa76
Parents: 19482e0
Author: Mingliang Liu <li...@apache.org>
Authored: Fri May 19 11:51:43 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Fri May 19 11:51:43 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 29 +++-
.../fs/s3a/ITestS3ACopyFromLocalFile.java | 153 +++++++++++++++++++
2 files changed, 175 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6672810e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index d53fd13..25f2671 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -1736,28 +1736,43 @@ public class S3AFileSystem extends FileSystem {
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file
- * @param src path
+ * @param src Source path: must be on local filesystem
* @param dst path
* @throws IOException IO problem
* @throws FileAlreadyExistsException the destination file exists and
- * overwrite==false
+ * overwrite==false, or if the destination is a directory.
+ * @throws FileNotFoundException if the source file does not exit
* @throws AmazonClientException failure in the AWS SDK
+ * @throws IllegalArgumentException if the source path is not on the local FS
*/
private void innerCopyFromLocalFile(boolean delSrc, boolean overwrite,
Path src, Path dst)
throws IOException, FileAlreadyExistsException, AmazonClientException {
incrementStatistic(INVOCATION_COPY_FROM_LOCAL_FILE);
- final String key = pathToKey(dst);
-
- if (!overwrite && exists(dst)) {
- throw new FileAlreadyExistsException(dst + " already exists");
- }
LOG.debug("Copying local file from {} to {}", src, dst);
// Since we have a local file, we don't need to stream into a temporary file
LocalFileSystem local = getLocal(getConf());
File srcfile = local.pathToFile(src);
+ if (!srcfile.exists()) {
+ throw new FileNotFoundException("No file: " + src);
+ }
+ if (!srcfile.isFile()) {
+ throw new FileNotFoundException("Not a file: " + src);
+ }
+ try {
+ FileStatus status = getFileStatus(dst);
+ if (!status.isFile()) {
+ throw new FileAlreadyExistsException(dst + " exists and is not a file");
+ }
+ if (!overwrite) {
+ throw new FileAlreadyExistsException(dst + " already exists");
+ }
+ } catch (FileNotFoundException e) {
+ // no destination, all is well
+ }
+ final String key = pathToKey(dst);
final ObjectMetadata om = newObjectMetadata(srcfile.length());
PutObjectRequest putObjectRequest = newPutObjectRequest(key, om, srcfile);
Upload up = putObject(putObjectRequest);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6672810e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
new file mode 100644
index 0000000..71776ac
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.Charset;
+
+import org.junit.Test;
+
+import org.apache.commons.io.Charsets;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test {@link S3AFileSystem#copyFromLocalFile(boolean, boolean, Path, Path)}.
+ */
+public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
+ private static final Charset ASCII = Charsets.US_ASCII;
+
+ private File file;
+
+ @Override
+ public void teardown() throws Exception {
+ super.teardown();
+ if (file != null) {
+ file.delete();
+ }
+ }
+
+ @Test
+ public void testCopyEmptyFile() throws Throwable {
+ file = File.createTempFile("test", ".txt");
+ Path dest = upload(file, true);
+ assertPathExists("uploaded file", dest);
+ }
+
+ @Test
+ public void testCopyFile() throws Throwable {
+ String message = "hello";
+ file = createTempFile(message);
+ Path dest = upload(file, true);
+ assertPathExists("uploaded file not found", dest);
+ S3AFileSystem fs = getFileSystem();
+ S3AFileStatus status = fs.getFileStatus(dest);
+ assertEquals("File length of " + status,
+ message.getBytes(ASCII).length, status.getLen());
+ assertFileTextEquals(dest, message);
+ }
+
+ public void assertFileTextEquals(Path path, String expected)
+ throws IOException {
+ assertEquals("Wrong data in " + path,
+ expected, IOUtils.toString(getFileSystem().open(path), ASCII));
+ }
+
+ @Test
+ public void testCopyFileNoOverwrite() throws Throwable {
+ file = createTempFile("hello");
+ Path dest = upload(file, true);
+ intercept(FileAlreadyExistsException.class,
+ () -> upload(file, false));
+ }
+
+ @Test
+ public void testCopyFileOverwrite() throws Throwable {
+ file = createTempFile("hello");
+ Path dest = upload(file, true);
+ String updated = "updated";
+ FileUtils.write(file, updated, ASCII);
+ upload(file, true);
+ assertFileTextEquals(dest, updated);
+ }
+
+ @Test
+ public void testCopyFileNoOverwriteDirectory() throws Throwable {
+ file = createTempFile("hello");
+ Path dest = upload(file, true);
+ S3AFileSystem fs = getFileSystem();
+ fs.delete(dest, false);
+ fs.mkdirs(dest);
+ intercept(FileAlreadyExistsException.class,
+ () -> upload(file, true));
+ }
+
+ @Test
+ public void testCopyMissingFile() throws Throwable {
+ file = File.createTempFile("test", ".txt");
+ file.delete();
+ // first upload to create
+ intercept(FileNotFoundException.class, "No file",
+ () -> upload(file, true));
+ }
+
+ @Test
+ public void testCopyDirectoryFile() throws Throwable {
+ file = File.createTempFile("test", ".txt");
+ // first upload to create
+ intercept(FileNotFoundException.class, "Not a file",
+ () -> upload(file.getParentFile(), true));
+ }
+
+
+ @Test
+ public void testLocalFilesOnly() throws Throwable {
+ Path dst = path("testLocalFilesOnly");
+ intercept(IllegalArgumentException.class,
+ () -> {
+ getFileSystem().copyFromLocalFile(false, true, dst, dst);
+ return "copy successful";
+ });
+ }
+
+ public Path upload(File srcFile, boolean overwrite) throws IOException {
+ Path src = new Path(srcFile.toURI());
+ Path dst = path(srcFile.getName());
+ getFileSystem().copyFromLocalFile(false, overwrite, src, dst);
+ return dst;
+ }
+
+ /**
+ * Create a temp file with some text.
+ * @param text text for the file
+ * @return the file
+ * @throws IOException on a failure
+ */
+ public File createTempFile(String text) throws IOException {
+ File f = File.createTempFile("test", ".txt");
+ FileUtils.write(f, text, ASCII);
+ return f;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] hadoop git commit: YARN-6447. Provide container sandbox
policies for groups (gphillips via rkanter)
Posted by wa...@apache.org.
YARN-6447. Provide container sandbox policies for groups (gphillips via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18c494a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18c494a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18c494a0
Branch: refs/heads/YARN-5734
Commit: 18c494a00c8ead768f3a868b450dceea485559df
Parents: 101852c
Author: Robert Kanter <rk...@apache.org>
Authored: Tue May 16 18:02:39 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue May 16 18:02:39 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 4 +
.../JavaSandboxLinuxContainerRuntime.java | 64 ++++++++---
.../TestJavaSandboxLinuxContainerRuntime.java | 106 ++++++++++++++++---
3 files changed, 147 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c494a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 82274fe..5e4c826 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1500,6 +1500,10 @@ public class YarnConfiguration extends Configuration {
public static final String YARN_CONTAINER_SANDBOX_POLICY =
YARN_CONTAINER_SANDBOX + ".policy";
+ /** Prefix for group to policy file mapping.*/
+ public static final String YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX =
+ YARN_CONTAINER_SANDBOX_POLICY + ".group.";
+
/** The group which will run by default without the java security manager.*/
public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP =
YARN_CONTAINER_SANDBOX + ".whitelist-group";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c494a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
index 1e5bf57..0b858bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
@@ -50,10 +50,13 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import static org.apache.hadoop.fs.Path.SEPARATOR;
import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR;
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
@@ -93,13 +96,21 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
* Accepts canonical path to a java policy file on the local filesystem.
* This file will be loaded as the base policy, any additional container
* grants will be appended to this base file. If not specified, the default
- * java.policy file provided with hadoop resources will be used.
+ * java.policy file provided with hadoop resources will be used.
* </li>
* <li>
* {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} :
* Optional setting to specify a YARN queue which will be exempt from the
* sand-boxing process.
* </li>
+ * <li>
+ * {@value
+ * YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX}$groupName :
+ * Optional setting to map groups to java policy files. The value is a path
+ * to the java policy file for $groupName. A user which is a member of
+ * multiple groups with different policies will receive the superset of all
+ * the permissions across their groups.
+ * </li>
* </ul>
*/
@InterfaceAudience.Private
@@ -138,7 +149,7 @@ public class JavaSandboxLinuxContainerRuntime
this.configuration = conf;
this.sandboxMode =
SandboxMode.get(
- this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
+ this.configuration.get(YARN_CONTAINER_SANDBOX,
YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
super.initialize(conf);
@@ -211,8 +222,10 @@ public class JavaSandboxLinuxContainerRuntime
ctx.getExecutionAttribute(CONTAINER_RUN_CMDS);
Map<String, String> env =
ctx.getContainer().getLaunchContext().getEnvironment();
+ String username =
+ ctx.getExecutionAttribute(USER);
- if(!isSandboxContainerWhitelisted(ctx, commands)) {
+ if(!isSandboxContainerWhitelisted(username, commands)) {
String tmpDirBase = configuration.get("hadoop.tmp.dir");
if (tmpDirBase == null) {
throw new ContainerExecutionException("hadoop.tmp.dir not set!");
@@ -223,6 +236,8 @@ public class JavaSandboxLinuxContainerRuntime
String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
initializePolicyDir();
+ List<String> groupPolicyFiles =
+ getGroupPolicyFiles(configuration, ctx.getExecutionAttribute(USER));
Path policyFilePath = Files.createFile(
Paths.get(policyFileDir.toString(),
containerID + "-" + NMContainerPolicyUtils.POLICY_FILE),
@@ -231,12 +246,12 @@ public class JavaSandboxLinuxContainerRuntime
containerPolicies.put(containerID, policyFilePath);
- NMContainerPolicyUtils.generatePolicyFile(
- policyOutputStream, localDirs, resources, configuration);
+ NMContainerPolicyUtils.generatePolicyFile(policyOutputStream,
+ localDirs, groupPolicyFiles, resources, configuration);
NMContainerPolicyUtils.appendSecurityFlags(
commands, env, policyFilePath, sandboxMode);
- } catch (Exception e) {
+ } catch (IOException e) {
throw new ContainerExecutionException(e);
} finally {
IOUtils.cleanup(LOG, policyOutputStream);
@@ -264,15 +279,32 @@ public class JavaSandboxLinuxContainerRuntime
return sandboxMode != SandboxMode.disabled;
}
+ private static List<String> getGroupPolicyFiles(Configuration conf,
+ String user) throws ContainerExecutionException {
+ Groups groups = Groups.getUserToGroupsMappingService(conf);
+ List<String> userGroups;
+ try {
+ userGroups = groups.getGroups(user);
+ } catch (IOException e) {
+ throw new ContainerExecutionException("Container user does not exist");
+ }
+
+ return userGroups.stream()
+ .map(group -> conf.get(YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX
+ + group))
+ .filter(groupPolicy -> groupPolicy != null)
+ .collect(Collectors.toList());
+ }
+
/**
* Determine if the container should be whitelisted (i.e. exempt from the
* Java Security Manager).
- * @param ctx The container runtime context for the requested container
+ * @param username The name of the user running the container
* @param commands The list of run commands for the container
* @return boolean value denoting whether the container should be whitelisted.
* @throws ContainerExecutionException If container user can not be resolved
*/
- private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx,
+ private boolean isSandboxContainerWhitelisted(String username,
List<String> commands) throws ContainerExecutionException {
String whitelistGroup = configuration.get(
YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
@@ -281,7 +313,7 @@ public class JavaSandboxLinuxContainerRuntime
boolean isWhitelisted = false;
try {
- userGroups = groups.getGroups(ctx.getExecutionAttribute(USER));
+ userGroups = groups.getGroups(username);
} catch (IOException e) {
throw new ContainerExecutionException("Container user does not exist");
}
@@ -399,8 +431,9 @@ public class JavaSandboxLinuxContainerRuntime
* base policy file or if it is unable to create a new policy file.
*/
static void generatePolicyFile(OutputStream policyOutStream,
- List<String> localDirs, Map<org.apache.hadoop.fs.Path,
- List<String>> resources, Configuration conf)
+ List<String> localDirs, List<String> groupPolicyPaths,
+ Map<org.apache.hadoop.fs.Path, List<String>> resources,
+ Configuration conf)
throws IOException {
String policyFilePath =
@@ -414,13 +447,16 @@ public class JavaSandboxLinuxContainerRuntime
cacheDirs.add(path.getParent().toString());
}
- if(policyFilePath == null) {
+ if (groupPolicyPaths != null) {
+ for(String policyPath : groupPolicyPaths) {
+ Files.copy(Paths.get(policyPath), policyOutStream);
+ }
+ } else if (policyFilePath == null) {
IOUtils.copyBytes(
NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE),
policyOutStream, conf, false);
} else {
Files.copy(Paths.get(policyFilePath), policyOutStream);
- policyOutStream.flush();
}
Formatter filePermissionFormat = new Formatter(policyOutStream,
@@ -484,7 +520,7 @@ public class JavaSandboxLinuxContainerRuntime
private static boolean validateJavaHome(String containerJavaHome)
throws ContainerExecutionException{
- if (System.getenv(JAVA_HOME.name()) == null){
+ if (System.getenv(JAVA_HOME.name()) == null) {
throw new ContainerExecutionException(
"JAVA_HOME is not set for NodeManager");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c494a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
index e10d0dd..bdd435e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
@@ -37,20 +37,29 @@ import org.junit.rules.ExpectedException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilePermission;
+import java.io.FileWriter;
+import java.io.IOException;
import java.io.OutputStream;
+import java.net.SocketPermission;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.security.Permission;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Formatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.LOG;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.MULTI_COMMAND_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_APPEND_FLAG;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FLAG;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.SECURITY_FLAG;
@@ -76,8 +85,9 @@ import static org.mockito.Mockito.when;
*/
public class TestJavaSandboxLinuxContainerRuntime {
- private static final String HADOOP_HOME = "hadoop.home.dir";
- private static String hadoopHomeDir = System.getProperty(HADOOP_HOME);
+ private final static String HADOOP_HOME = "hadoop.home.dir";
+ private final static String HADOOP_HOME_DIR = System.getProperty(HADOOP_HOME);
+ private final Properties baseProps = new Properties(System.getProperties());
@Rule
public ExpectedException exception = ExpectedException.none();
@@ -101,11 +111,12 @@ public class TestJavaSandboxLinuxContainerRuntime {
private final static String WHITELIST_GROUP = "captains";
private final static String CONTAINER_ID = "container_1234567890";
private final static String APPLICATION_ID = "application_1234567890";
+ private File baseTestDirectory;
@Before
public void setup() throws Exception {
- File baseTestDirectory = new File(System.getProperty("test.build.data",
+ baseTestDirectory = new File(System.getProperty("test.build.data",
System.getProperty("java.io.tmpdir", "target")),
TestJavaSandboxLinuxContainerRuntime.class.getName());
@@ -114,10 +125,8 @@ public class TestJavaSandboxLinuxContainerRuntime {
conf = new Configuration();
conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
- WHITELIST_USER + "=" + WHITELIST_GROUP + ";"
+ WHITELIST_USER + "=" + WHITELIST_GROUP + "," + NORMAL_GROUP + ";"
+ NORMAL_USER + "=" + NORMAL_GROUP + ";");
- conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
- WHITELIST_GROUP);
conf.set("hadoop.tmp.dir", baseTestDirectory.getAbsolutePath());
Files.deleteIfExists(Paths.get(baseTestDirectory.getAbsolutePath(),
@@ -151,21 +160,20 @@ public class TestJavaSandboxLinuxContainerRuntime {
runtimeContextBuilder = createRuntimeContext();
- if (hadoopHomeDir == null) {
+ if (HADOOP_HOME_DIR == null) {
System.setProperty(HADOOP_HOME, policyFile.getParent());
}
OutputStream outStream = new FileOutputStream(policyFile);
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
- .generatePolicyFile(outStream, symLinks, resources, conf);
+ .generatePolicyFile(outStream, symLinks, null, resources, conf);
outStream.close();
System.setProperty("java.security.policy", policyFile.getCanonicalPath());
securityManager = new SecurityManager();
-
}
- public ContainerRuntimeContext.Builder createRuntimeContext(){
+ public ContainerRuntimeContext.Builder createRuntimeContext(){
Container container = mock(Container.class);
ContainerLaunchContext ctx = mock(ContainerLaunchContext.class);
@@ -194,6 +202,71 @@ public class TestJavaSandboxLinuxContainerRuntime {
return builder;
}
+ public static final String SOCKET_PERMISSION_FORMAT =
+ "grant { \n" +
+ " permission %1s \"%2s\", \"%3s\";\n" +
+ "};\n";
+ public static final String RUNTIME_PERMISSION_FORMAT =
+ "grant { \n" +
+ " permission %1s \"%2s\";\n" +
+ "};\n";
+
+ @Test
+ public void testGroupPolicies()
+ throws IOException, ContainerExecutionException {
+ // Generate new policy files each containing one grant
+ File openSocketPolicyFile =
+ File.createTempFile("openSocket", "policy", baseTestDirectory);
+ File classLoaderPolicyFile =
+ File.createTempFile("createClassLoader", "policy", baseTestDirectory);
+ Permission socketPerm = new SocketPermission("localhost:0", "listen");
+ Permission runtimePerm = new RuntimePermission("createClassLoader");
+
+ StringBuilder socketPermString = new StringBuilder();
+ Formatter openSocketPolicyFormatter = new Formatter(socketPermString);
+ openSocketPolicyFormatter.format(SOCKET_PERMISSION_FORMAT,
+ socketPerm.getClass().getName(), socketPerm.getName(),
+ socketPerm.getActions());
+ FileWriter socketPermWriter = new FileWriter(openSocketPolicyFile);
+ socketPermWriter.write(socketPermString.toString());
+ socketPermWriter.close();
+
+ StringBuilder classLoaderPermString = new StringBuilder();
+ Formatter classLoaderPolicyFormatter = new Formatter(classLoaderPermString);
+ classLoaderPolicyFormatter.format(RUNTIME_PERMISSION_FORMAT,
+ runtimePerm.getClass().getName(), runtimePerm.getName());
+ FileWriter classLoaderPermWriter = new FileWriter(classLoaderPolicyFile);
+ classLoaderPermWriter.write(classLoaderPermString.toString());
+ classLoaderPermWriter.close();
+
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX +
+ WHITELIST_GROUP, openSocketPolicyFile.toString());
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX +
+ NORMAL_GROUP, classLoaderPolicyFile.toString());
+
+ String[] inputCommand = {"$JAVA_HOME/bin/java jar MyJob.jar"};
+ List<String> commands = Arrays.asList(inputCommand);
+
+ runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
+ runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
+
+ runtime.prepareContainer(runtimeContextBuilder.build());
+
+ //pull generated policy from cmd
+ Matcher policyMatches = Pattern.compile(POLICY_APPEND_FLAG + "=?([^ ]+)")
+ .matcher(commands.get(0));
+ policyMatches.find();
+ String generatedPolicy = policyMatches.group(1);
+
+ //Test that generated policy file has included both policies
+ Assert.assertTrue(
+ Files.readAllLines(Paths.get(generatedPolicy)).contains(
+ classLoaderPermString.toString().split("\n")[1]));
+ Assert.assertTrue(
+ Files.readAllLines(Paths.get(generatedPolicy)).contains(
+ socketPermString.toString().split("\n")[1]));
+ }
+
@Test
public void testGrant() throws Exception {
FilePermission grantPermission =
@@ -235,7 +308,6 @@ public class TestJavaSandboxLinuxContainerRuntime {
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
.appendSecurityFlags(commands, env, policyFilePath,
JavaSandboxLinuxContainerRuntime.SandboxMode.permissive);
-
}
@Test
@@ -247,6 +319,9 @@ public class TestJavaSandboxLinuxContainerRuntime {
};
List<String> commands = Arrays.asList(inputCommand);
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+ WHITELIST_GROUP);
+
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
runtime.prepareContainer(runtimeContextBuilder.build());
@@ -254,7 +329,6 @@ public class TestJavaSandboxLinuxContainerRuntime {
Assert.assertTrue("Command should not be modified when user is " +
"member of whitelisted group",
inputCommand[0].equals(commands.get(0)));
-
}
@Test
@@ -265,6 +339,9 @@ public class TestJavaSandboxLinuxContainerRuntime {
};
List<String> commands = Arrays.asList(inputCommand);
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+ WHITELIST_GROUP);
+
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
runtime.prepareContainer(runtimeContextBuilder.build());
@@ -283,6 +360,9 @@ public class TestJavaSandboxLinuxContainerRuntime {
};
List<String> commands = Arrays.asList(inputCommand);
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+ WHITELIST_GROUP);
+
runtimeContextBuilder.setExecutionAttribute(USER, NORMAL_USER);
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
runtime.prepareContainer(runtimeContextBuilder.build());
@@ -373,6 +453,6 @@ public class TestJavaSandboxLinuxContainerRuntime {
@After
public void cleanup(){
- System.setSecurityManager(null);
+ System.setProperties(baseProps);
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] hadoop git commit: YARN-5543. ResourceManager
SchedulingMonitor could potentially terminate the preemption checker thread.
Contributed by Min Shen.
Posted by wa...@apache.org.
YARN-5543. ResourceManager SchedulingMonitor could potentially terminate the preemption checker thread. Contributed by Min Shen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ada100d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ada100d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ada100d
Branch: refs/heads/YARN-5734
Commit: 2ada100da7cfe12946e43da2929bd80c2a8bd833
Parents: 09f28da
Author: Min Shen <ms...@linkedin.com>
Authored: Thu May 11 11:00:28 2017 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu May 11 13:25:31 2017 -0700
----------------------------------------------------------------------
.../monitor/SchedulingMonitor.java | 55 +++++++++++---------
.../monitor/TestSchedulingMonitor.java | 19 +++++++
2 files changed, 48 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ada100d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
index 03e180d..3def27f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
@@ -17,6 +17,12 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.monitor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -32,9 +38,10 @@ public class SchedulingMonitor extends AbstractService {
private final SchedulingEditPolicy scheduleEditPolicy;
private static final Log LOG = LogFactory.getLog(SchedulingMonitor.class);
- //thread which runs periodically to see the last time since a heartbeat is
- //received.
- private Thread checkerThread;
+ // ScheduledExecutorService which schedules the PreemptionChecker to run
+ // periodically.
+ private ScheduledExecutorService ses;
+ private ScheduledFuture<?> handler;
private volatile boolean stopped;
private long monitorInterval;
private RMContext rmContext;
@@ -61,17 +68,25 @@ public class SchedulingMonitor extends AbstractService {
@Override
public void serviceStart() throws Exception {
assert !stopped : "starting when already stopped";
- checkerThread = new Thread(new PreemptionChecker());
- checkerThread.setName(getName());
- checkerThread.start();
+ ses = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setName(getName());
+ return t;
+ }
+ });
+ handler = ses.scheduleAtFixedRate(new PreemptionChecker(),
+ 0, monitorInterval, TimeUnit.MILLISECONDS);
super.serviceStart();
}
@Override
public void serviceStop() throws Exception {
stopped = true;
- if (checkerThread != null) {
- checkerThread.interrupt();
+ if (handler != null) {
+ LOG.info("Stop " + getName());
+ handler.cancel(true);
+ ses.shutdown();
}
super.serviceStop();
}
@@ -84,24 +99,12 @@ public class SchedulingMonitor extends AbstractService {
private class PreemptionChecker implements Runnable {
@Override
public void run() {
- while (!stopped && !Thread.currentThread().isInterrupted()) {
- try {
- //invoke the preemption policy at a regular pace
- //the policy will generate preemption or kill events
- //managed by the dispatcher
- invokePolicy();
- } catch (YarnRuntimeException e) {
- LOG.error("YarnRuntimeException raised while executing preemption"
- + " checker, skip this run..., exception=", e);
- }
-
- // Wait before next run
- try {
- Thread.sleep(monitorInterval);
- } catch (InterruptedException e) {
- LOG.info(getName() + " thread interrupted");
- break;
- }
+ try {
+ //invoke the preemption policy
+ invokePolicy();
+ } catch (YarnRuntimeException e) {
+ LOG.error("YarnRuntimeException raised while executing preemption"
+ + " checker, skip this run..., exception=", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ada100d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
index 9ec17da..d6ac346 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.Proportion
import org.junit.Test;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.*;
public class TestSchedulingMonitor {
@@ -43,5 +44,23 @@ public class TestSchedulingMonitor {
fail("ResourceManager does not start when " +
YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS + " is set to true");
}
+
+ SchedulingEditPolicy mPolicy = mock(SchedulingEditPolicy.class);
+ when(mPolicy.getMonitoringInterval()).thenReturn(1000L);
+ SchedulingMonitor monitor = new SchedulingMonitor(rm.getRMContext(),
+ mPolicy);
+ try {
+ monitor.serviceInit(conf);
+ monitor.serviceStart();
+ } catch (Exception e) {
+ fail("SchedulingMonitor failes to start.");
+ }
+ verify(mPolicy, times(1)).editSchedule();
+ try {
+ monitor.close();
+ rm.close();
+ } catch (Exception e) {
+ fail("Failed to close.");
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] hadoop git commit: YARN-6618.
TestNMLeveldbStateStoreService#testCompactionCycle can fail if compaction
occurs more than once. Contributed by Jason Lowe.
Posted by wa...@apache.org.
YARN-6618. TestNMLeveldbStateStoreService#testCompactionCycle can fail if compaction occurs more than once. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19482e0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19482e0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19482e0d
Branch: refs/heads/YARN-5734
Commit: 19482e0d0990feb719678d71971aa849095982fd
Parents: 009b9f3
Author: Eric Payne <ep...@apache.org>
Authored: Fri May 19 11:24:54 2017 -0500
Committer: Eric Payne <ep...@apache.org>
Committed: Fri May 19 11:24:54 2017 -0500
----------------------------------------------------------------------
.../nodemanager/recovery/TestNMLeveldbStateStoreService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19482e0d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index 6909474..537f849 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -943,7 +943,7 @@ public class TestNMLeveldbStateStoreService {
};
store.init(conf);
store.start();
- verify(mockdb, timeout(10000)).compactRange(
+ verify(mockdb, timeout(10000).atLeastOnce()).compactRange(
(byte[]) isNull(), (byte[]) isNull());
store.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] hadoop git commit: HDFS-11641. Reduce cost of audit logging
by using FileStatus instead of HdfsFileStatus. Contributed by Daryn Sharp.
Posted by wa...@apache.org.
HDFS-11641. Reduce cost of audit logging by using FileStatus instead of HdfsFileStatus. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b90e52f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b90e52f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b90e52f
Branch: refs/heads/YARN-5734
Commit: 9b90e52f1ec22c18cd535af2a569defcef65b093
Parents: 1d1c52b
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue May 16 11:28:04 2017 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue May 16 11:28:46 2017 -0500
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/FSDirAclOp.java | 13 +++--
.../hdfs/server/namenode/FSDirAttrOp.java | 14 ++---
.../hdfs/server/namenode/FSDirConcatOp.java | 4 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 10 ++--
.../server/namenode/FSDirErasureCodingOp.java | 10 ++--
.../hdfs/server/namenode/FSDirMkdirOp.java | 4 +-
.../hdfs/server/namenode/FSDirRenameOp.java | 8 +--
.../hdfs/server/namenode/FSDirSymlinkOp.java | 4 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 8 +--
.../hdfs/server/namenode/FSDirXAttrOp.java | 6 +-
.../hdfs/server/namenode/FSDirectory.java | 43 +++++++++++++-
.../hdfs/server/namenode/FSNamesystem.java | 60 +++++++++++---------
12 files changed, 115 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
index efededd..cc51430 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.base.Preconditions;
+
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclEntryScope;
import org.apache.hadoop.fs.permission.AclEntryType;
@@ -26,7 +28,6 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.AclException;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
import java.io.IOException;
@@ -34,7 +35,7 @@ import java.util.Collections;
import java.util.List;
class FSDirAclOp {
- static HdfsFileStatus modifyAclEntries(
+ static FileStatus modifyAclEntries(
FSDirectory fsd, final String srcArg, List<AclEntry> aclSpec)
throws IOException {
String src = srcArg;
@@ -59,7 +60,7 @@ class FSDirAclOp {
return fsd.getAuditFileInfo(iip);
}
- static HdfsFileStatus removeAclEntries(
+ static FileStatus removeAclEntries(
FSDirectory fsd, final String srcArg, List<AclEntry> aclSpec)
throws IOException {
String src = srcArg;
@@ -84,7 +85,7 @@ class FSDirAclOp {
return fsd.getAuditFileInfo(iip);
}
- static HdfsFileStatus removeDefaultAcl(FSDirectory fsd, final String srcArg)
+ static FileStatus removeDefaultAcl(FSDirectory fsd, final String srcArg)
throws IOException {
String src = srcArg;
checkAclsConfigFlag(fsd);
@@ -108,7 +109,7 @@ class FSDirAclOp {
return fsd.getAuditFileInfo(iip);
}
- static HdfsFileStatus removeAcl(FSDirectory fsd, final String srcArg)
+ static FileStatus removeAcl(FSDirectory fsd, final String srcArg)
throws IOException {
String src = srcArg;
checkAclsConfigFlag(fsd);
@@ -127,7 +128,7 @@ class FSDirAclOp {
return fsd.getAuditFileInfo(iip);
}
- static HdfsFileStatus setAcl(
+ static FileStatus setAcl(
FSDirectory fsd, final String srcArg, List<AclEntry> aclSpec)
throws IOException {
String src = srcArg;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 4d26885..d4b24f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.StorageType;
@@ -28,7 +29,6 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
@@ -50,7 +50,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENAB
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
public class FSDirAttrOp {
- static HdfsFileStatus setPermission(
+ static FileStatus setPermission(
FSDirectory fsd, final String src, FsPermission permission)
throws IOException {
if (FSDirectory.isExactReservedName(src)) {
@@ -70,7 +70,7 @@ public class FSDirAttrOp {
return fsd.getAuditFileInfo(iip);
}
- static HdfsFileStatus setOwner(
+ static FileStatus setOwner(
FSDirectory fsd, String src, String username, String group)
throws IOException {
if (FSDirectory.isExactReservedName(src)) {
@@ -100,7 +100,7 @@ public class FSDirAttrOp {
return fsd.getAuditFileInfo(iip);
}
- static HdfsFileStatus setTimes(
+ static FileStatus setTimes(
FSDirectory fsd, String src, long mtime, long atime)
throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
@@ -153,13 +153,13 @@ public class FSDirAttrOp {
return isFile;
}
- static HdfsFileStatus unsetStoragePolicy(FSDirectory fsd, BlockManager bm,
+ static FileStatus unsetStoragePolicy(FSDirectory fsd, BlockManager bm,
String src) throws IOException {
return setStoragePolicy(fsd, bm, src,
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, "unset");
}
- static HdfsFileStatus setStoragePolicy(FSDirectory fsd, BlockManager bm,
+ static FileStatus setStoragePolicy(FSDirectory fsd, BlockManager bm,
String src, final String policyName) throws IOException {
// get the corresponding policy and make sure the policy name is valid
BlockStoragePolicy policy = bm.getStoragePolicy(policyName);
@@ -171,7 +171,7 @@ public class FSDirAttrOp {
return setStoragePolicy(fsd, bm, src, policy.getId(), "set");
}
- static HdfsFileStatus setStoragePolicy(FSDirectory fsd, BlockManager bm,
+ static FileStatus setStoragePolicy(FSDirectory fsd, BlockManager bm,
String src, final byte policyId, final String operation)
throws IOException {
if (!fsd.isStoragePolicyEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
index 40df120..6a41cd8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
@@ -21,9 +21,9 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
@@ -48,7 +48,7 @@ import static org.apache.hadoop.util.Time.now;
*/
class FSDirConcatOp {
- static HdfsFileStatus concat(FSDirectory fsd, String target, String[] srcs,
+ static FileStatus concat(FSDirectory fsd, String target, String[] srcs,
boolean logRetryCache) throws IOException {
validatePath(target, srcs);
assert srcs != null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
index d5f6be0..22039d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
@@ -41,7 +42,6 @@ import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
@@ -143,10 +143,10 @@ final class FSDirEncryptionZoneOp {
* KeyProvider
* @param logRetryCache whether to record RPC ids in editlog for retry cache
* rebuilding
- * @return HdfsFileStatus
+ * @return FileStatus
* @throws IOException
*/
- static HdfsFileStatus createEncryptionZone(final FSDirectory fsd,
+ static FileStatus createEncryptionZone(final FSDirectory fsd,
final String srcArg, final FSPermissionChecker pc, final String cipher,
final String keyName, final boolean logRetryCache) throws IOException {
final CipherSuite suite = CipherSuite.convert(cipher);
@@ -177,7 +177,7 @@ final class FSDirEncryptionZoneOp {
* @param pc permission checker to check fs permission
* @return the EZ with file status.
*/
- static Map.Entry<EncryptionZone, HdfsFileStatus> getEZForPath(
+ static Map.Entry<EncryptionZone, FileStatus> getEZForPath(
final FSDirectory fsd, final String srcArg, final FSPermissionChecker pc)
throws IOException {
final INodesInPath iip;
@@ -192,7 +192,7 @@ final class FSDirEncryptionZoneOp {
} finally {
fsd.readUnlock();
}
- HdfsFileStatus auditStat = fsd.getAuditFileInfo(iip);
+ FileStatus auditStat = fsd.getAuditFileInfo(iip);
return new AbstractMap.SimpleImmutableEntry<>(ret, auditStat);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index aa0babd..60a89e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -32,6 +32,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.FsAction;
@@ -39,7 +40,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.IllegalECPolicyException;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.WritableUtils;
@@ -100,12 +100,12 @@ final class FSDirErasureCodingOp {
* directory.
* @param logRetryCache whether to record RPC ids in editlog for retry
* cache rebuilding
- * @return {@link HdfsFileStatus}
+ * @return {@link FileStatus}
* @throws IOException
* @throws HadoopIllegalArgumentException if the policy is not enabled
* @throws AccessControlException if the user does not have write access
*/
- static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn,
+ static FileStatus setErasureCodingPolicy(final FSNamesystem fsn,
final String srcArg, final String ecPolicyName,
final FSPermissionChecker pc, final boolean logRetryCache)
throws IOException, AccessControlException {
@@ -179,11 +179,11 @@ final class FSDirErasureCodingOp {
* @param srcArg The path of the target directory.
* @param logRetryCache whether to record RPC ids in editlog for retry
* cache rebuilding
- * @return {@link HdfsFileStatus}
+ * @return {@link FileStatus}
* @throws IOException
* @throws AccessControlException if the user does not have write access
*/
- static HdfsFileStatus unsetErasureCodingPolicy(final FSNamesystem fsn,
+ static FileStatus unsetErasureCodingPolicy(final FSNamesystem fsn,
final String srcArg, final FSPermissionChecker pc,
final boolean logRetryCache) throws IOException {
assert fsn.hasWriteLock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index 6f7c5eb..89fd8a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.AclEntry;
@@ -27,7 +28,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.AclException;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
@@ -39,7 +39,7 @@ import static org.apache.hadoop.util.Time.now;
class FSDirMkdirOp {
- static HdfsFileStatus mkdirs(FSNamesystem fsn, String src,
+ static FileStatus mkdirs(FSNamesystem fsn, String src,
PermissionStatus permissions, boolean createParent) throws IOException {
FSDirectory fsd = fsn.getFSDirectory();
if(NameNode.stateChangeLog.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index 3beb3c0..bbbb724 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -19,13 +19,13 @@ package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
@@ -781,18 +781,18 @@ class FSDirRenameOp {
INodesInPath dst, boolean filesDeleted,
BlocksMapUpdateInfo collectedBlocks) throws IOException {
boolean success = (dst != null);
- HdfsFileStatus auditStat = success ? fsd.getAuditFileInfo(dst) : null;
+ FileStatus auditStat = success ? fsd.getAuditFileInfo(dst) : null;
return new RenameResult(
success, auditStat, filesDeleted, collectedBlocks);
}
static class RenameResult {
final boolean success;
- final HdfsFileStatus auditStat;
+ final FileStatus auditStat;
final boolean filesDeleted;
final BlocksMapUpdateInfo collectedBlocks;
- RenameResult(boolean success, HdfsFileStatus auditStat,
+ RenameResult(boolean success, FileStatus auditStat,
boolean filesDeleted, BlocksMapUpdateInfo collectedBlocks) {
this.success = success;
this.auditStat = auditStat;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
index 3b5f19d..ff7a3d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
@@ -17,13 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
@@ -33,7 +33,7 @@ import static org.apache.hadoop.util.Time.now;
class FSDirSymlinkOp {
- static HdfsFileStatus createSymlinkInt(
+ static FileStatus createSymlinkInt(
FSNamesystem fsn, String target, final String linkArg,
PermissionStatus dirPerms, boolean createParent, boolean logRetryCache)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
index 9546c59..a6305f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
@@ -21,11 +21,11 @@ import java.io.IOException;
import java.util.Set;
import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
@@ -348,9 +348,9 @@ final class FSDirTruncateOp {
*/
static class TruncateResult {
private final boolean result;
- private final HdfsFileStatus stat;
+ private final FileStatus stat;
- public TruncateResult(boolean result, HdfsFileStatus stat) {
+ public TruncateResult(boolean result, FileStatus stat) {
this.result = result;
this.stat = stat;
}
@@ -366,7 +366,7 @@ final class FSDirTruncateOp {
/**
* @return file information.
*/
- HdfsFileStatus getFileStatus() {
+ FileStatus getFileStatus() {
return stat;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index f676f36..e5243ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -21,13 +21,13 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
@@ -59,7 +59,7 @@ class FSDirXAttrOp {
* - xAttrs flags
* @throws IOException
*/
- static HdfsFileStatus setXAttr(
+ static FileStatus setXAttr(
FSDirectory fsd, String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
boolean logRetryCache)
throws IOException {
@@ -153,7 +153,7 @@ class FSDirXAttrOp {
* - xAttr to remove
* @throws IOException
*/
- static HdfsFileStatus removeXAttr(
+ static FileStatus removeXAttr(
FSDirectory fsd, String src, XAttr xAttr, boolean logRetryCache)
throws IOException {
FSDirXAttrOp.checkXAttrsConfigFlag(fsd);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 9ac6149..07dc5c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
@@ -38,6 +39,7 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
@@ -1796,10 +1798,45 @@ public class FSDirectory implements Closeable {
}
}
- HdfsFileStatus getAuditFileInfo(INodesInPath iip)
+ FileStatus getAuditFileInfo(INodesInPath iip)
throws IOException {
- return (namesystem.isAuditEnabled() && namesystem.isExternalInvocation())
- ? FSDirStatAndListingOp.getFileInfo(this, iip, false) : null;
+ if (!namesystem.isAuditEnabled() || !namesystem.isExternalInvocation()) {
+ return null;
+ }
+
+ final INode inode = iip.getLastINode();
+ if (inode == null) {
+ return null;
+ }
+ final int snapshot = iip.getPathSnapshotId();
+
+ Path symlink = null;
+ long size = 0; // length is zero for directories
+ short replication = 0;
+ long blocksize = 0;
+
+ if (inode.isFile()) {
+ final INodeFile fileNode = inode.asFile();
+ size = fileNode.computeFileSize(snapshot);
+ replication = fileNode.getFileReplication(snapshot);
+ blocksize = fileNode.getPreferredBlockSize();
+ } else if (inode.isSymlink()) {
+ symlink = new Path(
+ DFSUtilClient.bytes2String(inode.asSymlink().getSymlink()));
+ }
+
+ return new FileStatus(
+ size,
+ inode.isDirectory(),
+ replication,
+ blocksize,
+ inode.getModificationTime(snapshot),
+ inode.getAccessTime(snapshot),
+ inode.getFsPermission(snapshot),
+ inode.getUserName(snapshot),
+ inode.getGroupName(snapshot),
+ symlink,
+ new Path(iip.getPath()));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b90e52f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index afcc717..eb423cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -351,26 +351,34 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
private void logAuditEvent(boolean succeeded, String cmd, String src,
- String dst, HdfsFileStatus stat) throws IOException {
+ String dst, FileStatus stat) throws IOException {
if (isAuditEnabled() && isExternalInvocation()) {
logAuditEvent(succeeded, Server.getRemoteUser(), Server.getRemoteIp(),
cmd, src, dst, stat);
}
}
- private void logAuditEvent(boolean succeeded,
- UserGroupInformation ugi, InetAddress addr, String cmd, String src,
- String dst, HdfsFileStatus stat) {
+ private void logAuditEvent(boolean succeeded, String cmd, String src,
+ HdfsFileStatus stat) throws IOException {
+ if (!isAuditEnabled() || !isExternalInvocation()) {
+ return;
+ }
FileStatus status = null;
if (stat != null) {
Path symlink = stat.isSymlink() ? new Path(stat.getSymlink()) : null;
- Path path = dst != null ? new Path(dst) : new Path(src);
+ Path path = new Path(src);
status = new FileStatus(stat.getLen(), stat.isDir(),
stat.getReplication(), stat.getBlockSize(),
stat.getModificationTime(),
stat.getAccessTime(), stat.getPermission(), stat.getOwner(),
stat.getGroup(), symlink, path);
}
+ logAuditEvent(succeeded, cmd, src, null, status);
+ }
+
+ private void logAuditEvent(boolean succeeded,
+ UserGroupInformation ugi, InetAddress addr, String cmd, String src,
+ String dst, FileStatus status) {
final String ugiStr = ugi.toString();
for (AuditLogger logger : auditLoggers) {
if (logger instanceof HdfsAuditLogger) {
@@ -1725,7 +1733,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
void setPermission(String src, FsPermission permission) throws IOException {
final String operationName = "setPermission";
- HdfsFileStatus auditStat;
+ FileStatus auditStat;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -1749,7 +1757,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void setOwner(String src, String username, String group)
throws IOException {
final String operationName = "setOwner";
- HdfsFileStatus auditStat;
+ FileStatus auditStat;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -1886,7 +1894,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void concat(String target, String [] srcs, boolean logRetryCache)
throws IOException {
final String operationName = "concat";
- HdfsFileStatus stat = null;
+ FileStatus stat = null;
boolean success = false;
writeLock();
try {
@@ -1914,7 +1922,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
void setTimes(String src, long mtime, long atime) throws IOException {
final String operationName = "setTimes";
- HdfsFileStatus auditStat;
+ FileStatus auditStat;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -1989,7 +1997,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (!FileSystem.areSymlinksEnabled()) {
throw new UnsupportedOperationException("Symlinks not supported");
}
- HdfsFileStatus auditStat = null;
+ FileStatus auditStat = null;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2050,7 +2058,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
void setStoragePolicy(String src, String policyName) throws IOException {
final String operationName = "setStoragePolicy";
- HdfsFileStatus auditStat;
+ FileStatus auditStat;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -2075,7 +2083,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
void unsetStoragePolicy(String src) throws IOException {
final String operationName = "unsetStoragePolicy";
- HdfsFileStatus auditStat;
+ FileStatus auditStat;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -2195,7 +2203,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
logAuditEvent(false, "create", src);
throw e;
}
- logAuditEvent(true, "create", src, null, status);
+ logAuditEvent(true, "create", src, status);
return status;
}
@@ -2954,7 +2962,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean mkdirs(String src, PermissionStatus permissions,
boolean createParent) throws IOException {
final String operationName = "mkdirs";
- HdfsFileStatus auditStat = null;
+ FileStatus auditStat = null;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -6589,7 +6597,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void modifyAclEntries(final String src, List<AclEntry> aclSpec)
throws IOException {
final String operationName = "modifyAclEntries";
- HdfsFileStatus auditStat = null;
+ FileStatus auditStat = null;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -6610,7 +6618,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throws IOException {
final String operationName = "removeAclEntries";
checkOperation(OperationCategory.WRITE);
- HdfsFileStatus auditStat = null;
+ FileStatus auditStat = null;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -6628,7 +6636,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void removeDefaultAcl(final String src) throws IOException {
final String operationName = "removeDefaultAcl";
- HdfsFileStatus auditStat = null;
+ FileStatus auditStat = null;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -6647,7 +6655,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void removeAcl(final String src) throws IOException {
final String operationName = "removeAcl";
- HdfsFileStatus auditStat = null;
+ FileStatus auditStat = null;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -6666,7 +6674,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void setAcl(final String src, List<AclEntry> aclSpec) throws IOException {
final String operationName = "setAcl";
- HdfsFileStatus auditStat = null;
+ FileStatus auditStat = null;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -6722,7 +6730,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkSuperuserPrivilege();
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
- final HdfsFileStatus resultingStat;
+ final FileStatus resultingStat;
writeLock();
try {
checkSuperuserPrivilege();
@@ -6753,7 +6761,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
EncryptionZone getEZForPath(final String srcArg)
throws AccessControlException, UnresolvedLinkException, IOException {
final String operationName = "getEZForPath";
- HdfsFileStatus resultingStat = null;
+ FileStatus resultingStat = null;
boolean success = false;
EncryptionZone encryptionZone;
final FSPermissionChecker pc = getPermissionChecker();
@@ -6761,7 +6769,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
readLock();
try {
checkOperation(OperationCategory.READ);
- Entry<EncryptionZone, HdfsFileStatus> ezForPath = FSDirEncryptionZoneOp
+ Entry<EncryptionZone, FileStatus> ezForPath = FSDirEncryptionZoneOp
.getEZForPath(dir, srcArg, pc);
success = true;
resultingStat = ezForPath.getValue();
@@ -6810,7 +6818,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
UnresolvedLinkException, SafeModeException, AccessControlException {
final String operationName = "setErasureCodingPolicy";
checkOperation(OperationCategory.WRITE);
- HdfsFileStatus resultingStat = null;
+ FileStatus resultingStat = null;
final FSPermissionChecker pc = getPermissionChecker();
boolean success = false;
writeLock();
@@ -6868,7 +6876,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
UnresolvedLinkException, SafeModeException, AccessControlException {
final String operationName = "unsetErasureCodingPolicy";
checkOperation(OperationCategory.WRITE);
- HdfsFileStatus resultingStat = null;
+ FileStatus resultingStat = null;
final FSPermissionChecker pc = getPermissionChecker();
boolean success = false;
writeLock();
@@ -6926,7 +6934,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean logRetryCache)
throws IOException {
final String operationName = "setXAttr";
- HdfsFileStatus auditStat = null;
+ FileStatus auditStat = null;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -6982,7 +6990,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void removeXAttr(String src, XAttr xAttr, boolean logRetryCache)
throws IOException {
final String operationName = "removeXAttr";
- HdfsFileStatus auditStat = null;
+ FileStatus auditStat = null;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] hadoop git commit: HADOOP-14412.
HostsFileReader#getHostDetails is very expensive on large clusters.
Contributed by Jason Lowe.
Posted by wa...@apache.org.
HADOOP-14412. HostsFileReader#getHostDetails is very expensive on large clusters. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d87a63a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d87a63a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d87a63a9
Branch: refs/heads/YARN-5734
Commit: d87a63a9019d74a1c338c724e050952843a153e5
Parents: ec21ce4
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed May 17 08:26:29 2017 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed May 17 08:27:45 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/util/HostsFileReader.java | 263 ++++++++++---------
.../apache/hadoop/util/TestHostsFileReader.java | 19 +-
.../resourcemanager/NodesListManager.java | 30 +--
3 files changed, 161 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d87a63a9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
index 2ef1ead..2913b87 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
@@ -20,13 +20,12 @@ package org.apache.hadoop.util;
import java.io.*;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
import java.util.Set;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -48,39 +47,26 @@ import org.xml.sax.SAXException;
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Unstable
public class HostsFileReader {
- private Set<String> includes;
- // exclude host list with optional timeout.
- // If the value is null, it indicates default timeout.
- private Map<String, Integer> excludes;
- private String includesFile;
- private String excludesFile;
- private WriteLock writeLock;
- private ReadLock readLock;
-
private static final Log LOG = LogFactory.getLog(HostsFileReader.class);
- public HostsFileReader(String inFile,
+ private final AtomicReference<HostDetails> current;
+
+ public HostsFileReader(String inFile,
String exFile) throws IOException {
- includes = new HashSet<String>();
- excludes = new HashMap<String, Integer>();
- includesFile = inFile;
- excludesFile = exFile;
- ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
- this.writeLock = rwLock.writeLock();
- this.readLock = rwLock.readLock();
- refresh();
+ HostDetails hostDetails = new HostDetails(
+ inFile, Collections.emptySet(),
+ exFile, Collections.emptyMap());
+ current = new AtomicReference<>(hostDetails);
+ refresh(inFile, exFile);
}
@Private
public HostsFileReader(String includesFile, InputStream inFileInputStream,
String excludesFile, InputStream exFileInputStream) throws IOException {
- includes = new HashSet<String>();
- excludes = new HashMap<String, Integer>();
- this.includesFile = includesFile;
- this.excludesFile = excludesFile;
- ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
- this.writeLock = rwLock.writeLock();
- this.readLock = rwLock.readLock();
+ HostDetails hostDetails = new HostDetails(
+ includesFile, Collections.emptySet(),
+ excludesFile, Collections.emptyMap());
+ current = new AtomicReference<>(hostDetails);
refresh(inFileInputStream, exFileInputStream);
}
@@ -126,12 +112,8 @@ public class HostsFileReader {
}
public void refresh() throws IOException {
- this.writeLock.lock();
- try {
- refresh(includesFile, excludesFile);
- } finally {
- this.writeLock.unlock();
- }
+ HostDetails hostDetails = current.get();
+ refresh(hostDetails.includesFile, hostDetails.excludesFile);
}
public static void readFileToMap(String type,
@@ -201,128 +183,163 @@ public class HostsFileReader {
return (nodes.getLength() == 0)? null : nodes.item(0).getTextContent();
}
- public void refresh(String includeFiles, String excludeFiles)
+ public void refresh(String includesFile, String excludesFile)
throws IOException {
LOG.info("Refreshing hosts (include/exclude) list");
- this.writeLock.lock();
- try {
- // update instance variables
- updateFileNames(includeFiles, excludeFiles);
- Set<String> newIncludes = new HashSet<String>();
- Map<String, Integer> newExcludes = new HashMap<String, Integer>();
- boolean switchIncludes = false;
- boolean switchExcludes = false;
- if (includeFiles != null && !includeFiles.isEmpty()) {
- readFileToSet("included", includeFiles, newIncludes);
- switchIncludes = true;
- }
- if (excludeFiles != null && !excludeFiles.isEmpty()) {
- readFileToMap("excluded", excludeFiles, newExcludes);
- switchExcludes = true;
- }
-
- if (switchIncludes) {
- // switch the new hosts that are to be included
- includes = newIncludes;
- }
- if (switchExcludes) {
- // switch the excluded hosts
- excludes = newExcludes;
- }
- } finally {
- this.writeLock.unlock();
+ HostDetails oldDetails = current.get();
+ Set<String> newIncludes = oldDetails.includes;
+ Map<String, Integer> newExcludes = oldDetails.excludes;
+ if (includesFile != null && !includesFile.isEmpty()) {
+ newIncludes = new HashSet<>();
+ readFileToSet("included", includesFile, newIncludes);
+ newIncludes = Collections.unmodifiableSet(newIncludes);
+ }
+ if (excludesFile != null && !excludesFile.isEmpty()) {
+ newExcludes = new HashMap<>();
+ readFileToMap("excluded", excludesFile, newExcludes);
+ newExcludes = Collections.unmodifiableMap(newExcludes);
}
+ HostDetails newDetails = new HostDetails(includesFile, newIncludes,
+ excludesFile, newExcludes);
+ current.set(newDetails);
}
@Private
public void refresh(InputStream inFileInputStream,
InputStream exFileInputStream) throws IOException {
LOG.info("Refreshing hosts (include/exclude) list");
- this.writeLock.lock();
- try {
- Set<String> newIncludes = new HashSet<String>();
- Map<String, Integer> newExcludes = new HashMap<String, Integer>();
- boolean switchIncludes = false;
- boolean switchExcludes = false;
- if (inFileInputStream != null) {
- readFileToSetWithFileInputStream("included", includesFile,
- inFileInputStream, newIncludes);
- switchIncludes = true;
- }
- if (exFileInputStream != null) {
- readFileToMapWithFileInputStream("excluded", excludesFile,
- exFileInputStream, newExcludes);
- switchExcludes = true;
- }
- if (switchIncludes) {
- // switch the new hosts that are to be included
- includes = newIncludes;
- }
- if (switchExcludes) {
- // switch the excluded hosts
- excludes = newExcludes;
- }
- } finally {
- this.writeLock.unlock();
+ HostDetails oldDetails = current.get();
+ Set<String> newIncludes = oldDetails.includes;
+ Map<String, Integer> newExcludes = oldDetails.excludes;
+ if (inFileInputStream != null) {
+ newIncludes = new HashSet<>();
+ readFileToSetWithFileInputStream("included", oldDetails.includesFile,
+ inFileInputStream, newIncludes);
+ newIncludes = Collections.unmodifiableSet(newIncludes);
+ }
+ if (exFileInputStream != null) {
+ newExcludes = new HashMap<>();
+ readFileToMapWithFileInputStream("excluded", oldDetails.excludesFile,
+ exFileInputStream, newExcludes);
+ newExcludes = Collections.unmodifiableMap(newExcludes);
}
+ HostDetails newDetails = new HostDetails(
+ oldDetails.includesFile, newIncludes,
+ oldDetails.excludesFile, newExcludes);
+ current.set(newDetails);
}
public Set<String> getHosts() {
- this.readLock.lock();
- try {
- return includes;
- } finally {
- this.readLock.unlock();
- }
+ HostDetails hostDetails = current.get();
+ return hostDetails.getIncludedHosts();
}
public Set<String> getExcludedHosts() {
- this.readLock.lock();
- try {
- return excludes.keySet();
- } finally {
- this.readLock.unlock();
- }
+ HostDetails hostDetails = current.get();
+ return hostDetails.getExcludedHosts();
}
+ /**
+ * Retrieve an atomic view of the included and excluded hosts.
+ *
+ * @param includes set to populate with included hosts
+ * @param excludes set to populate with excluded hosts
+ * @deprecated use {@link #getHostDetails() instead}
+ */
+ @Deprecated
public void getHostDetails(Set<String> includes, Set<String> excludes) {
- this.readLock.lock();
- try {
- includes.addAll(this.includes);
- excludes.addAll(this.excludes.keySet());
- } finally {
- this.readLock.unlock();
- }
+ HostDetails hostDetails = current.get();
+ includes.addAll(hostDetails.getIncludedHosts());
+ excludes.addAll(hostDetails.getExcludedHosts());
}
+ /**
+ * Retrieve an atomic view of the included and excluded hosts.
+ *
+ * @param includeHosts set to populate with included hosts
+ * @param excludeHosts map to populate with excluded hosts
+ * @deprecated use {@link #getHostDetails() instead}
+ */
+ @Deprecated
public void getHostDetails(Set<String> includeHosts,
Map<String, Integer> excludeHosts) {
- this.readLock.lock();
- try {
- includeHosts.addAll(this.includes);
- excludeHosts.putAll(this.excludes);
- } finally {
- this.readLock.unlock();
- }
+ HostDetails hostDetails = current.get();
+ includeHosts.addAll(hostDetails.getIncludedHosts());
+ excludeHosts.putAll(hostDetails.getExcludedMap());
+ }
+
+ /**
+ * Retrieve an atomic view of the included and excluded hosts.
+ *
+ * @return the included and excluded hosts
+ */
+ public HostDetails getHostDetails() {
+ return current.get();
}
public void setIncludesFile(String includesFile) {
LOG.info("Setting the includes file to " + includesFile);
- this.includesFile = includesFile;
+ HostDetails oldDetails = current.get();
+ HostDetails newDetails = new HostDetails(includesFile, oldDetails.includes,
+ oldDetails.excludesFile, oldDetails.excludes);
+ current.set(newDetails);
}
public void setExcludesFile(String excludesFile) {
LOG.info("Setting the excludes file to " + excludesFile);
- this.excludesFile = excludesFile;
+ HostDetails oldDetails = current.get();
+ HostDetails newDetails = new HostDetails(
+ oldDetails.includesFile, oldDetails.includes,
+ excludesFile, oldDetails.excludes);
+ current.set(newDetails);
}
- public void updateFileNames(String includeFiles, String excludeFiles) {
- this.writeLock.lock();
- try {
- setIncludesFile(includeFiles);
- setExcludesFile(excludeFiles);
- } finally {
- this.writeLock.unlock();
+ public void updateFileNames(String includesFile, String excludesFile) {
+ LOG.info("Setting the includes file to " + includesFile);
+ LOG.info("Setting the excludes file to " + excludesFile);
+ HostDetails oldDetails = current.get();
+ HostDetails newDetails = new HostDetails(includesFile, oldDetails.includes,
+ excludesFile, oldDetails.excludes);
+ current.set(newDetails);
+ }
+
+ /**
+ * An atomic view of the included and excluded hosts
+ */
+ public static class HostDetails {
+ private final String includesFile;
+ private final Set<String> includes;
+ private final String excludesFile;
+ // exclude host list with optional timeout.
+ // If the value is null, it indicates default timeout.
+ private final Map<String, Integer> excludes;
+
+ HostDetails(String includesFile, Set<String> includes,
+ String excludesFile, Map<String, Integer> excludes) {
+ this.includesFile = includesFile;
+ this.includes = includes;
+ this.excludesFile = excludesFile;
+ this.excludes = excludes;
+ }
+
+ public String getIncludesFile() {
+ return includesFile;
+ }
+
+ public Set<String> getIncludedHosts() {
+ return includes;
+ }
+
+ public String getExcludesFile() {
+ return excludesFile;
+ }
+
+ public Set<String> getExcludedHosts() {
+ return excludes.keySet();
+ }
+
+ public Map<String, Integer> getExcludedMap() {
+ return excludes;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d87a63a9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
index 5766591..2462114 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
@@ -20,12 +20,10 @@ package org.apache.hadoop.util;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Set;
import java.util.Map;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.HostsFileReader.HostDetails;
import org.junit.*;
import static org.junit.Assert.*;
@@ -121,11 +119,11 @@ public class TestHostsFileReader {
assertTrue(hfp.getExcludedHosts().contains("node1"));
assertTrue(hfp.getHosts().contains("node2"));
- Set<String> hostsList = new HashSet<String>();
- Set<String> excludeList = new HashSet<String>();
- hfp.getHostDetails(hostsList, excludeList);
- assertTrue(excludeList.contains("node1"));
- assertTrue(hostsList.contains("node2"));
+ HostDetails hostDetails = hfp.getHostDetails();
+ assertTrue(hostDetails.getExcludedHosts().contains("node1"));
+ assertTrue(hostDetails.getIncludedHosts().contains("node2"));
+ assertEquals(newIncludesFile, hostDetails.getIncludesFile());
+ assertEquals(newExcludesFile, hostDetails.getExcludesFile());
}
/*
@@ -328,9 +326,8 @@ public class TestHostsFileReader {
assertEquals(4, includesLen);
assertEquals(9, excludesLen);
- Set<String> includes = new HashSet<String>();
- Map<String, Integer> excludes = new HashMap<String, Integer>();
- hfp.getHostDetails(includes, excludes);
+ HostDetails hostDetails = hfp.getHostDetails();
+ Map<String, Integer> excludes = hostDetails.getExcludedMap();
assertTrue(excludes.containsKey("host1"));
assertTrue(excludes.containsKey("host2"));
assertTrue(excludes.containsKey("host3"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d87a63a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index 7d69f93..edd173b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -40,6 +39,7 @@ import org.apache.hadoop.net.Node;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.HostsFileReader;
+import org.apache.hadoop.util.HostsFileReader.HostDetails;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -192,14 +192,11 @@ public class NodesListManager extends CompositeService implements
conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH));
- Set<String> hostsList = new HashSet<String>();
- Set<String> excludeList = new HashSet<String>();
- hostsReader.getHostDetails(hostsList, excludeList);
-
- for (String include : hostsList) {
+ HostDetails hostDetails = hostsReader.getHostDetails();
+ for (String include : hostDetails.getIncludedHosts()) {
LOG.debug("include: " + include);
}
- for (String exclude : excludeList) {
+ for (String exclude : hostDetails.getExcludedHosts()) {
LOG.debug("exclude: " + exclude);
}
}
@@ -262,9 +259,9 @@ public class NodesListManager extends CompositeService implements
// Nodes need to be decommissioned (graceful or forceful);
List<RMNode> nodesToDecom = new ArrayList<RMNode>();
- Set<String> includes = new HashSet<String>();
- Map<String, Integer> excludes = new HashMap<String, Integer>();
- hostsReader.getHostDetails(includes, excludes);
+ HostDetails hostDetails = hostsReader.getHostDetails();
+ Set<String> includes = hostDetails.getIncludedHosts();
+ Map<String, Integer> excludes = hostDetails.getExcludedMap();
for (RMNode n : this.rmContext.getRMNodes().values()) {
NodeState s = n.getState();
@@ -453,10 +450,9 @@ public class NodesListManager extends CompositeService implements
}
public boolean isValidNode(String hostName) {
- Set<String> hostsList = new HashSet<String>();
- Set<String> excludeList = new HashSet<String>();
- hostsReader.getHostDetails(hostsList, excludeList);
- return isValidNode(hostName, hostsList, excludeList);
+ HostDetails hostDetails = hostsReader.getHostDetails();
+ return isValidNode(hostName, hostDetails.getIncludedHosts(),
+ hostDetails.getExcludedHosts());
}
private boolean isValidNode(
@@ -563,9 +559,9 @@ public class NodesListManager extends CompositeService implements
public boolean isUntrackedNode(String hostName) {
String ip = resolver.resolve(hostName);
- Set<String> hostsList = new HashSet<String>();
- Set<String> excludeList = new HashSet<String>();
- hostsReader.getHostDetails(hostsList, excludeList);
+ HostDetails hostDetails = hostsReader.getHostDetails();
+ Set<String> hostsList = hostDetails.getIncludedHosts();
+ Set<String> excludeList = hostDetails.getExcludedHosts();
return !hostsList.isEmpty() && !hostsList.contains(hostName)
&& !hostsList.contains(ip) && !excludeList.contains(hostName)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] hadoop git commit: Sanitize arguments before launching Docker
containers.
Posted by wa...@apache.org.
Sanitize arguments before launching Docker containers.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b46cd315
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b46cd315
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b46cd315
Branch: refs/heads/YARN-5734
Commit: b46cd315f7c81cfd89be40f1edacaf9a11282e15
Parents: e120ee8
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu May 18 10:37:16 2017 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu May 18 10:37:16 2017 +0530
----------------------------------------------------------------------
.../impl/container-executor.c | 173 ++++++++++++++++++-
.../impl/container-executor.h | 6 +
.../test/test-container-executor.c | 44 +++++
3 files changed, 222 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46cd315/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index f11272e..3a87646 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -39,6 +39,7 @@
#include <sys/stat.h>
#include <sys/mount.h>
#include <sys/wait.h>
+#include <getopt.h>
#include "config.h"
@@ -1141,7 +1142,170 @@ int initialize_app(const char *user, const char *app_id,
return -1;
}
+static char* escape_single_quote(const char *str) {
+ int p = 0;
+ int i = 0;
+ char replacement[] = "'\"'\"'";
+ size_t replacement_length = strlen(replacement);
+ size_t ret_size = strlen(str) * replacement_length + 1;
+ char *ret = (char *) calloc(ret_size, sizeof(char));
+ if(ret == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ while(str[p] != '\0') {
+ if(str[p] == '\'') {
+ strncat(ret, replacement, ret_size - strlen(ret));
+ i += replacement_length;
+ }
+ else {
+ ret[i] = str[p];
+ ret[i + 1] = '\0';
+ i++;
+ }
+ p++;
+ }
+ return ret;
+}
+
+static void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
+ char *tmp = escape_single_quote(arg);
+ strcat(*str, param);
+ strcat(*str, "'");
+ if(strlen(*str) + strlen(tmp) > *size) {
+ *str = (char *) realloc(*str, strlen(*str) + strlen(tmp) + 1024);
+ if(*str == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ *size = strlen(*str) + strlen(tmp) + 1024;
+ }
+ strcat(*str, tmp);
+ strcat(*str, "' ");
+ free(tmp);
+}
+
+char** tokenize_docker_command(const char *input, int *split_counter) {
+ char *line = (char *)calloc(strlen(input) + 1, sizeof(char));
+ char **linesplit = (char **) malloc(sizeof(char *));
+ char *p = NULL;
+ int c = 0;
+ *split_counter = 0;
+ strncpy(line, input, strlen(input));
+
+ p = strtok(line, " ");
+ while(p != NULL) {
+ linesplit[*split_counter] = p;
+ (*split_counter)++;
+ linesplit = realloc(linesplit, (sizeof(char *) * (*split_counter + 1)));
+ if(linesplit == NULL) {
+ fprintf(ERRORFILE, "Cannot allocate memory to parse docker command %s",
+ strerror(errno));
+ fflush(ERRORFILE);
+ exit(OUT_OF_MEMORY);
+ }
+ p = strtok(NULL, " ");
+ }
+ linesplit[*split_counter] = NULL;
+ return linesplit;
+}
+
+char* sanitize_docker_command(const char *line) {
+ static struct option long_options[] = {
+ {"name", required_argument, 0, 'n' },
+ {"user", required_argument, 0, 'u' },
+ {"rm", no_argument, 0, 'r' },
+ {"workdir", required_argument, 0, 'w' },
+ {"net", required_argument, 0, 'e' },
+ {"cgroup-parent", required_argument, 0, 'g' },
+ {"privileged", no_argument, 0, 'p' },
+ {"cap-add", required_argument, 0, 'a' },
+ {"cap-drop", required_argument, 0, 'o' },
+ {"device", required_argument, 0, 'i' },
+ {"detach", required_argument, 0, 't' },
+ {0, 0, 0, 0}
+ };
+
+ int c = 0;
+ int option_index = 0;
+ char *output = NULL;
+ size_t output_size = 0;
+ char **linesplit;
+ int split_counter = 0;
+ int len = strlen(line);
+
+ linesplit = tokenize_docker_command(line, &split_counter);
+
+ output_size = len * 2;
+ output = (char *) calloc(output_size, sizeof(char));
+ if(output == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ strcat(output, linesplit[0]);
+ strcat(output, " ");
+ optind = 1;
+ while((c=getopt_long(split_counter, linesplit, "dv:", long_options, &option_index)) != -1) {
+ switch(c) {
+ case 'n':
+ quote_and_append_arg(&output, &output_size, "--name=", optarg);
+ break;
+ case 'w':
+ quote_and_append_arg(&output, &output_size, "--workdir=", optarg);
+ break;
+ case 'u':
+ quote_and_append_arg(&output, &output_size, "--user=", optarg);
+ break;
+ case 'e':
+ quote_and_append_arg(&output, &output_size, "--net=", optarg);
+ break;
+ case 'v':
+ quote_and_append_arg(&output, &output_size, "-v ", optarg);
+ break;
+ case 'a':
+ quote_and_append_arg(&output, &output_size, "--cap-add=", optarg);
+ break;
+ case 'o':
+ quote_and_append_arg(&output, &output_size, "--cap-drop=", optarg);
+ break;
+ case 'd':
+ strcat(output, "-d ");
+ break;
+ case 'r':
+ strcat(output, "--rm ");
+ break;
+ case 'g':
+ quote_and_append_arg(&output, &output_size, "--cgroup-parent=", optarg);
+ break;
+ case 'p':
+ strcat(output, "--privileged ");
+ break;
+ case 'i':
+ quote_and_append_arg(&output, &output_size, "--device=", optarg);
+ break;
+ case 't':
+ quote_and_append_arg(&output, &output_size, "--detach=", optarg);
+ break;
+ default:
+ fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind);
+ fflush(LOGFILE);
+ return NULL;
+ break;
+ }
+ }
+
+ if(optind < split_counter) {
+ quote_and_append_arg(&output, &output_size, "", linesplit[optind++]);
+ strcat(output, "'");
+ while(optind < split_counter) {
+ strcat(output, linesplit[optind++]);
+ strcat(output, " ");
+ }
+ strcat(output, "'");
+ }
+
+ return output;
+}
+
char* parse_docker_command_file(const char* command_file) {
+
size_t len = 0;
char *line = NULL;
ssize_t read;
@@ -1160,7 +1324,14 @@ char* parse_docker_command_file(const char* command_file) {
}
fclose(stream);
- return line;
+ char* ret = sanitize_docker_command(line);
+ if(ret == NULL) {
+ exit(ERROR_SANITIZING_DOCKER_COMMAND);
+ }
+ fprintf(LOGFILE, "Using command %s\n", ret);
+ fflush(LOGFILE);
+
+ return ret;
}
int run_docker(const char *command_file) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46cd315/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 826ff2c..e40bd90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -74,6 +74,7 @@ enum errorcodes {
COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36,
COULD_NOT_CREATE_TMP_DIRECTORIES = 37,
ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
+ ERROR_SANITIZING_DOCKER_COMMAND = 39
};
enum operations {
@@ -303,3 +304,8 @@ int is_docker_support_enabled();
* Run a docker command passing the command file as an argument
*/
int run_docker(const char *command_file);
+
+/**
+ * Sanitize docker commands. Returns NULL if there was any failure.
+*/
+char* sanitize_docker_command(const char *line);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46cd315/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 0968b35..ff76d4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -1085,6 +1085,47 @@ void test_trim_function() {
free(trimmed);
}
+void test_sanitize_docker_command() {
+
+/*
+ char *input[] = {
+ "run ''''''''"
+ };
+*/
+ char *input[] = {
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run ''''''''"
+ };
+/*
+ char *expected_output[] = {
+ "run ''''''''"
+ };
+*/
+ char *expected_output[] = {
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '|| touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '|| touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ''",
+ };
+
+ int input_size = sizeof(input) / sizeof(char *);
+ int i = 0;
+ for(i = 0; i < input_size; i++) {
+ char *command = (char *) calloc(strlen(input[i]), sizeof(char));
+ strncpy(command, input[i], strlen(input[i]));
+ char *op = sanitize_docker_command(command);
+ if(strncmp(expected_output[i], op, strlen(expected_output[i])) != 0) {
+ printf("FAIL: expected output %s does not match actual output '%s'\n", expected_output[i], op);
+ exit(1);
+ }
+ free(command);
+ }
+}
+
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -1176,6 +1217,9 @@ int main(int argc, char **argv) {
printf("\nTesting is_feature_enabled()\n");
test_is_feature_enabled();
+ printf("\nTesting sanitize docker commands()\n");
+ test_sanitize_docker_command();
+
test_check_user(0);
#ifdef __APPLE__
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] hadoop git commit: YARN-6540. Resource Manager is spelled
"Resource Manger" in ResourceManagerRestart.md and ResourceManagerHA.md.
(Grant Sohn via Yufei Gu)
Posted by wa...@apache.org.
YARN-6540. Resource Manager is spelled "Resource Manger" in ResourceManagerRestart.md and ResourceManagerHA.md. (Grant Sohn via Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e82da2b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e82da2b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e82da2b4
Branch: refs/heads/YARN-5734
Commit: e82da2b4f4de52de7e4570e011a8a2e1838281af
Parents: 6672810
Author: Yufei Gu <yu...@cloudera.com>
Authored: Fri May 19 14:16:00 2017 -0700
Committer: Yufei Gu <yu...@cloudera.com>
Committed: Fri May 19 14:16:00 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md | 4 ++--
.../hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e82da2b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md
index 984959f..7b2dd35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md
@@ -45,14 +45,14 @@ When there are multiple RMs, the configuration (yarn-site.xml) used by clients a
### Recovering prevous active-RM's state
-With the [ResourceManger Restart](./ResourceManagerRestart.html) enabled, the RM being promoted to an active state loads the RM internal state and continues to operate from where the previous active left off as much as possible depending on the RM restart feature. A new attempt is spawned for each managed application previously submitted to the RM. Applications can checkpoint periodically to avoid losing any work. The state-store must be visible from the both of Active/Standby RMs. Currently, there are two RMStateStore implementations for persistence - FileSystemRMStateStore and ZKRMStateStore. The `ZKRMStateStore` implicitly allows write access to a single RM at any point in time, and hence is the recommended store to use in an HA cluster. When using the ZKRMStateStore, there is no need for a separate fencing mechanism to address a potential split-brain situation where multiple RMs can potentially assume the Active role. When using the ZKRMStateStore, it is advisable to NOT set the
"`zookeeper.DigestAuthenticationProvider.superDigest`" property on the Zookeeper cluster to ensure that the zookeeper admin does not have access to YARN application/user credential information.
+With the [ResourceManager Restart](./ResourceManagerRestart.html) enabled, the RM being promoted to an active state loads the RM internal state and continues to operate from where the previous active left off as much as possible depending on the RM restart feature. A new attempt is spawned for each managed application previously submitted to the RM. Applications can checkpoint periodically to avoid losing any work. The state-store must be visible from the both of Active/Standby RMs. Currently, there are two RMStateStore implementations for persistence - FileSystemRMStateStore and ZKRMStateStore. The `ZKRMStateStore` implicitly allows write access to a single RM at any point in time, and hence is the recommended store to use in an HA cluster. When using the ZKRMStateStore, there is no need for a separate fencing mechanism to address a potential split-brain situation where multiple RMs can potentially assume the Active role. When using the ZKRMStateStore, it is advisable to NOT set th
e "`zookeeper.DigestAuthenticationProvider.superDigest`" property on the Zookeeper cluster to ensure that the zookeeper admin does not have access to YARN application/user credential information.
Deployment
----------
### Configurations
-Most of the failover functionality is tunable using various configuration properties. Following is a list of required/important ones. yarn-default.xml carries a full-list of knobs. See [yarn-default.xml](../hadoop-yarn-common/yarn-default.xml) for more information including default values. See the document for [ResourceManger Restart](./ResourceManagerRestart.html) also for instructions on setting up the state-store.
+Most of the failover functionality is tunable using various configuration properties. Following is a list of required/important ones. yarn-default.xml carries a full-list of knobs. See [yarn-default.xml](../hadoop-yarn-common/yarn-default.xml) for more information including default values. See the document for [ResourceManager Restart](./ResourceManagerRestart.html) also for instructions on setting up the state-store.
| Configuration Properties | Description |
|:---- |:---- |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e82da2b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md
index e18945d..86c1a6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md
@@ -12,7 +12,7 @@
limitations under the License. See accompanying LICENSE file.
-->
-ResourceManger Restart
+ResourceManager Restart
======================
<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] hadoop git commit: HADOOP-14401.
maven-project-info-reports-plugin can be removed. Contributed by Andras
Bokor.
Posted by wa...@apache.org.
HADOOP-14401. maven-project-info-reports-plugin can be removed. Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d5c8ed8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d5c8ed8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d5c8ed8
Branch: refs/heads/YARN-5734
Commit: 0d5c8ed8e0176909ee4d68b2969a882c21c0df55
Parents: 83dd14a
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu May 11 16:37:32 2017 -0500
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu May 11 16:37:32 2017 -0500
----------------------------------------------------------------------
hadoop-common-project/hadoop-auth/pom.xml | 15 ---------------
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 15 ---------------
hadoop-project/pom.xml | 6 ------
hadoop-tools/hadoop-aliyun/pom.xml | 8 --------
hadoop-tools/hadoop-aws/pom.xml | 8 --------
hadoop-tools/hadoop-azure-datalake/pom.xml | 10 ----------
hadoop-tools/hadoop-azure/pom.xml | 10 ----------
hadoop-tools/hadoop-kafka/pom.xml | 8 --------
hadoop-tools/hadoop-openstack/pom.xml | 9 ---------
9 files changed, 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5c8ed8/hadoop-common-project/hadoop-auth/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index 8900f58..b4beb82 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -252,21 +252,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <executions>
- <execution>
- <configuration>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- <phase>package</phase>
- <goals>
- <goal>dependencies</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5c8ed8/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index bce8bc2..d0a01cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -299,21 +299,6 @@
</executions>
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <executions>
- <execution>
- <configuration>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- <goals>
- <goal>dependencies</goal>
- </goals>
- <phase>package</phase>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5c8ed8/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 4097a0a..1d97cfc 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -123,7 +123,6 @@
<maven-jar-plugin.version>2.5</maven-jar-plugin.version>
<maven-war-plugin.version>2.4</maven-war-plugin.version>
<maven-source-plugin.version>2.3</maven-source-plugin.version>
- <maven-project-info-reports-plugin.version>2.9</maven-project-info-reports-plugin.version>
<maven-pdf-plugin.version>1.2</maven-pdf-plugin.version>
<maven-remote-resources-plugin.version>1.5</maven-remote-resources-plugin.version>
<build-helper-maven-plugin.version>1.9</build-helper-maven-plugin.version>
@@ -1388,11 +1387,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <version>${maven-project-info-reports-plugin.version}</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>${maven-resources-plugin.version}</version>
</plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5c8ed8/hadoop-tools/hadoop-aliyun/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
index 4282ea3..74461fe 100644
--- a/hadoop-tools/hadoop-aliyun/pom.xml
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -70,14 +70,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <configuration>
- <dependencyDetailsEnabled>false</dependencyDetailsEnabled>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5c8ed8/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 1084881..e00ae10 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -286,14 +286,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <configuration>
- <dependencyDetailsEnabled>false</dependencyDetailsEnabled>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5c8ed8/hadoop-tools/hadoop-azure-datalake/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index 25197cb..6e1306b 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -38,16 +38,6 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
-
- <configuration>
- <dependencyDetailsEnabled>false</dependencyDetailsEnabled>
- <dependencyLocationsEnabled>false
- </dependencyLocationsEnabled>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5c8ed8/hadoop-tools/hadoop-azure/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index de31a49..0f59c46 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -50,16 +50,6 @@
</configuration>
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
-
- <configuration>
- <dependencyDetailsEnabled>false</dependencyDetailsEnabled>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
-
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5c8ed8/hadoop-tools/hadoop-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-kafka/pom.xml b/hadoop-tools/hadoop-kafka/pom.xml
index 13e0ac0..b9273fe 100644
--- a/hadoop-tools/hadoop-kafka/pom.xml
+++ b/hadoop-tools/hadoop-kafka/pom.xml
@@ -49,14 +49,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <configuration>
- <dependencyDetailsEnabled>false</dependencyDetailsEnabled>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5c8ed8/hadoop-tools/hadoop-openstack/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml
index 0e0defb..f80fcdf 100644
--- a/hadoop-tools/hadoop-openstack/pom.xml
+++ b/hadoop-tools/hadoop-openstack/pom.xml
@@ -78,15 +78,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
-
- <configuration>
- <dependencyDetailsEnabled>false</dependencyDetailsEnabled>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] hadoop git commit: Validate docker image name before
launching container.
Posted by wa...@apache.org.
Validate docker image name before launching container.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e120ee86
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e120ee86
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e120ee86
Branch: refs/heads/YARN-5734
Commit: e120ee865aa78eaff4a80122c5f3207a97b4924d
Parents: b23fcc8
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu May 18 10:29:34 2017 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu May 18 10:29:34 2017 +0530
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 24 +++++++++++++---
.../runtime/TestDockerContainerRuntime.java | 29 ++++++++++++++++++++
2 files changed, 49 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e120ee86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index ed81331..8db03bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -57,6 +57,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -128,6 +129,12 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private static final Log LOG = LogFactory.getLog(
DockerLinuxContainerRuntime.class);
+ // This validates that the image is a proper docker image
+ public static final String DOCKER_IMAGE_PATTERN =
+ "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$";
+ private static final Pattern dockerImagePattern =
+ Pattern.compile(DOCKER_IMAGE_PATTERN);
+
@InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_IMAGE =
"YARN_CONTAINER_RUNTIME_DOCKER_IMAGE";
@@ -413,10 +420,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
validateContainerNetworkType(network);
- if (imageName == null) {
- throw new ContainerExecutionException(ENV_DOCKER_CONTAINER_IMAGE
- + " not set!");
- }
+ validateImageName(imageName);
String containerIdStr = container.getContainerId().toString();
String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
@@ -652,4 +656,16 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
return launchOp;
}
+
+ public static void validateImageName(String imageName)
+ throws ContainerExecutionException {
+ if (imageName == null || imageName.isEmpty()) {
+ throw new ContainerExecutionException(
+ ENV_DOCKER_CONTAINER_IMAGE + " not set!");
+ }
+ if (!dockerImagePattern.matcher(imageName).matches()) {
+ throw new ContainerExecutionException("Image name '" + imageName
+ + "' doesn't match docker image name pattern");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e120ee86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index ee1f25c..f611843 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -898,4 +898,33 @@ public class TestDockerContainerRuntime {
return conf;
}
+ @Test
+ public void testDockerImageNamePattern() throws Exception {
+ String[] validNames =
+ { "ubuntu", "fedora/httpd:version1.0",
+ "fedora/httpd:version1.0.test",
+ "fedora/httpd:version1.0.TEST",
+ "myregistryhost:5000/ubuntu",
+ "myregistryhost:5000/fedora/httpd:version1.0",
+ "myregistryhost:5000/fedora/httpd:version1.0.test",
+ "myregistryhost:5000/fedora/httpd:version1.0.TEST"};
+
+ String[] invalidNames = { "Ubuntu", "ubuntu || fedora", "ubuntu#",
+ "myregistryhost:50AB0/ubuntu", "myregistry#host:50AB0/ubuntu",
+ ":8080/ubuntu"
+ };
+
+ for (String name : validNames) {
+ DockerLinuxContainerRuntime.validateImageName(name);
+ }
+
+ for (String name : invalidNames) {
+ try {
+ DockerLinuxContainerRuntime.validateImageName(name);
+ Assert.fail(name + " is an invalid name and should fail the regex");
+ } catch (ContainerExecutionException ce) {
+ continue;
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] hadoop git commit: YARN-5949. Add pluggable configuration ACL
policy interface and implementation. (Jonathan Hung via wangda)
Posted by wa...@apache.org.
YARN-5949. Add pluggable configuration ACL policy interface and implementation. (Jonathan Hung via wangda)
Change-Id: Ib98e82ff753bede21fcab2e6ca9ec1e7a5a2008f
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d28ca16
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d28ca16
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d28ca16
Branch: refs/heads/YARN-5734
Commit: 8d28ca16dea9de1a3ba76b4d3f614155ee8ad172
Parents: bb7a0e8
Author: Wangda Tan <wa...@apache.org>
Authored: Mon May 22 13:38:31 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon May 22 13:38:31 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 3 +
.../src/main/resources/yarn-default.xml | 11 ++
.../ConfigurationMutationACLPolicy.java | 47 ++++++
.../ConfigurationMutationACLPolicyFactory.java | 49 ++++++
.../DefaultConfigurationMutationACLPolicy.java | 45 ++++++
.../scheduler/MutableConfScheduler.java | 19 ++-
.../scheduler/MutableConfigurationProvider.java | 8 +-
.../scheduler/capacity/CapacityScheduler.java | 6 +-
.../conf/MutableCSConfigurationProvider.java | 151 +++++++++++++++++-
...ueueAdminConfigurationMutationACLPolicy.java | 96 ++++++++++++
.../resourcemanager/webapp/RMWebServices.java | 132 +---------------
.../TestConfigurationMutationACLPolicies.java | 154 +++++++++++++++++++
.../TestMutableCSConfigurationProvider.java | 40 +++--
13 files changed, 610 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index fa9e79d..6e0cd32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -613,6 +613,9 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_CONFIGURATION_STORE =
MEMORY_CONFIGURATION_STORE;
+ public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
+ YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
+
public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX
+ "authorization-provider";
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 02284e6..5f2769e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3120,4 +3120,15 @@
<value>memory</value>
</property>
+ <property>
+ <description>
+ The class to use for configuration mutation ACL policy if using a mutable
+ configuration provider. Controls whether a mutation request is allowed.
+ The DefaultConfigurationMutationACLPolicy checks if the requestor is a
+ YARN admin.
+ </description>
+ <name>yarn.scheduler.configuration.mutation.acl-policy.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..724487b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+/**
+ * Interface for determining whether configuration mutations are allowed.
+ */
+public interface ConfigurationMutationACLPolicy {
+
+ /**
+ * Initialize ACL policy with configuration and RMContext.
+ * @param conf Configuration to initialize with.
+ * @param rmContext rmContext
+ */
+ void init(Configuration conf, RMContext rmContext);
+
+ /**
+ * Check if mutation is allowed.
+ * @param user User issuing the request
+ * @param confUpdate configurations to be updated
+ * @return whether provided mutation is allowed or not
+ */
+ boolean isMutationAllowed(UserGroupInformation user, QueueConfigsUpdateInfo
+ confUpdate);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
new file mode 100644
index 0000000..2898785
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Factory class for creating instances of
+ * {@link ConfigurationMutationACLPolicy}.
+ */
+public final class ConfigurationMutationACLPolicyFactory {
+
+ private static final Log LOG = LogFactory.getLog(
+ ConfigurationMutationACLPolicyFactory.class);
+
+ private ConfigurationMutationACLPolicyFactory() {
+ // Unused.
+ }
+
+ public static ConfigurationMutationACLPolicy getPolicy(Configuration conf) {
+ Class<? extends ConfigurationMutationACLPolicy> policyClass =
+ conf.getClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ DefaultConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ LOG.info("Using ConfigurationMutationACLPolicy implementation - " +
+ policyClass);
+ return ReflectionUtils.newInstance(policyClass, conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..680c3b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+/**
+ * Default configuration mutation ACL policy. Checks if user is YARN admin.
+ */
+public class DefaultConfigurationMutationACLPolicy implements
+ ConfigurationMutationACLPolicy {
+
+ private YarnAuthorizationProvider authorizer;
+
+ @Override
+ public void init(Configuration conf, RMContext rmContext) {
+ authorizer = YarnAuthorizationProvider.getInstance(conf);
+ }
+
+ @Override
+ public boolean isMutationAllowed(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) {
+ return authorizer.isAdmin(user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 35e36e1..93a935e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -17,10 +17,11 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import java.io.IOException;
-import java.util.Map;
/**
* Interface for a scheduler that supports changing configuration at runtime.
@@ -31,10 +32,22 @@ public interface MutableConfScheduler extends ResourceScheduler {
/**
* Update the scheduler's configuration.
* @param user Caller of this update
- * @param confUpdate key-value map of the configuration update
+ * @param confUpdate configuration update
* @throws IOException if update is invalid
*/
void updateConfiguration(UserGroupInformation user,
- Map<String, String> confUpdate) throws IOException;
+ QueueConfigsUpdateInfo confUpdate) throws IOException;
+ /**
+ * Get the scheduler configuration.
+ * @return the scheduler configuration
+ */
+ Configuration getConfiguration();
+
+ /**
+ * Get queue object based on queue name.
+ * @param queueName the queue name
+ * @return the queue object
+ */
+ Queue getQueue(String queueName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 889c3bc..f04c128 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
import java.io.IOException;
-import java.util.Map;
/**
* Interface for allowing changing scheduler configurations.
@@ -32,7 +34,7 @@ public interface MutableConfigurationProvider {
* @param confUpdate Key-value pairs for configurations to be updated.
* @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(String user, Map<String, String> confUpdate)
- throws IOException;
+ void mutateConfiguration(UserGroupInformation user, QueueConfigsUpdateInfo
+ confUpdate) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 85138d5..a0e7391 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -136,6 +136,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -619,6 +620,7 @@ public class CapacityScheduler extends
preemptionManager.refreshQueues(null, this.getRootQueue());
}
+ @Override
public CSQueue getQueue(String queueName) {
if (queueName == null) {
return null;
@@ -2491,10 +2493,10 @@ public class CapacityScheduler extends
@Override
public void updateConfiguration(UserGroupInformation user,
- Map<String, String> confUpdate) throws IOException {
+ QueueConfigsUpdateInfo confUpdate) throws IOException {
if (csConfProvider instanceof MutableConfigurationProvider) {
((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
- user.getShortUserName(), confUpdate);
+ user, confUpdate);
} else {
throw new UnsupportedOperationException("Configured CS configuration " +
"provider does not support updating configuration.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index ea1b3c0..8b879b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -18,14 +18,27 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import com.google.common.base.Joiner;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
/**
@@ -38,6 +51,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
private Configuration schedConf;
private YarnConfigurationStore confStore;
+ private ConfigurationMutationACLPolicy aclMutationPolicy;
private RMContext rmContext;
private Configuration conf;
@@ -68,6 +82,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
schedConf.set(kv.getKey(), kv.getValue());
}
confStore.initialize(config, schedConf);
+ this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory
+ .getPolicy(config);
+ aclMutationPolicy.init(config, rmContext);
this.conf = config;
}
@@ -80,12 +97,17 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
@Override
- public void mutateConfiguration(String user,
- Map<String, String> confUpdate) throws IOException {
+ public void mutateConfiguration(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) throws IOException {
+ if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
+ throw new AccessControlException("User is not admin of all modified" +
+ " queues.");
+ }
Configuration oldConf = new Configuration(schedConf);
- LogMutation log = new LogMutation(confUpdate, user);
+ Map<String, String> kvUpdate = constructKeyValueConfUpdate(confUpdate);
+ LogMutation log = new LogMutation(kvUpdate, user.getShortUserName());
long id = confStore.logMutation(log);
- for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
+ for (Map.Entry<String, String> kv : kvUpdate.entrySet()) {
if (kv.getValue() == null) {
schedConf.unset(kv.getKey());
} else {
@@ -101,4 +123,125 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
confStore.confirmMutation(id, true);
}
+
+
+ private Map<String, String> constructKeyValueConfUpdate(
+ QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ CapacitySchedulerConfiguration proposedConf =
+ new CapacitySchedulerConfiguration(cs.getConfiguration(), false);
+ Map<String, String> confUpdate = new HashMap<>();
+ for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
+ removeQueue(queueToRemove, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
+ addQueue(addQueueInfo, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
+ updateQueue(updateQueueInfo, proposedConf, confUpdate);
+ }
+ return confUpdate;
+ }
+
+ private void removeQueue(
+ String queueToRemove, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (queueToRemove == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ String queueName = queueToRemove.substring(
+ queueToRemove.lastIndexOf('.') + 1);
+ CSQueue queue = cs.getQueue(queueName);
+ if (queue == null ||
+ !queue.getQueuePath().equals(queueToRemove)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+ } else if (queueToRemove.lastIndexOf('.') == -1) {
+ throw new IOException("Can't remove queue " + queueToRemove);
+ }
+ String parentQueuePath = queueToRemove.substring(0, queueToRemove
+ .lastIndexOf('.'));
+ String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
+ List<String> newSiblingQueues = new ArrayList<>();
+ for (String siblingQueue : siblingQueues) {
+ if (!siblingQueue.equals(queueName)) {
+ newSiblingQueues.add(siblingQueue);
+ }
+ }
+ proposedConf.setQueues(parentQueuePath, newSiblingQueues
+ .toArray(new String[0]));
+ String queuesConfig = CapacitySchedulerConfiguration.PREFIX
+ + parentQueuePath + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.QUEUES;
+ if (newSiblingQueues.size() == 0) {
+ confUpdate.put(queuesConfig, null);
+ } else {
+ confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
+ }
+ for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+ ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
+ .entrySet()) {
+ proposedConf.unset(confRemove.getKey());
+ confUpdate.put(confRemove.getKey(), null);
+ }
+ }
+ }
+
+ private void addQueue(
+ QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (addInfo == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ String queuePath = addInfo.getQueue();
+ String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
+ if (cs.getQueue(queueName) != null) {
+ throw new IOException("Can't add existing queue " + queuePath);
+ } else if (queuePath.lastIndexOf('.') == -1) {
+ throw new IOException("Can't add invalid queue " + queuePath);
+ }
+ String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+ String[] siblings = proposedConf.getQueues(parentQueue);
+ List<String> siblingQueues = siblings == null ? new ArrayList<>() :
+ new ArrayList<>(Arrays.<String>asList(siblings));
+ siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
+ proposedConf.setQueues(parentQueue,
+ siblingQueues.toArray(new String[0]));
+ confUpdate.put(CapacitySchedulerConfiguration.PREFIX
+ + parentQueue + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.QUEUES,
+ Joiner.on(',').join(siblingQueues));
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX
+ + queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
+
+ private void updateQueue(QueueConfigInfo updateInfo,
+ CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) {
+ if (updateInfo == null) {
+ return;
+ } else {
+ String queuePath = updateInfo.getQueue();
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX
+ + queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..1f94c1c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A configuration mutation ACL policy which checks that user has admin
+ * privileges on all queues they are changing.
+ */
+public class QueueAdminConfigurationMutationACLPolicy implements
+ ConfigurationMutationACLPolicy {
+
+ private RMContext rmContext;
+
+ @Override
+ public void init(Configuration conf, RMContext context) {
+ this.rmContext = context;
+ }
+
+ @Override
+ public boolean isMutationAllowed(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) {
+ Set<String> queues = new HashSet<>();
+ for (QueueConfigInfo addQueueInfo : confUpdate.getAddQueueInfo()) {
+ queues.add(addQueueInfo.getQueue());
+ }
+ for (String removeQueue : confUpdate.getRemoveQueueInfo()) {
+ queues.add(removeQueue);
+ }
+ for (QueueConfigInfo updateQueueInfo : confUpdate.getUpdateQueueInfo()) {
+ queues.add(updateQueueInfo.getQueue());
+ }
+ for (String queuePath : queues) {
+ String queueName = queuePath.lastIndexOf('.') != -1 ?
+ queuePath.substring(queuePath.lastIndexOf('.') + 1) : queuePath;
+ QueueInfo queueInfo = null;
+ try {
+ queueInfo = rmContext.getScheduler()
+ .getQueueInfo(queueName, false, false);
+ } catch (IOException e) {
+ // Queue is not found, do nothing.
+ }
+ String parentPath = queuePath;
+ // TODO: handle global config change.
+ while (queueInfo == null) {
+ // We are adding a queue (whose parent we are possibly also adding).
+ // Check ACL of lowest parent queue which already exists.
+ parentPath = parentPath.substring(0, parentPath.lastIndexOf('.'));
+ String parentName = parentPath.lastIndexOf('.') != -1 ?
+ parentPath.substring(parentPath.lastIndexOf('.') + 1) : parentPath;
+ try {
+ queueInfo = rmContext.getScheduler()
+ .getQueueInfo(parentName, false, false);
+ } catch (IOException e) {
+ // Queue is not found, do nothing.
+ }
+ }
+ Queue queue = ((MutableConfScheduler) rmContext.getScheduler())
+ .getQueue(queueInfo.getQueueName());
+ if (queue != null && !queue.hasAccess(QueueACL.ADMINISTER_QUEUE, user)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 78eb4de..e084cca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -57,7 +57,6 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
-import com.google.common.base.Joiner;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -143,7 +142,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
@@ -2643,10 +2641,8 @@ public class RMWebServices extends WebServices {
callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws IOException, YarnException {
- Map<String, String> confUpdate =
- constructKeyValueConfUpdate(mutationInfo);
- ((CapacityScheduler) scheduler).updateConfiguration(callerUGI,
- confUpdate);
+ ((MutableConfScheduler) scheduler).updateConfiguration(callerUGI,
+ mutationInfo);
return null;
}
});
@@ -2658,129 +2654,9 @@ public class RMWebServices extends WebServices {
"successfully applied.").build();
} else {
return Response.status(Status.BAD_REQUEST)
- .entity("Configuration change only supported by CapacityScheduler.")
+ .entity("Configuration change only supported by " +
+ "MutableConfScheduler.")
.build();
}
}
-
- private Map<String, String> constructKeyValueConfUpdate(
- QueueConfigsUpdateInfo mutationInfo) throws IOException {
- CapacitySchedulerConfiguration currentConf =
- ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
- CapacitySchedulerConfiguration proposedConf =
- new CapacitySchedulerConfiguration(currentConf, false);
- Map<String, String> confUpdate = new HashMap<>();
- for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
- removeQueue(queueToRemove, proposedConf, confUpdate);
- }
- for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
- addQueue(addQueueInfo, proposedConf, confUpdate);
- }
- for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
- updateQueue(updateQueueInfo, proposedConf, confUpdate);
- }
- return confUpdate;
- }
-
- private void removeQueue(
- String queueToRemove, CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) throws IOException {
- if (queueToRemove == null) {
- return;
- } else {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- String queueName = queueToRemove.substring(
- queueToRemove.lastIndexOf('.') + 1);
- CSQueue queue = cs.getQueue(queueName);
- if (queue == null ||
- !queue.getQueuePath().equals(queueToRemove)) {
- throw new IOException("Queue " + queueToRemove + " not found");
- } else if (queueToRemove.lastIndexOf('.') == -1) {
- throw new IOException("Can't remove queue " + queueToRemove);
- }
- String parentQueuePath = queueToRemove.substring(0, queueToRemove
- .lastIndexOf('.'));
- String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
- List<String> newSiblingQueues = new ArrayList<>();
- for (String siblingQueue : siblingQueues) {
- if (!siblingQueue.equals(queueName)) {
- newSiblingQueues.add(siblingQueue);
- }
- }
- proposedConf.setQueues(parentQueuePath, newSiblingQueues
- .toArray(new String[0]));
- String queuesConfig = CapacitySchedulerConfiguration.PREFIX +
- parentQueuePath + CapacitySchedulerConfiguration.DOT +
- CapacitySchedulerConfiguration.QUEUES;
- if (newSiblingQueues.size() == 0) {
- confUpdate.put(queuesConfig, null);
- } else {
- confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
- }
- for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
- ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
- .entrySet()) {
- proposedConf.unset(confRemove.getKey());
- confUpdate.put(confRemove.getKey(), null);
- }
- }
- }
-
- private void addQueue(
- QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) throws IOException {
- if (addInfo == null) {
- return;
- } else {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- String queuePath = addInfo.getQueue();
- String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
- if (cs.getQueue(queueName) != null) {
- throw new IOException("Can't add existing queue " + queuePath);
- } else if (queuePath.lastIndexOf('.') == -1) {
- throw new IOException("Can't add invalid queue " + queuePath);
- }
- String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
- String[] siblings = proposedConf.getQueues(parentQueue);
- List<String> siblingQueues = siblings == null ? new ArrayList<>() :
- new ArrayList<>(Arrays.<String>asList(siblings));
- siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
- proposedConf.setQueues(parentQueue,
- siblingQueues.toArray(new String[0]));
- confUpdate.put(CapacitySchedulerConfiguration.PREFIX +
- parentQueue + CapacitySchedulerConfiguration.DOT +
- CapacitySchedulerConfiguration.QUEUES,
- Joiner.on(',').join(siblingQueues));
- String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
- queuePath + CapacitySchedulerConfiguration.DOT;
- for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
- if (kv.getValue() == null) {
- proposedConf.unset(keyPrefix + kv.getKey());
- } else {
- proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
- }
- confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
- }
- }
- }
-
- private void updateQueue(QueueConfigInfo updateInfo,
- CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) {
- if (updateInfo == null) {
- return;
- } else {
- String queuePath = updateInfo.getQueue();
- String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
- queuePath + CapacitySchedulerConfiguration.DOT;
- for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
- if (kv.getValue() == null) {
- proposedConf.unset(keyPrefix + kv.getKey());
- } else {
- proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
- }
- confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
new file mode 100644
index 0000000..4016dcf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestConfigurationMutationACLPolicies {
+
+ private ConfigurationMutationACLPolicy policy;
+ private RMContext rmContext;
+ private MutableConfScheduler scheduler;
+
+ private static final UserGroupInformation GOOD_USER = UserGroupInformation
+ .createUserForTesting("goodUser", new String[] {});
+ private static final UserGroupInformation BAD_USER = UserGroupInformation
+ .createUserForTesting("badUser", new String[] {});
+ private static final Map<String, String> EMPTY_MAP =
+ Collections.<String, String>emptyMap();
+
+ @Before
+ public void setUp() throws IOException {
+ rmContext = mock(RMContext.class);
+ scheduler = mock(MutableConfScheduler.class);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
+ mockQueue("a", scheduler);
+ mockQueue("b", scheduler);
+ mockQueue("b1", scheduler);
+ }
+
+ private void mockQueue(String queueName, MutableConfScheduler scheduler)
+ throws IOException {
+ QueueInfo queueInfo = QueueInfo.newInstance(queueName, 0, 0, 0, null, null,
+ null, null, null, null, false);
+ when(scheduler.getQueueInfo(eq(queueName), anyBoolean(), anyBoolean()))
+ .thenReturn(queueInfo);
+ Queue queue = mock(Queue.class);
+ when(queue.hasAccess(eq(QueueACL.ADMINISTER_QUEUE), eq(GOOD_USER)))
+ .thenReturn(true);
+ when(queue.hasAccess(eq(QueueACL.ADMINISTER_QUEUE), eq(BAD_USER)))
+ .thenReturn(false);
+ when(scheduler.getQueue(eq(queueName))).thenReturn(queue);
+ }
+ @Test
+ public void testDefaultPolicy() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, GOOD_USER.getShortUserName());
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ DefaultConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, null));
+ assertFalse(policy.isMutationAllowed(BAD_USER, null));
+ }
+
+ @Test
+ public void testQueueAdminBasedPolicy() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.a", EMPTY_MAP);
+ updateInfo.getUpdateQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyAddQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Add root.b.b1. Should check ACL of root.b queue.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2", EMPTY_MAP);
+ updateInfo.getAddQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyAddNestedQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Add root.b.b1.b11. Should check ACL of root.b queue.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2.b21", EMPTY_MAP);
+ updateInfo.getAddQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyRemoveQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Remove root.b.b1.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.b.b1");
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d28ca16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 254da31..13229b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -19,8 +19,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import org.junit.Before;
import org.junit.Test;
@@ -43,22 +47,34 @@ public class TestMutableCSConfigurationProvider {
private MutableCSConfigurationProvider confProvider;
private RMContext rmContext;
- private Map<String, String> goodUpdate;
- private Map<String, String> badUpdate;
+ private QueueConfigsUpdateInfo goodUpdate;
+ private QueueConfigsUpdateInfo badUpdate;
private CapacityScheduler cs;
- private static final String TEST_USER = "testUser";
+ private static final UserGroupInformation TEST_USER = UserGroupInformation
+ .createUserForTesting("testUser", new String[] {});
@Before
public void setUp() {
cs = mock(CapacityScheduler.class);
rmContext = mock(RMContext.class);
when(rmContext.getScheduler()).thenReturn(cs);
+ when(cs.getConfiguration()).thenReturn(
+ new CapacitySchedulerConfiguration());
confProvider = new MutableCSConfigurationProvider(rmContext);
- goodUpdate = new HashMap<>();
- goodUpdate.put("goodKey", "goodVal");
- badUpdate = new HashMap<>();
- badUpdate.put("badKey", "badVal");
+ goodUpdate = new QueueConfigsUpdateInfo();
+ Map<String, String> goodUpdateMap = new HashMap<>();
+ goodUpdateMap.put("goodKey", "goodVal");
+ QueueConfigInfo goodUpdateInfo = new
+ QueueConfigInfo("root.a", goodUpdateMap);
+ goodUpdate.getUpdateQueueInfo().add(goodUpdateInfo);
+
+ badUpdate = new QueueConfigsUpdateInfo();
+ Map<String, String> badUpdateMap = new HashMap<>();
+ badUpdateMap.put("badKey", "badVal");
+ QueueConfigInfo badUpdateInfo = new
+ QueueConfigInfo("root.a", badUpdateMap);
+ badUpdate.getUpdateQueueInfo().add(badUpdateInfo);
}
@Test
@@ -66,15 +82,16 @@ public class TestMutableCSConfigurationProvider {
Configuration conf = new Configuration();
confProvider.init(conf);
assertNull(confProvider.loadConfiguration(conf)
- .get("goodKey"));
+ .get("yarn.scheduler.capacity.root.a.goodKey"));
doNothing().when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
confProvider.mutateConfiguration(TEST_USER, goodUpdate);
assertEquals("goodVal", confProvider.loadConfiguration(conf)
- .get("goodKey"));
+ .get("yarn.scheduler.capacity.root.a.goodKey"));
- assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ assertNull(confProvider.loadConfiguration(conf).get(
+ "yarn.scheduler.capacity.root.a.badKey"));
doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
try {
@@ -82,6 +99,7 @@ public class TestMutableCSConfigurationProvider {
} catch (IOException e) {
// Expected exception.
}
- assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ assertNull(confProvider.loadConfiguration(conf).get(
+ "yarn.scheduler.capacity.root.a.badKey"));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] hadoop git commit: HADOOP-14376. Memory leak when reading a
compressed file using the native library. Contributed by Eli Acherkan
Posted by wa...@apache.org.
HADOOP-14376. Memory leak when reading a compressed file using the native library. Contributed by Eli Acherkan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bc21722
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bc21722
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bc21722
Branch: refs/heads/YARN-5734
Commit: 7bc217224891b7f7f0a2e35e37e46b36d8c5309d
Parents: 6c35001
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Fri May 12 16:54:08 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Fri May 12 16:54:08 2017 -0500
----------------------------------------------------------------------
.../apache/hadoop/io/compress/BZip2Codec.java | 20 ++--
.../apache/hadoop/io/compress/CodecPool.java | 10 +-
.../io/compress/CompressionInputStream.java | 11 +-
.../io/compress/CompressionOutputStream.java | 16 ++-
.../hadoop/io/compress/CompressorStream.java | 3 +-
.../hadoop/io/compress/DecompressorStream.java | 7 +-
.../apache/hadoop/io/compress/TestCodec.java | 103 +++++++++++--------
7 files changed, 102 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc21722/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
index 08b4d4d..331606e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
@@ -336,15 +336,11 @@ public class BZip2Codec implements Configurable, SplittableCompressionCodec {
}
public void close() throws IOException {
- if (needsReset) {
- // In the case that nothing is written to this stream, we still need to
- // write out the header before closing, otherwise the stream won't be
- // recognized by BZip2CompressionInputStream.
- internalReset();
+ try {
+ super.close();
+ } finally {
+ output.close();
}
- this.output.flush();
- this.output.close();
- needsReset = true;
}
}// end of class BZip2CompressionOutputStream
@@ -454,8 +450,12 @@ public class BZip2Codec implements Configurable, SplittableCompressionCodec {
public void close() throws IOException {
if (!needsReset) {
- input.close();
- needsReset = true;
+ try {
+ input.close();
+ needsReset = true;
+ } finally {
+ super.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc21722/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
index bb566de..01bffa7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
@@ -157,7 +157,10 @@ public class CodecPool {
LOG.debug("Got recycled compressor");
}
}
- updateLeaseCount(compressorCounts, compressor, 1);
+ if (compressor != null &&
+ !compressor.getClass().isAnnotationPresent(DoNotPool.class)) {
+ updateLeaseCount(compressorCounts, compressor, 1);
+ }
return compressor;
}
@@ -184,7 +187,10 @@ public class CodecPool {
LOG.debug("Got recycled decompressor");
}
}
- updateLeaseCount(decompressorCounts, decompressor, 1);
+ if (decompressor != null &&
+ !decompressor.getClass().isAnnotationPresent(DoNotPool.class)) {
+ updateLeaseCount(decompressorCounts, decompressor, 1);
+ }
return decompressor;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc21722/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionInputStream.java
index cf3ac40..2dfa30b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionInputStream.java
@@ -59,10 +59,13 @@ public abstract class CompressionInputStream extends InputStream implements Seek
@Override
public void close() throws IOException {
- in.close();
- if (trackedDecompressor != null) {
- CodecPool.returnDecompressor(trackedDecompressor);
- trackedDecompressor = null;
+ try {
+ in.close();
+ } finally {
+ if (trackedDecompressor != null) {
+ CodecPool.returnDecompressor(trackedDecompressor);
+ trackedDecompressor = null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc21722/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionOutputStream.java
index 00e272a..71c7f32 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionOutputStream.java
@@ -56,11 +56,17 @@ public abstract class CompressionOutputStream extends OutputStream {
@Override
public void close() throws IOException {
- finish();
- out.close();
- if (trackedCompressor != null) {
- CodecPool.returnCompressor(trackedCompressor);
- trackedCompressor = null;
+ try {
+ finish();
+ } finally {
+ try {
+ out.close();
+ } finally {
+ if (trackedCompressor != null) {
+ CodecPool.returnCompressor(trackedCompressor);
+ trackedCompressor = null;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc21722/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressorStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressorStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressorStream.java
index 34426f8..be5eee0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressorStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressorStream.java
@@ -103,10 +103,9 @@ public class CompressorStream extends CompressionOutputStream {
public void close() throws IOException {
if (!closed) {
try {
- finish();
+ super.close();
}
finally {
- out.close();
closed = true;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc21722/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java
index dab366a..756ccf3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java
@@ -221,8 +221,11 @@ public class DecompressorStream extends CompressionInputStream {
@Override
public void close() throws IOException {
if (!closed) {
- in.close();
- closed = true;
+ try {
+ super.close();
+ } finally {
+ closed = true;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc21722/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
index 3955aa2..1ea9dc8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
@@ -205,66 +205,83 @@ public class TestCodec {
// Compress data
DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
- CompressionOutputStream deflateFilter =
+ int leasedCompressorsBefore = codec.getCompressorType() == null ? -1
+ : CodecPool.getLeasedCompressorsCount(codec);
+ try (CompressionOutputStream deflateFilter =
codec.createOutputStream(compressedDataBuffer);
- DataOutputStream deflateOut =
- new DataOutputStream(new BufferedOutputStream(deflateFilter));
- deflateOut.write(data.getData(), 0, data.getLength());
- deflateOut.flush();
- deflateFilter.finish();
+ DataOutputStream deflateOut =
+ new DataOutputStream(new BufferedOutputStream(deflateFilter))) {
+ deflateOut.write(data.getData(), 0, data.getLength());
+ deflateOut.flush();
+ deflateFilter.finish();
+ }
+ if (leasedCompressorsBefore > -1) {
+ assertEquals("leased compressor not returned to the codec pool",
+ leasedCompressorsBefore, CodecPool.getLeasedCompressorsCount(codec));
+ }
LOG.info("Finished compressing data");
// De-compress data
DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0,
compressedDataBuffer.getLength());
- CompressionInputStream inflateFilter =
- codec.createInputStream(deCompressedDataBuffer);
- DataInputStream inflateIn =
- new DataInputStream(new BufferedInputStream(inflateFilter));
-
- // Check
DataInputBuffer originalData = new DataInputBuffer();
- originalData.reset(data.getData(), 0, data.getLength());
- DataInputStream originalIn = new DataInputStream(new BufferedInputStream(originalData));
- for(int i=0; i < count; ++i) {
- RandomDatum k1 = new RandomDatum();
- RandomDatum v1 = new RandomDatum();
- k1.readFields(originalIn);
- v1.readFields(originalIn);
+ int leasedDecompressorsBefore =
+ CodecPool.getLeasedDecompressorsCount(codec);
+ try (CompressionInputStream inflateFilter =
+ codec.createInputStream(deCompressedDataBuffer);
+ DataInputStream inflateIn =
+ new DataInputStream(new BufferedInputStream(inflateFilter))) {
+
+ // Check
+ originalData.reset(data.getData(), 0, data.getLength());
+ DataInputStream originalIn =
+ new DataInputStream(new BufferedInputStream(originalData));
+ for(int i=0; i < count; ++i) {
+ RandomDatum k1 = new RandomDatum();
+ RandomDatum v1 = new RandomDatum();
+ k1.readFields(originalIn);
+ v1.readFields(originalIn);
- RandomDatum k2 = new RandomDatum();
- RandomDatum v2 = new RandomDatum();
- k2.readFields(inflateIn);
- v2.readFields(inflateIn);
- assertTrue("original and compressed-then-decompressed-output not equal",
- k1.equals(k2) && v1.equals(v2));
+ RandomDatum k2 = new RandomDatum();
+ RandomDatum v2 = new RandomDatum();
+ k2.readFields(inflateIn);
+ v2.readFields(inflateIn);
+ assertTrue("original and compressed-then-decompressed-output not equal",
+ k1.equals(k2) && v1.equals(v2));
- // original and compressed-then-decompressed-output have the same hashCode
- Map<RandomDatum, String> m = new HashMap<RandomDatum, String>();
- m.put(k1, k1.toString());
- m.put(v1, v1.toString());
- String result = m.get(k2);
- assertEquals("k1 and k2 hashcode not equal", result, k1.toString());
- result = m.get(v2);
- assertEquals("v1 and v2 hashcode not equal", result, v1.toString());
+ // original and compressed-then-decompressed-output have the same
+ // hashCode
+ Map<RandomDatum, String> m = new HashMap<RandomDatum, String>();
+ m.put(k1, k1.toString());
+ m.put(v1, v1.toString());
+ String result = m.get(k2);
+ assertEquals("k1 and k2 hashcode not equal", result, k1.toString());
+ result = m.get(v2);
+ assertEquals("v1 and v2 hashcode not equal", result, v1.toString());
+ }
}
+ assertEquals("leased decompressor not returned to the codec pool",
+ leasedDecompressorsBefore,
+ CodecPool.getLeasedDecompressorsCount(codec));
// De-compress data byte-at-a-time
originalData.reset(data.getData(), 0, data.getLength());
deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0,
compressedDataBuffer.getLength());
- inflateFilter =
+ try (CompressionInputStream inflateFilter =
codec.createInputStream(deCompressedDataBuffer);
-
- // Check
- originalIn = new DataInputStream(new BufferedInputStream(originalData));
- int expected;
- do {
- expected = originalIn.read();
- assertEquals("Inflated stream read by byte does not match",
- expected, inflateFilter.read());
- } while (expected != -1);
+ DataInputStream originalIn =
+ new DataInputStream(new BufferedInputStream(originalData))) {
+
+ // Check
+ int expected;
+ do {
+ expected = originalIn.read();
+ assertEquals("Inflated stream read by byte does not match",
+ expected, inflateFilter.read());
+ } while (expected != -1);
+ }
LOG.info("SUCCESS! Completed checking " + count + " records");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] hadoop git commit: HDFS-11803. Add -v option for du command
to show header line. Contributed by Xiaobing Zhou
Posted by wa...@apache.org.
HDFS-11803. Add -v option for du command to show header line. Contributed by Xiaobing Zhou
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1db186f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1db186f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1db186f6
Branch: refs/heads/YARN-5734
Commit: 1db186f662af971d956d74aed03150b0692a2315
Parents: e82da2b
Author: Mingliang Liu <li...@apache.org>
Authored: Fri May 19 14:17:14 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Fri May 19 14:21:01 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/shell/FsUsage.java | 14 +++++++++++---
.../src/site/markdown/FileSystemShell.md | 1 +
.../hadoop-common/src/test/resources/testConf.xml | 2 +-
3 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1db186f6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
index 6b1d7e0..6596527 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
@@ -160,7 +160,7 @@ class FsUsage extends FsCommand {
/** show disk usage */
public static class Du extends FsUsage {
public static final String NAME = "du";
- public static final String USAGE = "[-s] [-h] [-x] <path> ...";
+ public static final String USAGE = "[-s] [-h] [-v] [-x] <path> ...";
public static final String DESCRIPTION =
"Show the amount of space, in bytes, used by the files that match " +
"the specified file pattern. The following flags are optional:\n" +
@@ -168,6 +168,7 @@ class FsUsage extends FsCommand {
" matches the pattern, shows the total (summary) size.\n" +
"-h: Formats the sizes of files in a human-readable fashion" +
" rather than a number of bytes.\n" +
+ "-v: option displays a header line.\n" +
"-x: Excludes snapshots from being counted.\n\n" +
"Note that, even without the -s option, this only shows size " +
"summaries one level deep into a directory.\n\n" +
@@ -175,14 +176,16 @@ class FsUsage extends FsCommand {
"\tsize\tdisk space consumed\tname(full path)\n";
protected boolean summary = false;
+ private boolean showHeaderLine = false;
private boolean excludeSnapshots = false;
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
- CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s", "x");
+ CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s", "v", "x");
cf.parse(args);
setHumanReadable(cf.getOpt("h"));
summary = cf.getOpt("s");
+ showHeaderLine = cf.getOpt("v");
excludeSnapshots = cf.getOpt("x");
if (args.isEmpty()) args.add(Path.CUR_DIR);
}
@@ -190,7 +193,12 @@ class FsUsage extends FsCommand {
@Override
protected void processArguments(LinkedList<PathData> args)
throws IOException {
- setUsagesTable(new TableBuilder(3));
+ if (showHeaderLine) {
+ setUsagesTable(new TableBuilder("SIZE",
+ "DISK_SPACE_CONSUMED_WITH_ALL_REPLICAS", "FULL_PATH_NAME"));
+ } else {
+ setUsagesTable(new TableBuilder(3));
+ }
super.processArguments(args);
if (!getUsagesTable().isEmpty()) {
getUsagesTable().printToStream(out);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1db186f6/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 0412f1f..c5e0524 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -232,6 +232,7 @@ Options:
* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files. Without the -s option, calculation is done by going 1-level deep from the given path.
* The -h option will format file sizes in a "human-readable" fashion (e.g 64.0m instead of 67108864)
+* The -v option will display the names of columns as a header line.
* The -x option will exclude snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path.
The du returns three columns with the following format:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1db186f6/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 6347aa0..6644cd8 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -200,7 +200,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-du \[-s\] \[-h\] \[-x\] <path> \.\.\. :\s*</expected-output>
+ <expected-output>^-du \[-s\] \[-h\] \[-v\] \[-x\] <path> \.\.\. :\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] hadoop git commit: YARN-6249. TestFairSchedulerPreemption
fails inconsistently. (Tao Jie via Yufei Gu)
Posted by wa...@apache.org.
YARN-6249. TestFairSchedulerPreemption fails inconsistently. (Tao Jie via Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66bba8c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66bba8c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66bba8c0
Branch: refs/heads/YARN-5734
Commit: 66bba8c0244e5b17d496e2fa3999991705c11bc7
Parents: 1db186f
Author: Yufei Gu <yu...@cloudera.com>
Authored: Fri May 19 14:53:27 2017 -0700
Committer: Yufei Gu <yu...@cloudera.com>
Committed: Fri May 19 14:53:27 2017 -0700
----------------------------------------------------------------------
.../scheduler/fair/TestFairSchedulerPreemption.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66bba8c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
index 59d243b..3163024 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateS
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.After;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -188,6 +189,9 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
private void setupCluster() throws IOException {
resourceManager = new MockRM(conf);
scheduler = (FairScheduler) resourceManager.getResourceScheduler();
+ // YARN-6249, FSLeafQueue#lastTimeAtMinShare is initialized to the time in
+ // the real world, so we should keep the clock up with it.
+ clock.setTime(SystemClock.getInstance().getTime());
scheduler.setClock(clock);
resourceManager.start();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] hadoop git commit: YARN-6577. Remove unused
ContainerLocalization classes. Contributed by ZhangBing Lin
Posted by wa...@apache.org.
YARN-6577. Remove unused ContainerLocalization classes. Contributed by ZhangBing Lin
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b23fcc86
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b23fcc86
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b23fcc86
Branch: refs/heads/YARN-5734
Commit: b23fcc86c670aa896151a2bd8878154d7bb45d13
Parents: 44e9ef2
Author: Chris Douglas <cd...@apache.org>
Authored: Wed May 17 18:15:23 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Wed May 17 20:40:06 2017 -0700
----------------------------------------------------------------------
.../containermanager/ContainerLocalization.java | 21 --------------
.../ContainerLocalizationImpl.java | 29 --------------------
2 files changed, 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23fcc86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerLocalization.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerLocalization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerLocalization.java
deleted file mode 100644
index 992ee6f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerLocalization.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.hadoop.yarn.server.nodemanager.containermanager;
-
-public interface ContainerLocalization {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23fcc86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerLocalizationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerLocalizationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerLocalizationImpl.java
deleted file mode 100644
index ffeb1f8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerLocalizationImpl.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.hadoop.yarn.server.nodemanager.containermanager;
-
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
-
-public class ContainerLocalizationImpl implements ContainerLocalization {
-
- public ContainerLocalizationImpl(Dispatcher dispatcher, Application app,
- LocalizationProtocol localization) {
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] hadoop git commit: HDFS-11674. reserveSpaceForReplicas is not
released if append request failed due to mirror down and replica recovered
(Contributed by Vinayakumar B)
Posted by wa...@apache.org.
HDFS-11674. reserveSpaceForReplicas is not released if append request failed due to mirror down and replica recovered (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1411612a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1411612a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1411612a
Branch: refs/heads/YARN-5734
Commit: 1411612aa4e70c704b941723217ed4efd8a0125b
Parents: 0d5c8ed
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri May 12 07:38:18 2017 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Fri May 12 07:38:18 2017 +0530
----------------------------------------------------------------------
.../datanode/fsdataset/impl/FsDatasetImpl.java | 4 ++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 5 ++
.../fsdataset/impl/TestSpaceReservation.java | 67 +++++++++++++++++++-
3 files changed, 75 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1411612a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 9a5002a..e7d4d25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -2455,6 +2455,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
LOG.info("initReplicaRecovery: changing replica state for "
+ block + " from " + replica.getState()
+ " to " + rur.getState());
+ if (replica.getState() == ReplicaState.TEMPORARY || replica
+ .getState() == ReplicaState.RBW) {
+ ((ReplicaInPipeline) replica).releaseAllBytesReserved();
+ }
}
return rur.createInfo();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1411612a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index c98a336..2cfcc2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1462,6 +1462,11 @@ public class DFSTestUtil {
out.abort();
}
+ public static void setPipeline(DFSOutputStream out, LocatedBlock lastBlock)
+ throws IOException {
+ out.getStreamer().setPipelineInConstruction(lastBlock);
+ }
+
public static byte[] asArray(ByteBuffer buf) {
byte arr[] = new byte[buf.remaining()];
buf.duplicate().get(arr);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1411612a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
index fad5216..2daca86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
@@ -31,10 +31,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.HdfsBlockLocation;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.*;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -44,6 +47,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Daemon;
import org.apache.log4j.Level;
import org.junit.After;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -82,9 +86,12 @@ public class TestSpaceReservation {
private static Random rand = new Random();
- private void initConfig(int blockSize) {
+ @Before
+ public void before() {
conf = new HdfsConfiguration();
+ }
+ private void initConfig(int blockSize) {
// Refresh disk usage information frequently.
conf.setInt(FS_DU_INTERVAL_KEY, DU_REFRESH_INTERVAL_MSEC);
conf.setLong(DFS_BLOCK_SIZE_KEY, blockSize);
@@ -680,4 +687,62 @@ public class TestSpaceReservation {
}
}
}
+
+ @Test(timeout = 60000)
+ public void testReservedSpaceForLeaseRecovery() throws Exception {
+ final short replication = 3;
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 2);
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY,
+ 1000);
+ startCluster(BLOCK_SIZE, replication, -1);
+
+ final String methodName = GenericTestUtils.getMethodName();
+ final Path file = new Path("/" + methodName + ".01.dat");
+ // Write to the file and kill the writer.
+ FSDataOutputStream os = fs.create(file, replication);
+ os.write(new byte[8192]);
+ os.hflush();
+ os.close();
+ /*
+ * Reset the pipeline for the append in such a way that, datanode which is
+ * down is one of the mirror, not the first datanode.
+ */
+ HdfsBlockLocation blockLocation = (HdfsBlockLocation) fs.getClient()
+ .getBlockLocations(file.toString(), 0, BLOCK_SIZE)[0];
+ LocatedBlock lastBlock = blockLocation.getLocatedBlock();
+ // stop 3rd node.
+ cluster.stopDataNode(lastBlock.getLocations()[2].getName());
+ try {
+ os = fs.append(file);
+ DFSTestUtil.setPipeline((DFSOutputStream) os.getWrappedStream(),
+ lastBlock);
+ os.writeBytes("hi");
+ os.hsync();
+ } catch (IOException e) {
+ // Append will fail due to not able to replace datanodes in 3 nodes
+ // cluster.
+ LOG.info("", e);
+ }
+ DFSTestUtil.abortStream((DFSOutputStream) os.getWrappedStream());
+ /*
+ * There is a chance that stopped DN could be chosen as primary for
+ * recovery. If so, then recovery will not happen in time. So mark stopped
+ * node as dead to exclude that node.
+ */
+ cluster.setDataNodeDead(lastBlock.getLocations()[2]);
+ fs.recoverLease(file);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ return fs.isFileClosed(file);
+ } catch (IOException e) {
+ return false;
+ }
+ }
+ }, 500, 30000);
+ checkReservedSpace(0);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org