You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2018/07/27 00:12:24 UTC
[01/50] hadoop git commit: HDFS-13719. Docs around
dfs.image.transfer.timeout are misleading. Contributed by Kitti Nansi.
Repository: hadoop
Updated Branches:
refs/remotes/origin/branch-3.1 6f31faf92 -> a869bd970
HDFS-13719. Docs around dfs.image.transfer.timeout are misleading. Contributed by Kitti Nansi.
(cherry picked from commit eecb5baaaaa54599aeae758abd4007e55e5b531f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/242b5acd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/242b5acd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/242b5acd
Branch: refs/remotes/origin/branch-3.1
Commit: 242b5acdb358d886b6cc23707bc4e7b484ff514e
Parents: e994c4f
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Jul 9 15:17:21 2018 +0200
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Jul 9 15:17:34 2018 +0200
----------------------------------------------------------------------
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/242b5acd/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index d17020d..c092bff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -1286,11 +1286,10 @@
<name>dfs.image.transfer.timeout</name>
<value>60000</value>
<description>
- Socket timeout for image transfer in milliseconds. This timeout and the related
- dfs.image.transfer.bandwidthPerSec parameter should be configured such
- that normal image transfer can complete successfully.
- This timeout prevents client hangs when the sender fails during
- image transfer. This is socket timeout during image transfer.
+ Socket timeout for the HttpURLConnection instance used in the image
+ transfer. This is measured in milliseconds.
+ This timeout prevents client hangs if the connection is idle
+ for this configured timeout, during image transfer.
</description>
</property>
@@ -1301,9 +1300,7 @@
Maximum bandwidth used for regular image transfers (instead of
bootstrapping the standby namenode), in bytes per second.
This can help keep normal namenode operations responsive during
- checkpointing. The maximum bandwidth and timeout in
- dfs.image.transfer.timeout should be set such that normal image
- transfers can complete successfully.
+ checkpointing.
A default value of 0 indicates that throttling is disabled.
The maximum bandwidth used for bootstrapping standby namenode is
configured with dfs.image.transfer-bootstrap-standby.bandwidthPerSec.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] hadoop git commit: HADOOP-15612. Improve exception when tfile
fails to load LzoCodec. (gera)
Posted by ey...@apache.org.
HADOOP-15612. Improve exception when tfile fails to load LzoCodec. (gera)
(cherry picked from commit 6bec03cfc8bdcf6aa3df9c22231ab959ba31f2f5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1396fa2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1396fa2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1396fa2a
Branch: refs/remotes/origin/branch-3.1
Commit: 1396fa2a2727b8a678628d59768b355845eeb9a9
Parents: 40c06b3
Author: Gera Shegalov <ge...@apache.org>
Authored: Tue Jul 17 00:05:39 2018 -0700
Committer: Gera Shegalov <ge...@apache.org>
Committed: Tue Jul 24 23:05:34 2018 -0700
----------------------------------------------------------------------
.../hadoop/io/file/tfile/Compression.java | 31 +++++++++++-------
.../hadoop/io/file/tfile/TestCompression.java | 34 +++++++++++++++++++-
2 files changed, 53 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1396fa2a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
index fa85ed7..c4347e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
@@ -5,9 +5,9 @@
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
@@ -24,6 +24,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -78,25 +79,33 @@ public final class Compression {
public enum Algorithm {
LZO(TFile.COMPRESSION_LZO) {
private transient boolean checked = false;
+ private transient ClassNotFoundException cnf;
+ private transient boolean reinitCodecInTests;
private static final String defaultClazz =
"org.apache.hadoop.io.compress.LzoCodec";
+ private transient String clazz;
private transient CompressionCodec codec = null;
+ private String getLzoCodecClass() {
+ String extClazzConf = conf.get(CONF_LZO_CLASS);
+ String extClazz = (extClazzConf != null) ?
+ extClazzConf : System.getProperty(CONF_LZO_CLASS);
+ return (extClazz != null) ? extClazz : defaultClazz;
+ }
+
@Override
public synchronized boolean isSupported() {
- if (!checked) {
+ if (!checked || reinitCodecInTests) {
checked = true;
- String extClazzConf = conf.get(CONF_LZO_CLASS);
- String extClazz = (extClazzConf != null) ?
- extClazzConf : System.getProperty(CONF_LZO_CLASS);
- String clazz = (extClazz != null) ? extClazz : defaultClazz;
+ reinitCodecInTests = conf.getBoolean("test.reload.lzo.codec", false);
+ clazz = getLzoCodecClass();
try {
LOG.info("Trying to load Lzo codec class: " + clazz);
codec =
(CompressionCodec) ReflectionUtils.newInstance(Class
.forName(clazz), conf);
} catch (ClassNotFoundException e) {
- // that is okay
+ cnf = e;
}
}
return codec != null;
@@ -105,9 +114,9 @@ public final class Compression {
@Override
CompressionCodec getCodec() throws IOException {
if (!isSupported()) {
- throw new IOException(
- "LZO codec class not specified. Did you forget to set property "
- + CONF_LZO_CLASS + "?");
+ throw new IOException(String.format(
+ "LZO codec %s=%s could not be loaded", CONF_LZO_CLASS, clazz),
+ cnf);
}
return codec;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1396fa2a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
index ff6c72a..b1bf077 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
@@ -17,14 +17,28 @@
*/
package org.apache.hadoop.io.file.tfile;
-import org.junit.Test;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.*;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class TestCompression {
+ @BeforeClass
+ public static void resetConfigBeforeAll() {
+ Compression.Algorithm.LZO.conf.setBoolean("test.reload.lzo.codec", true);
+ }
+
+ @AfterClass
+ public static void resetConfigAfterAll() {
+ Compression.Algorithm.LZO.conf.setBoolean("test.reload.lzo.codec", false);
+ }
+
/**
* Regression test for HADOOP-11418.
* Verify we can set a LZO codec different from default LZO codec.
@@ -38,4 +52,22 @@ public class TestCompression {
assertEquals(defaultCodec,
Compression.Algorithm.LZO.getCodec().getClass().getName());
}
+
+
+ @Test
+ public void testMisconfiguredLZOCodec() throws Exception {
+ // Dummy codec
+ String defaultCodec = "org.apache.hadoop.io.compress.InvalidLzoCodec";
+ Compression.Algorithm.conf.set(
+ Compression.Algorithm.CONF_LZO_CLASS, defaultCodec);
+ IOException ioEx = LambdaTestUtils.intercept(
+ IOException.class,
+ defaultCodec,
+ () -> Compression.Algorithm.LZO.getCodec());
+
+ if (!(ioEx.getCause() instanceof ClassNotFoundException)) {
+ throw ioEx;
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] hadoop git commit: HDFS-13743. RBF: Router throws
NullPointerException due to the invalid initialization of MountTableResolver.
Contributed by Takanobu Asanuma.
Posted by ey...@apache.org.
HDFS-13743. RBF: Router throws NullPointerException due to the invalid initialization of MountTableResolver. Contributed by Takanobu Asanuma.
(cherry picked from commit 7b25fb949bf6f02df997beeca7df46c9e84c8d96)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/823d576a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/823d576a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/823d576a
Branch: refs/remotes/origin/branch-3.1
Commit: 823d576a66e51c4eb93b18d28a378505cdd34561
Parents: 23624c9
Author: Yiqun Lin <yq...@apache.org>
Authored: Fri Jul 20 17:28:57 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Fri Jul 20 17:31:13 2018 +0800
----------------------------------------------------------------------
.../federation/resolver/MountTableResolver.java | 28 +++++--
.../TestInitializeMountTableResolver.java | 82 ++++++++++++++++++++
2 files changed, 102 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823d576a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index 3f6efd6..c264de3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.federation.resolver;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMESERVICES;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMESERVICE_ID;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT;
@@ -42,7 +44,6 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -149,14 +150,25 @@ public class MountTableResolver
* @param conf Configuration for this resolver.
*/
private void initDefaultNameService(Configuration conf) {
- try {
- this.defaultNameService = conf.get(
- DFS_ROUTER_DEFAULT_NAMESERVICE,
- DFSUtil.getNamenodeNameServiceId(conf));
- } catch (HadoopIllegalArgumentException e) {
- LOG.error("Cannot find default name service, setting it to the first");
+ this.defaultNameService = conf.get(
+ DFS_ROUTER_DEFAULT_NAMESERVICE,
+ DFSUtil.getNamenodeNameServiceId(conf));
+
+ if (defaultNameService == null) {
+ LOG.warn(
+ "{} and {} is not set. Fallback to {} as the default name service.",
+ DFS_ROUTER_DEFAULT_NAMESERVICE, DFS_NAMESERVICE_ID, DFS_NAMESERVICES);
Collection<String> nsIds = DFSUtilClient.getNameServiceIds(conf);
- this.defaultNameService = nsIds.iterator().next();
+ if (nsIds.isEmpty()) {
+ this.defaultNameService = "";
+ } else {
+ this.defaultNameService = nsIds.iterator().next();
+ }
+ }
+
+ if (this.defaultNameService.equals("")) {
+ LOG.warn("Default name service is not set.");
+ } else {
LOG.info("Default name service: {}", this.defaultNameService);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823d576a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestInitializeMountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestInitializeMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestInitializeMountTableResolver.java
new file mode 100644
index 0000000..5db7531
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestInitializeMountTableResolver.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMESERVICES;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test {@link MountTableResolver} initialization.
+ */
+public class TestInitializeMountTableResolver {
+
+ @Test
+ public void testDefaultNameserviceIsMissing() {
+ Configuration conf = new Configuration();
+ MountTableResolver mountTable = new MountTableResolver(conf);
+ assertEquals("", mountTable.getDefaultNamespace());
+ }
+
+ @Test
+ public void testDefaultNameserviceWithEmptyString() {
+ Configuration conf = new Configuration();
+ conf.set(DFS_ROUTER_DEFAULT_NAMESERVICE, "");
+ MountTableResolver mountTable = new MountTableResolver(conf);
+ assertEquals("", mountTable.getDefaultNamespace());
+ }
+
+ @Test
+ public void testRouterDefaultNameservice() {
+ Configuration conf = new Configuration();
+ conf.set(DFS_ROUTER_DEFAULT_NAMESERVICE, "router_ns"); // this is priority
+ conf.set(DFS_NAMESERVICE_ID, "ns_id");
+ conf.set(DFS_NAMESERVICES, "nss");
+ MountTableResolver mountTable = new MountTableResolver(conf);
+ assertEquals("router_ns", mountTable.getDefaultNamespace());
+ }
+
+ @Test
+ public void testNameserviceID() {
+ Configuration conf = new Configuration();
+ conf.set(DFS_NAMESERVICE_ID, "ns_id"); // this is priority
+ conf.set(DFS_NAMESERVICES, "nss");
+ MountTableResolver mountTable = new MountTableResolver(conf);
+ assertEquals("ns_id", mountTable.getDefaultNamespace());
+ }
+
+ @Test
+ public void testSingleNameservices() {
+ Configuration conf = new Configuration();
+ conf.set(DFS_NAMESERVICES, "ns1");
+ MountTableResolver mountTable = new MountTableResolver(conf);
+ assertEquals("ns1", mountTable.getDefaultNamespace());
+ }
+
+ @Test
+ public void testMultipleNameservices() {
+ Configuration conf = new Configuration();
+ conf.set(DFS_NAMESERVICES, "ns1,ns2");
+ MountTableResolver mountTable = new MountTableResolver(conf);
+ assertEquals("ns1", mountTable.getDefaultNamespace());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] hadoop git commit: YARN-8538. Fixed memory leaks in
container-executor and test cases. Contributed by Billie Rinaldi
Posted by ey...@apache.org.
YARN-8538. Fixed memory leaks in container-executor and test cases.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d82edec3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d82edec3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d82edec3
Branch: refs/remotes/origin/branch-3.1
Commit: d82edec3c02db4eae3af91760aa31dfbbe7a9e03
Parents: 228508a
Author: Eric Yang <ey...@apache.org>
Authored: Mon Jul 16 17:38:49 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Jul 18 13:44:49 2018 -0400
----------------------------------------------------------------------
.../container-executor/impl/configuration.c | 3 +
.../main/native/container-executor/impl/main.c | 4 +-
.../impl/modules/cgroups/cgroups-operations.c | 2 +
.../container-executor/impl/utils/docker-util.c | 234 ++++++++++---------
.../test/modules/cgroups/test-cgroups-module.cc | 8 +
.../test/modules/fpga/test-fpga-module.cc | 24 +-
.../test/modules/gpu/test-gpu-module.cc | 24 +-
.../test/test_configuration.cc | 34 ++-
.../native/container-executor/test/test_util.cc | 5 +
.../test/utils/test-string-utils.cc | 6 +
.../test/utils/test_docker_util.cc | 128 ++++++----
11 files changed, 307 insertions(+), 165 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index f23cff0..baaa4dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -58,6 +58,7 @@ void free_section(struct section *section) {
section->name = NULL;
}
section->size = 0;
+ free(section);
}
//clean up method for freeing configuration
@@ -466,6 +467,7 @@ static void merge_sections(struct section *section1, struct section *section2, c
section1->size += section2->size;
if (free_second_section) {
free(section2->name);
+ free(section2->kv_pairs);
memset(section2, 0, sizeof(*section2));
free(section2);
}
@@ -708,6 +710,7 @@ char *get_config_path(const char *argv0) {
const char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
char *conf_file = resolve_config_path(orig_conf_file, executable_file);
+ free(executable_file);
if (conf_file == NULL) {
fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 3d7b19a..1ed3ce8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -128,11 +128,13 @@ static void flush_and_close_log_files() {
LOGFILE = NULL;
}
-if (ERRORFILE != NULL) {
+ if (ERRORFILE != NULL) {
fflush(ERRORFILE);
fclose(ERRORFILE);
ERRORFILE = NULL;
}
+
+ free_executor_configurations();
}
/** Validates the current container-executor setup. Causes program exit
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
index b234109..ea1d36d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
@@ -83,6 +83,8 @@ char* get_cgroups_path_to_write(
}
cleanup:
+ free((void *) cgroups_root);
+ free((void *) yarn_hierarchy_name);
if (failed) {
if (buffer) {
free(buffer);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index d364227..580cd37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -143,9 +143,9 @@ int check_trusted_image(const struct configuration *command_config, const struct
fprintf(ERRORFILE, "image: %s is not trusted.\n", image_name);
ret = INVALID_DOCKER_IMAGE_TRUST;
}
- free(image_name);
free_and_exit:
+ free(image_name);
free_values(privileged_registry);
return ret;
}
@@ -195,7 +195,8 @@ static int add_param_to_command_if_allowed(const struct configuration *command_c
if (strcmp(key, "net") != 0) {
if (check_trusted_image(command_config, executor_cfg) != 0) {
fprintf(ERRORFILE, "Disable %s for untrusted image\n", key);
- return INVALID_DOCKER_IMAGE_TRUST;
+ ret = INVALID_DOCKER_IMAGE_TRUST;
+ goto free_and_exit;
}
}
@@ -225,13 +226,13 @@ static int add_param_to_command_if_allowed(const struct configuration *command_c
dst = strndup(values[i], tmp_ptr - values[i]);
pattern = strdup(permitted_values[j] + 6);
ret = execute_regex_match(pattern, dst);
+ free(dst);
+ free(pattern);
} else {
ret = strncmp(values[i], permitted_values[j], tmp_ptr - values[i]);
}
}
if (ret == 0) {
- free(dst);
- free(pattern);
permitted = 1;
break;
}
@@ -259,7 +260,7 @@ static int add_param_to_command_if_allowed(const struct configuration *command_c
}
}
- free_and_exit:
+free_and_exit:
free_values(values);
free_values(permitted_values);
return ret;
@@ -379,6 +380,7 @@ int get_docker_command(const char *command_file, const struct configuration *con
ret = read_config(command_file, &command_config);
if (ret != 0) {
+ free_configuration(&command_config);
return INVALID_COMMAND_FILE;
}
@@ -392,36 +394,41 @@ int get_docker_command(const char *command_file, const struct configuration *con
ret = add_to_args(args, docker);
free(docker);
if (ret != 0) {
+ free_configuration(&command_config);
return BUFFER_TOO_SMALL;
}
ret = add_docker_config_param(&command_config, args);
if (ret != 0) {
+ free_configuration(&command_config);
return BUFFER_TOO_SMALL;
}
char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ free_configuration(&command_config);
if (strcmp(DOCKER_INSPECT_COMMAND, command) == 0) {
- return get_docker_inspect_command(command_file, conf, args);
+ ret = get_docker_inspect_command(command_file, conf, args);
} else if (strcmp(DOCKER_KILL_COMMAND, command) == 0) {
- return get_docker_kill_command(command_file, conf, args);
+ ret = get_docker_kill_command(command_file, conf, args);
} else if (strcmp(DOCKER_LOAD_COMMAND, command) == 0) {
- return get_docker_load_command(command_file, conf, args);
+ ret = get_docker_load_command(command_file, conf, args);
} else if (strcmp(DOCKER_PULL_COMMAND, command) == 0) {
- return get_docker_pull_command(command_file, conf, args);
+ ret = get_docker_pull_command(command_file, conf, args);
} else if (strcmp(DOCKER_RM_COMMAND, command) == 0) {
- return get_docker_rm_command(command_file, conf, args);
+ ret = get_docker_rm_command(command_file, conf, args);
} else if (strcmp(DOCKER_RUN_COMMAND, command) == 0) {
- return get_docker_run_command(command_file, conf, args);
+ ret = get_docker_run_command(command_file, conf, args);
} else if (strcmp(DOCKER_STOP_COMMAND, command) == 0) {
- return get_docker_stop_command(command_file, conf, args);
+ ret = get_docker_stop_command(command_file, conf, args);
} else if (strcmp(DOCKER_VOLUME_COMMAND, command) == 0) {
- return get_docker_volume_command(command_file, conf, args);
+ ret = get_docker_volume_command(command_file, conf, args);
} else if (strcmp(DOCKER_START_COMMAND, command) == 0) {
- return get_docker_start_command(command_file, conf, args);
+ ret = get_docker_start_command(command_file, conf, args);
} else {
- return UNKNOWN_DOCKER_COMMAND;
+ ret = UNKNOWN_DOCKER_COMMAND;
}
+ free(command);
+ return ret;
}
// check if a key is permitted in the configuration
@@ -456,7 +463,7 @@ int get_docker_volume_command(const char *command_file, const struct configurati
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_VOLUME_COMMAND, &command_config);
if (ret != 0) {
- return ret;
+ goto cleanup;
}
sub_command = get_configuration_value("sub-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
@@ -533,6 +540,7 @@ int get_docker_volume_command(const char *command_file, const struct configurati
}
cleanup:
+ free_configuration(&command_config);
free(driver);
free(volume_name);
free(sub_command);
@@ -548,18 +556,19 @@ int get_docker_inspect_command(const char *command_file, const struct configurat
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_INSPECT_COMMAND, &command_config);
if (ret != 0) {
- return ret;
+ goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
- return INVALID_DOCKER_CONTAINER_NAME;
+ ret = INVALID_DOCKER_CONTAINER_NAME;
+ goto free_and_exit;
}
format = get_configuration_value("format", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (format == NULL) {
- free(container_name);
- return INVALID_DOCKER_INSPECT_FORMAT;
+ ret = INVALID_DOCKER_INSPECT_FORMAT;
+ goto free_and_exit;
}
for (i = 0; i < 2; ++i) {
if (strcmp(format, valid_format_strings[i]) == 0) {
@@ -569,9 +578,8 @@ int get_docker_inspect_command(const char *command_file, const struct configurat
}
if (valid_format != 1) {
fprintf(ERRORFILE, "Invalid format option '%s' not permitted\n", format);
- free(container_name);
- free(format);
- return INVALID_DOCKER_INSPECT_FORMAT;
+ ret = INVALID_DOCKER_INSPECT_FORMAT;
+ goto free_and_exit;
}
ret = add_to_args(args, DOCKER_INSPECT_COMMAND);
@@ -588,14 +596,12 @@ int get_docker_inspect_command(const char *command_file, const struct configurat
if (ret != 0) {
goto free_and_exit;
}
- free(format);
- free(container_name);
- return 0;
- free_and_exit:
+free_and_exit:
+ free_configuration(&command_config);
free(format);
free(container_name);
- return BUFFER_TOO_SMALL;
+ return ret;
}
int get_docker_load_command(const char *command_file, const struct configuration *conf, args *args) {
@@ -604,12 +610,13 @@ int get_docker_load_command(const char *command_file, const struct configuration
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_LOAD_COMMAND, &command_config);
if (ret != 0) {
- return ret;
+ goto free_and_exit;
}
image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (image_name == NULL) {
- return INVALID_DOCKER_IMAGE_NAME;
+ ret = INVALID_DOCKER_IMAGE_NAME;
+ goto free_and_exit;
}
ret = add_to_args(args, DOCKER_LOAD_COMMAND);
@@ -617,14 +624,14 @@ int get_docker_load_command(const char *command_file, const struct configuration
char *tmp_buffer = make_string("--i=%s", image_name);
ret = add_to_args(args, tmp_buffer);
free(tmp_buffer);
- free(image_name);
if (ret != 0) {
- return BUFFER_TOO_SMALL;
+ ret = BUFFER_TOO_SMALL;
}
- return 0;
}
+free_and_exit:
free(image_name);
- return BUFFER_TOO_SMALL;
+ free_configuration(&command_config);
+ return ret;
}
static int validate_docker_image_name(const char *image_name) {
@@ -638,26 +645,23 @@ int get_docker_pull_command(const char *command_file, const struct configuration
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_PULL_COMMAND, &command_config);
if (ret != 0) {
- return ret;
+ goto free_pull;
}
image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (image_name == NULL || validate_docker_image_name(image_name) != 0) {
- return INVALID_DOCKER_IMAGE_NAME;
+ ret = INVALID_DOCKER_IMAGE_NAME;
+ goto free_pull;
}
ret = add_to_args(args, DOCKER_PULL_COMMAND);
if (ret == 0) {
ret = add_to_args(args, image_name);
- free(image_name);
- if (ret != 0) {
- goto free_pull;
- }
- return 0;
}
- free_pull:
+free_pull:
free(image_name);
- return BUFFER_TOO_SMALL;
+ free_configuration(&command_config);
+ return ret;
}
int get_docker_rm_command(const char *command_file, const struct configuration *conf, args *args) {
@@ -666,25 +670,26 @@ int get_docker_rm_command(const char *command_file, const struct configuration *
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_RM_COMMAND, &command_config);
if (ret != 0) {
- return ret;
+ goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
- return INVALID_DOCKER_CONTAINER_NAME;
+ ret = INVALID_DOCKER_CONTAINER_NAME;
+ goto free_and_exit;
}
ret = add_to_args(args, DOCKER_RM_COMMAND);
if (ret == 0) {
ret = add_to_args(args, container_name);
- free(container_name);
if (ret != 0) {
- return BUFFER_TOO_SMALL;
+ ret = BUFFER_TOO_SMALL;
}
- return 0;
}
+free_and_exit:
free(container_name);
- return BUFFER_TOO_SMALL;
+ free_configuration(&command_config);
+ return ret;
}
int get_docker_stop_command(const char *command_file, const struct configuration *conf,
@@ -696,12 +701,13 @@ int get_docker_stop_command(const char *command_file, const struct configuration
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_STOP_COMMAND, &command_config);
if (ret != 0) {
- return ret;
+ goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
- return INVALID_DOCKER_CONTAINER_NAME;
+ ret = INVALID_DOCKER_CONTAINER_NAME;
+ goto free_and_exit;
}
ret = add_to_args(args, DOCKER_STOP_COMMAND);
@@ -726,7 +732,9 @@ int get_docker_stop_command(const char *command_file, const struct configuration
ret = add_to_args(args, container_name);
}
free_and_exit:
+ free(value);
free(container_name);
+ free_configuration(&command_config);
return ret;
}
@@ -739,12 +747,13 @@ int get_docker_kill_command(const char *command_file, const struct configuration
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_KILL_COMMAND, &command_config);
if (ret != 0) {
- return ret;
+ goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
- return INVALID_DOCKER_CONTAINER_NAME;
+ ret = INVALID_DOCKER_CONTAINER_NAME;
+ goto free_and_exit;
}
ret = add_to_args(args, DOCKER_KILL_COMMAND);
@@ -770,7 +779,9 @@ int get_docker_kill_command(const char *command_file, const struct configuration
ret = add_to_args(args, container_name);
}
free_and_exit:
+ free(value);
free(container_name);
+ free_configuration(&command_config);
return ret;
}
@@ -780,12 +791,13 @@ int get_docker_start_command(const char *command_file, const struct configuratio
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_START_COMMAND, &command_config);
if (ret != 0) {
- return ret;
+ goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
- return INVALID_DOCKER_CONTAINER_NAME;
+ ret = INVALID_DOCKER_CONTAINER_NAME;
+ goto free_and_exit;
}
ret = add_to_args(args, DOCKER_START_COMMAND);
@@ -795,6 +807,7 @@ int get_docker_start_command(const char *command_file, const struct configuratio
ret = add_to_args(args, container_name);
free_and_exit:
free(container_name);
+ free_configuration(&command_config);
return ret;
}
@@ -1092,7 +1105,9 @@ static int add_mounts(const struct configuration *command_config, const struct c
char **permitted_rw_mounts = get_configuration_values_delimiter("docker.allowed.rw-mounts",
CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
char **values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
- const char *container_executor_cfg_path = normalize_mount(get_config_path(""), 0);
+ char *config_path = get_config_path("");
+ const char *container_executor_cfg_path = normalize_mount(config_path, 0);
+ free(config_path);
int i = 0, permitted_rw = 0, permitted_ro = 0, ret = 0;
if (ro != 0) {
ro_suffix = ":ro";
@@ -1172,6 +1187,8 @@ static int add_mounts(const struct configuration *command_config, const struct c
ret = BUFFER_TOO_SMALL;
goto free_and_exit;
}
+ free(mount_src);
+ mount_src = NULL;
}
}
@@ -1312,7 +1329,7 @@ static int set_privileged(const struct configuration *command_config, const stru
}
}
- free_and_exit:
+free_and_exit:
free(value);
free(privileged_container_enabled);
free(user);
@@ -1325,42 +1342,41 @@ int get_docker_run_command(const char *command_file, const struct configuration
char *tmp_buffer = NULL;
char **launch_command = NULL;
char *privileged = NULL;
+ char *no_new_privileges_enabled = NULL;
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_RUN_COMMAND, &command_config);
if (ret != 0) {
- return ret;
+ goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
- if (container_name != NULL) {
- free(container_name);
- }
- return INVALID_DOCKER_CONTAINER_NAME;
+ ret = INVALID_DOCKER_CONTAINER_NAME;
+ goto free_and_exit;
}
user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (user == NULL) {
- return INVALID_DOCKER_USER_NAME;
+ ret = INVALID_DOCKER_USER_NAME;
+ goto free_and_exit;
}
image = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (image == NULL || validate_docker_image_name(image) != 0) {
- if (image != NULL) {
- free(image);
- }
- return INVALID_DOCKER_IMAGE_NAME;
+ ret = INVALID_DOCKER_IMAGE_NAME;
+ goto free_and_exit;
}
ret = add_to_args(args, DOCKER_RUN_COMMAND);
if(ret != 0) {
- reset_args(args);
- return BUFFER_TOO_SMALL;
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
}
tmp_buffer = make_string("--name=%s", container_name);
ret = add_to_args(args, tmp_buffer);
+ free(tmp_buffer);
if (ret != 0) {
- reset_args(args);
- return BUFFER_TOO_SMALL;
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
}
privileged = get_configuration_value("privileged", DOCKER_COMMAND_FILE_SECTION, &command_config);
@@ -1370,111 +1386,95 @@ int get_docker_run_command(const char *command_file, const struct configuration
ret = add_to_args(args, user_buffer);
free(user_buffer);
if (ret != 0) {
- reset_args(args);
- return BUFFER_TOO_SMALL;
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
}
- char *no_new_privileges_enabled =
+ no_new_privileges_enabled =
get_configuration_value("docker.no-new-privileges.enabled",
CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
if (no_new_privileges_enabled != NULL &&
strcasecmp(no_new_privileges_enabled, "True") == 0) {
ret = add_to_args(args, "--security-opt=no-new-privileges");
if (ret != 0) {
- reset_args(args);
- return BUFFER_TOO_SMALL;
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
}
}
- free(no_new_privileges_enabled);
}
- free(privileged);
ret = detach_container(&command_config, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = rm_container_on_exit(&command_config, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = set_container_workdir(&command_config, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = set_network(&command_config, conf, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = set_pid_namespace(&command_config, conf, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = add_ro_mounts(&command_config, conf, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = add_rw_mounts(&command_config, conf, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = set_cgroup_parent(&command_config, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = set_privileged(&command_config, conf, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = set_capabilities(&command_config, conf, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = set_hostname(&command_config, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = set_group_add(&command_config, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = set_devices(&command_config, conf, args);
if (ret != 0) {
- reset_args(args);
- return ret;
+ goto free_and_exit;
}
ret = set_env(&command_config, args);
if (ret != 0) {
- return BUFFER_TOO_SMALL;
+ goto free_and_exit;
}
ret = add_to_args(args, image);
if (ret != 0) {
- reset_args(args);
- return BUFFER_TOO_SMALL;
+ goto free_and_exit;
}
launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION, &command_config,
@@ -1483,13 +1483,21 @@ int get_docker_run_command(const char *command_file, const struct configuration
for (i = 0; launch_command[i] != NULL; ++i) {
ret = add_to_args(args, launch_command[i]);
if (ret != 0) {
- free_values(launch_command);
- reset_args(args);
- return BUFFER_TOO_SMALL;
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
}
}
- free_values(launch_command);
}
- free(tmp_buffer);
- return 0;
+free_and_exit:
+ if (ret != 0) {
+ reset_args(args);
+ }
+ free(user);
+ free(image);
+ free(privileged);
+ free(no_new_privileges_enabled);
+ free(container_name);
+ free_values(launch_command);
+ free_configuration(&command_config);
+ return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
index 8ffbe88..078456d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
@@ -73,6 +73,8 @@ TEST_F(TestCGroupsModule, test_cgroups_get_path_without_define_root) {
char* path = get_cgroups_path_to_write("devices", "deny", "container_1");
ASSERT_TRUE(NULL == path) << "Should fail.\n";
+
+ free_executor_configurations();
}
TEST_F(TestCGroupsModule, test_cgroups_get_path_without_define_yarn_hierarchy) {
@@ -92,6 +94,8 @@ TEST_F(TestCGroupsModule, test_cgroups_get_path_without_define_yarn_hierarchy) {
char* path = get_cgroups_path_to_write("devices", "deny", "container_1");
ASSERT_TRUE(NULL == path) << "Should fail.\n";
+
+ free_executor_configurations();
}
TEST_F(TestCGroupsModule, test_cgroups_get_path_succeeded) {
@@ -117,5 +121,9 @@ TEST_F(TestCGroupsModule, test_cgroups_get_path_succeeded) {
ASSERT_STREQ(EXPECTED, path)
<< "Return cgroup-path-to-write is not expected\n";
+
+ free(path);
+
+ free_executor_configurations();
}
} // namespace ContainerExecutor
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/fpga/test-fpga-module.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/fpga/test-fpga-module.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/fpga/test-fpga-module.cc
index 1e5c5ea..a5d1dff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/fpga/test-fpga-module.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/fpga/test-fpga-module.cc
@@ -84,6 +84,13 @@ static int mock_update_cgroups_parameters(
return 0;
}
+static void clear_cgroups_parameters_invoked() {
+ for (std::vector<const char*>::size_type i = 0; i < cgroups_parameters_invoked.size(); i++) {
+ free((void *) cgroups_parameters_invoked[i]);
+ }
+ cgroups_parameters_invoked.clear();
+}
+
static void verify_param_updated_to_cgroups(
int argc, const char** argv) {
ASSERT_EQ(argc, cgroups_parameters_invoked.size());
@@ -133,6 +140,9 @@ static void test_fpga_module_enabled_disabled(int enabled) {
EXPECTED_RC = -1;
}
ASSERT_EQ(EXPECTED_RC, rc);
+
+ clear_cgroups_parameters_invoked();
+ free_executor_configurations();
}
TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
@@ -146,7 +156,7 @@ TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
container_id };
/* Test case 1: block 2 devices */
- cgroups_parameters_invoked.clear();
+ clear_cgroups_parameters_invoked();
int rc = handle_fpga_request(&mock_update_cgroups_parameters,
"fpga", 5, argv);
ASSERT_EQ(0, rc) << "Should success.\n";
@@ -157,7 +167,7 @@ TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
verify_param_updated_to_cgroups(8, expected_cgroups_argv);
/* Test case 2: block 0 devices */
- cgroups_parameters_invoked.clear();
+ clear_cgroups_parameters_invoked();
char* argv_1[] = { (char*) "--module-fpga", (char*) "--container_id", container_id };
rc = handle_fpga_request(&mock_update_cgroups_parameters,
"fpga", 3, argv_1);
@@ -167,7 +177,7 @@ TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
verify_param_updated_to_cgroups(0, NULL);
/* Test case 3: block 2 non-sequential devices */
- cgroups_parameters_invoked.clear();
+ clear_cgroups_parameters_invoked();
char* argv_2[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "1,3",
(char*) "--container_id", container_id };
rc = handle_fpga_request(&mock_update_cgroups_parameters,
@@ -178,6 +188,9 @@ TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
const char* expected_cgroups_argv_2[] = { "devices", "deny", container_id, "c 246:1 rwm",
"devices", "deny", container_id, "c 246:3 rwm"};
verify_param_updated_to_cgroups(8, expected_cgroups_argv_2);
+
+ clear_cgroups_parameters_invoked();
+ free_executor_configurations();
}
TEST_F(TestFpgaModule, test_illegal_cli_parameters) {
@@ -193,6 +206,7 @@ TEST_F(TestFpgaModule, test_illegal_cli_parameters) {
ASSERT_NE(0, rc) << "Should fail.\n";
// Illegal container id - 2
+ clear_cgroups_parameters_invoked();
char* argv_1[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "0,1",
(char*) "--container_id", (char*) "container_1" };
rc = handle_fpga_request(&mock_update_cgroups_parameters,
@@ -200,10 +214,14 @@ TEST_F(TestFpgaModule, test_illegal_cli_parameters) {
ASSERT_NE(0, rc) << "Should fail.\n";
// Illegal container id - 3
+ clear_cgroups_parameters_invoked();
char* argv_2[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "0,1" };
rc = handle_fpga_request(&mock_update_cgroups_parameters,
"fpga", 3, argv_2);
ASSERT_NE(0, rc) << "Should fail.\n";
+
+ clear_cgroups_parameters_invoked();
+ free_executor_configurations();
}
TEST_F(TestFpgaModule, test_fpga_module_disabled) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
index b3d93dc..fcf8b0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
@@ -84,6 +84,13 @@ static int mock_update_cgroups_parameters(
return 0;
}
+static void clear_cgroups_parameters_invoked() {
+ for (std::vector<const char*>::size_type i = 0; i < cgroups_parameters_invoked.size(); i++) {
+ free((void *) cgroups_parameters_invoked[i]);
+ }
+ cgroups_parameters_invoked.clear();
+}
+
static void verify_param_updated_to_cgroups(
int argc, const char** argv) {
ASSERT_EQ(argc, cgroups_parameters_invoked.size());
@@ -133,6 +140,9 @@ static void test_gpu_module_enabled_disabled(int enabled) {
EXPECTED_RC = -1;
}
ASSERT_EQ(EXPECTED_RC, rc);
+
+ clear_cgroups_parameters_invoked();
+ free_executor_configurations();
}
TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
@@ -146,7 +156,7 @@ TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
container_id };
/* Test case 1: block 2 devices */
- cgroups_parameters_invoked.clear();
+ clear_cgroups_parameters_invoked();
int rc = handle_gpu_request(&mock_update_cgroups_parameters,
"gpu", 5, argv);
ASSERT_EQ(0, rc) << "Should success.\n";
@@ -157,7 +167,7 @@ TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
verify_param_updated_to_cgroups(8, expected_cgroups_argv);
/* Test case 2: block 0 devices */
- cgroups_parameters_invoked.clear();
+ clear_cgroups_parameters_invoked();
char* argv_1[] = { (char*) "--module-gpu", (char*) "--container_id", container_id };
rc = handle_gpu_request(&mock_update_cgroups_parameters,
"gpu", 3, argv_1);
@@ -167,7 +177,7 @@ TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
verify_param_updated_to_cgroups(0, NULL);
/* Test case 3: block 2 non-sequential devices */
- cgroups_parameters_invoked.clear();
+ clear_cgroups_parameters_invoked();
char* argv_2[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "1,3",
(char*) "--container_id", container_id };
rc = handle_gpu_request(&mock_update_cgroups_parameters,
@@ -178,6 +188,9 @@ TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
const char* expected_cgroups_argv_2[] = { "devices", "deny", container_id, "c 195:1 rwm",
"devices", "deny", container_id, "c 195:3 rwm"};
verify_param_updated_to_cgroups(8, expected_cgroups_argv_2);
+
+ clear_cgroups_parameters_invoked();
+ free_executor_configurations();
}
TEST_F(TestGpuModule, test_illegal_cli_parameters) {
@@ -193,6 +206,7 @@ TEST_F(TestGpuModule, test_illegal_cli_parameters) {
ASSERT_NE(0, rc) << "Should fail.\n";
// Illegal container id - 2
+ clear_cgroups_parameters_invoked();
char* argv_1[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "0,1",
(char*) "--container_id", (char*) "container_1" };
rc = handle_gpu_request(&mock_update_cgroups_parameters,
@@ -200,10 +214,14 @@ TEST_F(TestGpuModule, test_illegal_cli_parameters) {
ASSERT_NE(0, rc) << "Should fail.\n";
// Illegal container id - 3
+ clear_cgroups_parameters_invoked();
char* argv_2[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "0,1" };
rc = handle_gpu_request(&mock_update_cgroups_parameters,
"gpu", 3, argv_2);
ASSERT_NE(0, rc) << "Should fail.\n";
+
+ clear_cgroups_parameters_invoked();
+ free_executor_configurations();
}
TEST_F(TestGpuModule, test_gpu_module_disabled) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc
index 6ee0ab2..0e80fcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc
@@ -51,6 +51,7 @@ namespace ContainerExecutor {
virtual void TearDown() {
free_configuration(&new_config_format);
free_configuration(&old_config_format);
+ free_configuration(&mixed_config_format);
return;
}
@@ -84,12 +85,12 @@ namespace ContainerExecutor {
ASSERT_STREQ("/var/run/yarn", split_values[0]);
ASSERT_STREQ("/tmp/mydir", split_values[1]);
ASSERT_EQ(NULL, split_values[2]);
- free(split_values);
+ free_values(split_values);
split_values = get_configuration_values_delimiter("allowed.system.users",
"", &old_config_format, "%");
ASSERT_STREQ("nobody,daemon", split_values[0]);
ASSERT_EQ(NULL, split_values[1]);
- free(split_values);
+ free_values(split_values);
}
TEST_F(TestConfiguration, test_get_configuration_values) {
@@ -105,13 +106,13 @@ namespace ContainerExecutor {
split_values = get_configuration_values("yarn.local.dirs", "", &old_config_format);
ASSERT_STREQ("/var/run/yarn%/tmp/mydir", split_values[0]);
ASSERT_EQ(NULL, split_values[1]);
- free(split_values);
+ free_values(split_values);
split_values = get_configuration_values("allowed.system.users", "",
&old_config_format);
ASSERT_STREQ("nobody", split_values[0]);
ASSERT_STREQ("daemon", split_values[1]);
ASSERT_EQ(NULL, split_values[2]);
- free(split_values);
+ free_values(split_values);
}
TEST_F(TestConfiguration, test_get_configuration_value) {
@@ -149,21 +150,28 @@ namespace ContainerExecutor {
char *value = NULL;
value = get_section_value("yarn.nodemanager.linux-container-executor.group", executor_cfg);
ASSERT_STREQ("yarn", value);
+ free(value);
value = get_section_value("feature.docker.enabled", executor_cfg);
ASSERT_STREQ("1", value);
+ free(value);
value = get_section_value("feature.tc.enabled", executor_cfg);
ASSERT_STREQ("0", value);
+ free(value);
value = get_section_value("min.user.id", executor_cfg);
ASSERT_STREQ("1000", value);
+ free(value);
value = get_section_value("docker.binary", executor_cfg);
ASSERT_STREQ("/usr/bin/docker", value);
+ free(value);
char **list = get_section_values("allowed.system.users", executor_cfg);
ASSERT_STREQ("nobody", list[0]);
ASSERT_STREQ("daemon", list[1]);
+ free_values(list);
list = get_section_values("banned.users", executor_cfg);
ASSERT_STREQ("root", list[0]);
ASSERT_STREQ("testuser1", list[1]);
ASSERT_STREQ("testuser2", list[2]);
+ free_values(list);
}
TEST_F(TestConfiguration, test_get_section_values_delimiter) {
@@ -176,12 +184,16 @@ namespace ContainerExecutor {
free(value);
value = get_section_value("key2", section);
ASSERT_EQ(NULL, value);
+ free(value);
split_values = get_section_values_delimiter(NULL, section, "%");
ASSERT_EQ(NULL, split_values);
+ free_values(split_values);
split_values = get_section_values_delimiter("split-key", NULL, "%");
ASSERT_EQ(NULL, split_values);
+ free_values(split_values);
split_values = get_section_values_delimiter("split-key", section, NULL);
ASSERT_EQ(NULL, split_values);
+ free_values(split_values);
split_values = get_section_values_delimiter("split-key", section, "%");
ASSERT_FALSE(split_values == NULL);
ASSERT_STREQ("val1,val2,val3", split_values[0]);
@@ -192,6 +204,7 @@ namespace ContainerExecutor {
ASSERT_STREQ("perc-val1", split_values[0]);
ASSERT_STREQ("perc-val2", split_values[1]);
ASSERT_TRUE(split_values[2] == NULL);
+ free_values(split_values);
}
TEST_F(TestConfiguration, test_get_section_values) {
@@ -201,13 +214,16 @@ namespace ContainerExecutor {
section = get_configuration_section("section-1", &new_config_format);
value = get_section_value(NULL, section);
ASSERT_EQ(NULL, value);
+ free(value);
value = get_section_value("key1", NULL);
ASSERT_EQ(NULL, value);
+ free(value);
value = get_section_value("key1", section);
ASSERT_STREQ("value1", value);
free(value);
value = get_section_value("key2", section);
ASSERT_EQ(NULL, value);
+ free(value);
split_values = get_section_values("split-key", section);
ASSERT_FALSE(split_values == NULL);
ASSERT_STREQ("val1", split_values[0]);
@@ -235,14 +251,16 @@ namespace ContainerExecutor {
section = get_configuration_section("split-section", &new_config_format);
value = get_section_value(NULL, section);
ASSERT_EQ(NULL, value);
+ free(value);
value = get_section_value("key3", NULL);
ASSERT_EQ(NULL, value);
+ free(value);
value = get_section_value("key3", section);
ASSERT_STREQ("value3", value);
free(value);
value = get_section_value("key4", section);
ASSERT_STREQ("value4", value);
-
+ free(value);
}
TEST_F(TestConfiguration, test_get_configuration_section) {
@@ -343,6 +361,7 @@ namespace ContainerExecutor {
oss.str("");
oss << "value" << i;
ASSERT_STREQ(oss.str().c_str(), value);
+ free((void *) value);
}
remove(sample_file_name.c_str());
free_configuration(&cfg);
@@ -372,6 +391,7 @@ namespace ContainerExecutor {
oss.str("");
oss << "value" << i;
ASSERT_STREQ(oss.str().c_str(), value);
+ free((void *) value);
}
remove(sample_file_name.c_str());
free_configuration(&cfg);
@@ -415,18 +435,22 @@ namespace ContainerExecutor {
char *value = NULL;
value = get_section_value("key1", executor_cfg);
ASSERT_STREQ("value1", value);
+ free(value);
value = get_section_value("key2", executor_cfg);
ASSERT_STREQ("value2", value);
ASSERT_EQ(2, executor_cfg->size);
+ free(value);
executor_cfg = get_configuration_section("section-1",
&mixed_config_format);
value = get_section_value("key3", executor_cfg);
ASSERT_STREQ("value3", value);
+ free(value);
value = get_section_value("key1", executor_cfg);
ASSERT_STREQ("value4", value);
ASSERT_EQ(2, executor_cfg->size);
ASSERT_EQ(2, mixed_config_format.size);
ASSERT_STREQ("", mixed_config_format.sections[0]->name);
ASSERT_STREQ("section-1", mixed_config_format.sections[1]->name);
+ free(value);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
index 8cdbf2f..51e3d52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
@@ -51,6 +51,7 @@ namespace ContainerExecutor {
ASSERT_STREQ(oss.str().c_str(), splits[i-1]);
}
ASSERT_EQ(NULL, splits[count]);
+ free(split_string);
free_values(splits);
split_string = (char *) calloc(str.length() + 1, sizeof(char));
@@ -59,6 +60,7 @@ namespace ContainerExecutor {
ASSERT_TRUE(splits != NULL);
ASSERT_TRUE(splits[1] == NULL);
ASSERT_STREQ(str.c_str(), splits[0]);
+ free(split_string);
free_values(splits);
splits = split_delimiter(NULL, ",");
@@ -82,6 +84,7 @@ namespace ContainerExecutor {
ASSERT_STREQ(oss.str().c_str(), splits[i-1]);
}
ASSERT_EQ(NULL, splits[count]);
+ free(split_string);
free_values(splits);
str = "1,2,3,4,5,6,7,8,9,10,11";
@@ -91,6 +94,8 @@ namespace ContainerExecutor {
ASSERT_TRUE(splits != NULL);
ASSERT_TRUE(splits[1] == NULL);
ASSERT_STREQ(str.c_str(), splits[0]);
+ free(split_string);
+ free_values(splits);
return;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
index b259c6e..138e32a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
@@ -59,6 +59,7 @@
ASSERT_EQ(1, numbers[0]);
ASSERT_EQ(-1, numbers[3]);
ASSERT_EQ(0, numbers[5]);
+ free(numbers);
input = "3";
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
@@ -66,28 +67,33 @@
ASSERT_EQ(0, rc) << "Should succeeded\n";
ASSERT_EQ(1, n_numbers);
ASSERT_EQ(3, numbers[0]);
+ free(numbers);
input = "";
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_EQ(0, rc) << "Should succeeded\n";
ASSERT_EQ(0, n_numbers);
+ free(numbers);
input = ",,";
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_EQ(0, rc) << "Should succeeded\n";
ASSERT_EQ(0, n_numbers);
+ free(numbers);
input = "1,2,aa,bb";
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_TRUE(0 != rc) << "Should failed\n";
+ free(numbers);
input = "1,2,3,-12312312312312312312321311231231231";
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_TRUE(0 != rc) << "Should failed\n";
+ free(numbers);
}
TEST_F(TestStringUtils, test_validate_container_id) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d82edec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index cd671ce..007e737 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -119,23 +119,22 @@ namespace ContainerExecutor {
struct args tmp = ARGS_INITIAL_VALUE;
std::vector<std::pair<std::string, std::string> >::const_iterator itr;
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
- reset_args(&tmp);
write_command_file(itr->first);
int ret = (*docker_func)(docker_command_file.c_str(), &container_executor_cfg, &tmp);
ASSERT_EQ(0, ret) << "error message: " << get_docker_error_message(ret) << " for input " << itr->first;
char *actual = flatten(&tmp);
ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&tmp);
free(actual);
}
std::vector<std::pair<std::string, int> >::const_iterator itr2;
for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) {
- reset_args(&tmp);
write_command_file(itr2->first);
int ret = (*docker_func)(docker_command_file.c_str(), &container_executor_cfg, &tmp);
ASSERT_EQ(itr2->second, ret) << " for " << itr2->first << std::endl;
+ reset_args(&tmp);
}
- reset_args(&tmp);
int ret = (*docker_func)("unknown-file", &container_executor_cfg, &tmp);
ASSERT_EQ(static_cast<int>(INVALID_COMMAND_FILE), ret);
reset_args(&tmp);
@@ -147,7 +146,6 @@ namespace ContainerExecutor {
for(itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
struct configuration cfg;
struct args buff = ARGS_INITIAL_VALUE;
- reset_args(&buff);
write_command_file(itr->first);
int ret = read_config(docker_command_file.c_str(), &cfg);
if(ret == 0) {
@@ -155,7 +153,9 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&cfg);
}
}
}
@@ -445,7 +445,6 @@ namespace ContainerExecutor {
TEST_F(TestDockerUtil, test_set_network) {
struct configuration container_cfg;
struct args buff = ARGS_INITIAL_VALUE;
- reset_args(&buff);
int ret = 0;
std::string container_executor_cfg_contents = "[docker]\n docker.allowed.networks=sdn1,bridge";
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
@@ -464,7 +463,6 @@ namespace ContainerExecutor {
}
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
struct configuration cmd_cfg;
- reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@@ -474,7 +472,9 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&cmd_cfg);
}
struct configuration cmd_cfg_1;
write_command_file("[docker-command-execution]\n docker-command=run\n net=sdn2");
@@ -482,10 +482,11 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_network(&cmd_cfg_1, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_NETWORK, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&container_cfg);
container_executor_cfg_contents = "[docker]\n";
write_container_executor_cfg(container_executor_cfg_contents);
@@ -493,10 +494,12 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_network(&cmd_cfg_1, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_NETWORK, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg_1);
+ free_configuration(&container_cfg);
}
TEST_F(TestDockerUtil, test_set_pid_namespace) {
@@ -529,7 +532,6 @@ namespace ContainerExecutor {
FAIL();
}
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
- reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@@ -539,10 +541,11 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&cmd_cfg);
}
for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) {
- reset_args(&buff);
write_command_file(itr2->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@@ -551,7 +554,10 @@ namespace ContainerExecutor {
ret = set_pid_namespace(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
}
+ free_configuration(&container_cfg);
}
// check default case and when it's turned off
@@ -575,6 +581,7 @@ namespace ContainerExecutor {
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
free(actual);
+ free_configuration(&cmd_cfg);
}
bad_file_cmd_vec.clear();
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
@@ -584,7 +591,6 @@ namespace ContainerExecutor {
"[docker-command-execution]\n docker-command=run\n pid=host",
static_cast<int>(PID_HOST_DISABLED)));
for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) {
- reset_args(&buff);
write_command_file(itr2->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@@ -593,7 +599,10 @@ namespace ContainerExecutor {
ret = set_pid_namespace(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
}
+ free_configuration(&container_cfg);
}
}
@@ -633,6 +642,7 @@ namespace ContainerExecutor {
for (int i = 0; i < entries; ++i) {
ASSERT_STREQ(expected[i], ptr[i]);
}
+ free_values(ptr);
}
TEST_F(TestDockerUtil, test_set_privileged) {
@@ -665,7 +675,6 @@ namespace ContainerExecutor {
FAIL();
}
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
- reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@@ -674,19 +683,22 @@ namespace ContainerExecutor {
ret = set_privileged(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(6, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
}
write_command_file("[docker-command-execution]\n docker-command=run\n user=nobody\n privileged=true\n image=nothadoop/image");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_privileged(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(PRIVILEGED_CONTAINERS_DISABLED, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
+ free_configuration(&container_cfg);
}
-
// check default case and when it's turned off
for (int i = 3; i < 6; ++i) {
write_container_executor_cfg(container_executor_cfg_contents[i]);
@@ -698,7 +710,6 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n user=root\n privileged=false", ""));
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
- reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@@ -708,7 +719,9 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&cmd_cfg);
}
write_command_file("[docker-command-execution]\n docker-command=run\n user=root\n privileged=true");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
@@ -718,6 +731,9 @@ namespace ContainerExecutor {
ret = set_privileged(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(PRIVILEGED_CONTAINERS_DISABLED, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
+ free_configuration(&container_cfg);
}
}
@@ -752,7 +768,6 @@ namespace ContainerExecutor {
FAIL();
}
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
- reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@@ -762,16 +777,19 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&cmd_cfg);
}
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n cap-add=SETGID");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_capabilities(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
+ reset_args(&buff);
+ free_configuration(&container_cfg);
container_executor_cfg_contents = "[docker]\n docker.trusted.registries=hadoop\n";
write_container_executor_cfg(container_executor_cfg_contents);
@@ -779,15 +797,16 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_capabilities(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
+ free_configuration(&container_cfg);
}
TEST_F(TestDockerUtil, test_set_devices) {
struct configuration container_cfg, cmd_cfg;
struct args buff = ARGS_INITIAL_VALUE;
- reset_args(&buff);
int ret = 0;
std::string container_executor_cfg_contents = "[docker]\n"
" docker.trusted.registries=hadoop\n"
@@ -821,7 +840,6 @@ namespace ContainerExecutor {
FAIL();
}
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
- reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@@ -831,67 +849,75 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&cmd_cfg);
}
write_command_file("[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n devices=/dev/test-device:/dev/test-device");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device3:/dev/device3");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device1");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/testnvidia:/dev/testnvidia");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/gpu-nvidia-uvm:/dev/gpu-nvidia-uvm");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device1");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&container_cfg);
container_executor_cfg_contents = "[docker]\n";
write_container_executor_cfg(container_executor_cfg_contents);
@@ -899,10 +925,12 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
+ free_configuration(&container_cfg);
}
@@ -951,7 +979,6 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, std::string> >::const_iterator itr;
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
- reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@@ -961,7 +988,9 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&cmd_cfg);
}
std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
@@ -978,18 +1007,18 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> >::const_iterator itr2;
for (itr2 = bad_file_cmds_vec.begin(); itr2 != bad_file_cmds_vec.end(); ++itr2) {
- reset_args(&buff);
write_command_file(itr2->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_STREQ("", actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&cmd_cfg);
}
// verify that you can't mount any directory in the container-executor.cfg path
@@ -997,17 +1026,17 @@ namespace ContainerExecutor {
while (strlen(ce_path) != 0) {
std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=";
cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop");
- reset_args(&buff);
write_command_file(cmd_file_contents);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret) << " for input " << cmd_file_contents;
char *actual = flatten(&buff);
ASSERT_STREQ("", actual);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
free(actual);
char *tmp = strrchr(ce_path, '/');
if (tmp != NULL) {
@@ -1015,6 +1044,7 @@ namespace ContainerExecutor {
}
}
free(ce_path);
+ free_configuration(&container_cfg);
// For untrusted image, container add_rw_mounts will pass through
// without mounting or report error code.
@@ -1024,12 +1054,13 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ("", actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&container_cfg);
}
TEST_F(TestDockerUtil, test_add_ro_mounts) {
@@ -1080,7 +1111,6 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, std::string> >::const_iterator itr;
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
- reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@@ -1090,7 +1120,9 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&cmd_cfg);
}
std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
@@ -1104,19 +1136,20 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> >::const_iterator itr2;
for (itr2 = bad_file_cmds_vec.begin(); itr2 != bad_file_cmds_vec.end(); ++itr2) {
- reset_args(&buff);
write_command_file(itr2->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- reset_args(&buff);
ret = add_ro_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_STREQ("", actual);
+ reset_args(&buff);
free(actual);
+ free_configuration(&cmd_cfg);
}
+ free_configuration(&container_cfg);
container_executor_cfg_contents = "[docker]\n docker.trusted.registries=hadoop\n";
write_container_executor_cfg(container_executor_cfg_contents);
@@ -1125,10 +1158,16 @@ namespace ContainerExecutor {
FAIL();
}
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home:/home");
- reset_args(&buff);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
ret = add_ro_mounts(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_RO_MOUNT, ret);
ASSERT_EQ(0, buff.length);
+ reset_args(&buff);
+ free_configuration(&cmd_cfg);
+ free_configuration(&container_cfg);
}
TEST_F(TestDockerUtil, test_docker_run_privileged) {
@@ -1310,6 +1349,7 @@ namespace ContainerExecutor {
static_cast<int>(INVALID_DOCKER_NETWORK)));
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ free_configuration(&container_executor_cfg);
}
TEST_F(TestDockerUtil, test_docker_run_entry_point) {
@@ -1352,6 +1392,7 @@ namespace ContainerExecutor {
static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ free_configuration(&container_executor_cfg);
}
TEST_F(TestDockerUtil, test_docker_run_no_privileged) {
@@ -1441,6 +1482,7 @@ namespace ContainerExecutor {
static_cast<int>(PRIVILEGED_CONTAINERS_DISABLED)));
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ free_configuration(&container_executor_cfg);
}
}
@@ -1471,13 +1513,14 @@ namespace ContainerExecutor {
struct args buffer = ARGS_INITIAL_VALUE;
struct configuration cfg = {0, NULL};
for (itr = input_output_map.begin(); itr != input_output_map.end(); ++itr) {
- reset_args(&buffer);
write_command_file(itr->first);
int ret = get_docker_command(docker_command_file.c_str(), &cfg, &buffer);
+ char *actual = flatten(&buffer);
ASSERT_EQ(0, ret) << "for input " << itr->first;
- ASSERT_STREQ(itr->second.c_str(), flatten(&buffer));
+ ASSERT_STREQ(itr->second.c_str(), actual);
+ reset_args(&buffer);
+ free(actual);
}
- reset_args(&buffer);
}
TEST_F(TestDockerUtil, test_docker_module_enabled) {
@@ -1497,6 +1540,7 @@ namespace ContainerExecutor {
ret = docker_module_enabled(&container_executor_cfg);
ASSERT_EQ(input_out_vec[i].second, ret) << " incorrect output for "
<< input_out_vec[i].first;
+ free_configuration(&container_executor_cfg);
}
}
@@ -1544,6 +1588,7 @@ namespace ContainerExecutor {
static_cast<int>(INVALID_DOCKER_VOLUME_DRIVER)));
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_volume_command);
+ free_configuration(&container_executor_cfg);
}
TEST_F(TestDockerUtil, test_docker_no_new_privileges) {
@@ -1589,6 +1634,7 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ free_configuration(&container_executor_cfg);
}
for (int i = 2; i < 3; ++i) {
@@ -1611,6 +1657,7 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ free_configuration(&container_executor_cfg);
}
for (int i = 3; i < 5; ++i) {
@@ -1633,6 +1680,7 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ free_configuration(&container_executor_cfg);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] hadoop git commit: HADOOP-15609. Retry KMS calls when
SSLHandshakeException occurs. Contributed by Kitti Nanasi.
Posted by ey...@apache.org.
HADOOP-15609. Retry KMS calls when SSLHandshakeException occurs. Contributed by Kitti Nanasi.
(cherry picked from commit 81d59506e539673edde12e19c0df5c2edd9d02ad)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40c06b38
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40c06b38
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40c06b38
Branch: refs/remotes/origin/branch-3.1
Commit: 40c06b389a15eeef655f108b422993ef12a1fb5b
Parents: 00c476a
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Jul 24 21:45:14 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Jul 24 21:46:21 2018 -0700
----------------------------------------------------------------------
.../key/kms/LoadBalancingKMSClientProvider.java | 17 ++++-
.../kms/TestLoadBalancingKMSClientProvider.java | 79 ++++++++++++++++++++
2 files changed, 92 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40c06b38/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
index 42cd47d..9677b0d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.crypto.key.kms;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.net.ConnectException;
import java.security.GeneralSecurityException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
@@ -27,6 +28,8 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
+import javax.net.ssl.SSLHandshakeException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
@@ -115,7 +118,6 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements
if (providers.length == 0) {
throw new IOException("No providers configured !");
}
- IOException ex = null;
int numFailovers = 0;
for (int i = 0;; i++, numFailovers++) {
KMSClientProvider provider = providers[(currPos + i) % providers.length];
@@ -130,8 +132,15 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements
} catch (IOException ioe) {
LOG.warn("KMS provider at [{}] threw an IOException: ",
provider.getKMSUrl(), ioe);
- ex = ioe;
-
+ // SSLHandshakeException can occur here because of lost connection
+ // with the KMS server, creating a ConnectException from it,
+ // so that the FailoverOnNetworkExceptionRetry policy will retry
+ if (ioe instanceof SSLHandshakeException) {
+ Exception cause = ioe;
+ ioe = new ConnectException("SSLHandshakeException: "
+ + cause.getMessage());
+ ioe.initCause(cause);
+ }
RetryAction action = null;
try {
action = retryPolicy.shouldRetry(ioe, 0, numFailovers, false);
@@ -153,7 +162,7 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements
CommonConfigurationKeysPublic.
KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, providers.length),
providers.length);
- throw ex;
+ throw ioe;
}
if (((numFailovers + 1) % providers.length) == 0) {
// Sleep only after we try all the providers for every cycle.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40c06b38/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
index bd68dca..4e7aed9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.crypto.key.kms;
import static org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -26,12 +27,15 @@ import static org.mockito.Mockito.when;
import static org.mockito.Mockito.verify;
import java.io.IOException;
+import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.URI;
import java.net.UnknownHostException;
import java.security.GeneralSecurityException;
import java.security.NoSuchAlgorithmException;
+import javax.net.ssl.SSLHandshakeException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider.Options;
@@ -44,13 +48,18 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.junit.After;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.Timeout;
import org.mockito.Mockito;
import com.google.common.collect.Sets;
public class TestLoadBalancingKMSClientProvider {
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 1000);
+
@BeforeClass
public static void setup() throws IOException {
SecurityUtil.setTokenServiceUseIp(false);
@@ -638,4 +647,74 @@ public class TestLoadBalancingKMSClientProvider {
verify(p2, Mockito.times(1)).createKey(Mockito.eq("test3"),
Mockito.any(Options.class));
}
+
+ /**
+ * Tests the operation succeeds second time after SSLHandshakeException.
+ * @throws Exception
+ */
+ @Test
+ public void testClientRetriesWithSSLHandshakeExceptionSucceedsSecondTime()
+ throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(
+ CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3);
+ final String keyName = "test";
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new SSLHandshakeException("p1"))
+ .thenReturn(new KMSClientProvider.KMSKeyVersion(keyName, "v1",
+ new byte[0]));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new ConnectException("p2"));
+
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+
+ LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {p1, p2}, 0, conf);
+
+ kp.createKey(keyName, new Options(conf));
+ verify(p1, Mockito.times(2)).createKey(Mockito.eq(keyName),
+ Mockito.any(Options.class));
+ verify(p2, Mockito.times(1)).createKey(Mockito.eq(keyName),
+ Mockito.any(Options.class));
+ }
+
+ /**
+ * Tests the operation fails at every attempt after SSLHandshakeException.
+ * @throws Exception
+ */
+ @Test
+ public void testClientRetriesWithSSLHandshakeExceptionFailsAtEveryAttempt()
+ throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(
+ CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 2);
+ final String keyName = "test";
+ final String exceptionMessage = "p1 exception message";
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ Exception originalSslEx = new SSLHandshakeException(exceptionMessage);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(originalSslEx);
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new ConnectException("p2 exception message"));
+
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+
+ LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {p1, p2}, 0, conf);
+
+ Exception interceptedEx = intercept(ConnectException.class,
+ "SSLHandshakeException: " + exceptionMessage,
+ ()-> kp.createKey(keyName, new Options(conf)));
+ assertEquals(originalSslEx, interceptedEx.getCause());
+
+ verify(p1, Mockito.times(2)).createKey(Mockito.eq(keyName),
+ Mockito.any(Options.class));
+ verify(p2, Mockito.times(1)).createKey(Mockito.eq(keyName),
+ Mockito.any(Options.class));
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] hadoop git commit: YARN-8380. Support bind propagation
options for mounts in docker runtime. Contributed by Billie Rinaldi
Posted by ey...@apache.org.
YARN-8380. Support bind propagation options for mounts in docker runtime.
Contributed by Billie Rinaldi
(cherry picked from commit 8688a0c7f88f2adf1a7fce695e06f3dd1f745080)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23b8546a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23b8546a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23b8546a
Branch: refs/remotes/origin/branch-3.1
Commit: 23b8546a802fdd30d5a3382cec136d1b85e34cdf
Parents: e665c0a
Author: Eric Yang <ey...@apache.org>
Authored: Mon Jul 23 20:12:04 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Mon Jul 23 20:13:41 2018 -0400
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 37 ++-
.../linux/runtime/docker/DockerRunCommand.java | 18 +-
.../container-executor/impl/utils/docker-util.c | 196 ++++++++------
.../test/utils/test_docker_util.cc | 133 +++++-----
.../runtime/TestDockerContainerRuntime.java | 259 +++++++++----------
.../gpu/TestNvidiaDockerV1CommandPlugin.java | 2 +-
.../src/site/markdown/DockerContainers.md | 13 +-
7 files changed, 349 insertions(+), 309 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23b8546a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 5d95875..1793b2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -157,9 +157,13 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
* {@code YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS} allows users to specify
+ additional volume mounts for the Docker container. The value of the
* environment variable should be a comma-separated list of mounts.
- * All such mounts must be given as {@code source:dest:mode}, and the mode
+ * All such mounts must be given as {@code source:dest[:mode]} and the mode
* must be "ro" (read-only) or "rw" (read-write) to specify the type of
- * access being requested. The requested mounts will be validated by
+ * access being requested. If neither is specified, read-write will be
+ * assumed. The mode may include a bind propagation option. In that case,
+ * the mode should either be of the form [option], rw+[option], or
+ * ro+[option]. Valid bind propagation options are shared, rshared, slave,
+ * rslave, private, and rprivate. The requested mounts will be validated by
* container-executor based on the values set in container-executor.cfg for
* {@code docker.allowed.ro-mounts} and {@code docker.allowed.rw-mounts}.
* </li>
@@ -192,7 +196,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private static final Pattern hostnamePattern = Pattern.compile(
HOSTNAME_PATTERN);
private static final Pattern USER_MOUNT_PATTERN = Pattern.compile(
- "(?<=^|,)([^:\\x00]+):([^:\\x00]+):([a-z]+)");
+ "(?<=^|,)([^:\\x00]+):([^:\\x00]+)" +
+ "(:(r[ow]|(r[ow][+])?(r?shared|r?slave|r?private)))?(?:,|$)");
private static final int HOST_NAME_LENGTH = 64;
private static final String DEFAULT_PROCFS = "/proc";
@@ -844,24 +849,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
+ environment.get(ENV_DOCKER_CONTAINER_MOUNTS));
}
parsedMounts.reset();
+ long mountCount = 0;
while (parsedMounts.find()) {
+ mountCount++;
String src = parsedMounts.group(1);
java.nio.file.Path srcPath = java.nio.file.Paths.get(src);
if (!srcPath.isAbsolute()) {
src = mountReadOnlyPath(src, localizedResources);
}
String dst = parsedMounts.group(2);
- String mode = parsedMounts.group(3);
- if (!mode.equals("ro") && !mode.equals("rw")) {
- throw new ContainerExecutionException(
- "Invalid mount mode requested for mount: "
- + parsedMounts.group());
- }
- if (mode.equals("ro")) {
- runCommand.addReadOnlyMountLocation(src, dst);
- } else {
- runCommand.addReadWriteMountLocation(src, dst);
+ String mode = parsedMounts.group(4);
+ if (mode == null) {
+ mode = "rw";
+ } else if (!mode.startsWith("ro") && !mode.startsWith("rw")) {
+ mode = "rw+" + mode;
}
+ runCommand.addMountLocation(src, dst, mode);
+ }
+ long commaCount = environment.get(ENV_DOCKER_CONTAINER_MOUNTS).chars()
+ .filter(c -> c == ',').count();
+ if (mountCount != commaCount + 1) {
+ // this means the matcher skipped an improperly formatted mount
+ throw new ContainerExecutionException(
+ "Unable to parse some mounts in user supplied mount list: "
+ + environment.get(ENV_DOCKER_CONTAINER_MOUNTS));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23b8546a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index af16178..48ada5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -65,19 +65,15 @@ public class DockerRunCommand extends DockerCommand {
}
public DockerRunCommand addMountLocation(String sourcePath, String
- destinationPath, boolean createSource) {
- boolean sourceExists = new File(sourcePath).exists();
- if (!sourceExists && !createSource) {
- return this;
- }
- super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
+ destinationPath, String mode) {
+ super.addCommandArguments("mounts", sourcePath + ":" +
+ destinationPath + ":" + mode);
return this;
}
public DockerRunCommand addReadWriteMountLocation(String sourcePath, String
destinationPath) {
- super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
- return this;
+ return addMountLocation(sourcePath, destinationPath, "rw");
}
public DockerRunCommand addAllReadWriteMountLocations(List<String> paths) {
@@ -93,14 +89,12 @@ public class DockerRunCommand extends DockerCommand {
if (!sourceExists && !createSource) {
return this;
}
- super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
- return this;
+ return addReadOnlyMountLocation(sourcePath, destinationPath);
}
public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
destinationPath) {
- super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
- return this;
+ return addMountLocation(sourcePath, destinationPath, "ro");
}
public DockerRunCommand addAllReadOnlyMountLocations(List<String> paths) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23b8546a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index 580cd37..899c46a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -1095,105 +1095,150 @@ static char* get_mount_source(const char *mount) {
return strndup(mount, len);
}
-static int add_mounts(const struct configuration *command_config, const struct configuration *conf, const char *key,
- const int ro, args *args) {
- const char *ro_suffix = "";
+static char* get_mount_type(const char *mount) {
+ const char *tmp = strrchr(mount, ':');
+ if (tmp == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", mount);
+ return NULL;
+ }
+ if (strlen(tmp) < 2) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", mount);
+ return NULL;
+ }
+ char *mount_type = strdup(&tmp[1]);
+ if (strncmp("ro", mount_type, 2) != 0 &&
+ strncmp("rw", mount_type, 2) != 0) {
+ fprintf(ERRORFILE, "Invalid docker mount type '%s'\n", mount_type);
+ free(mount_type);
+ return NULL;
+ }
+ if (strlen(mount_type) > 2) {
+ if (strlen(mount_type) < 8 ||
+ (strcmp("shared", mount_type + 3) != 0 &&
+ strcmp("rshared", mount_type + 3) != 0 &&
+ strcmp("slave", mount_type + 3) != 0 &&
+ strcmp("rslave", mount_type + 3) != 0 &&
+ strcmp("private", mount_type + 3) != 0 &&
+ strcmp("rprivate", mount_type + 3) != 0)) {
+ fprintf(ERRORFILE, "Invalid docker mount type '%s'\n", mount_type);
+ free(mount_type);
+ return NULL;
+ }
+ mount_type[2] = ',';
+ }
+ return mount_type;
+}
+
+static int add_mounts(const struct configuration *command_config, const struct configuration *conf, args *args) {
const char *tmp_path_buffer[2] = {NULL, NULL};
char *mount_src = NULL;
+ char *mount_type = NULL;
char **permitted_ro_mounts = get_configuration_values_delimiter("docker.allowed.ro-mounts",
CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
char **permitted_rw_mounts = get_configuration_values_delimiter("docker.allowed.rw-mounts",
CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
- char **values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ char **values = get_configuration_values_delimiter("mounts", DOCKER_COMMAND_FILE_SECTION, command_config, ",");
char *config_path = get_config_path("");
const char *container_executor_cfg_path = normalize_mount(config_path, 0);
free(config_path);
int i = 0, permitted_rw = 0, permitted_ro = 0, ret = 0;
- if (ro != 0) {
- ro_suffix = ":ro";
+ if (values == NULL) {
+ goto free_and_exit;
}
- if (values != NULL) {
- // Disable mount volumes if image is not trusted.
- if (check_trusted_image(command_config, conf) != 0) {
- fprintf(ERRORFILE, "Disable mount volume for untrusted image\n");
- // YARN will implicitly bind node manager local directory to
- // docker image. This can create file system security holes,
- // if docker container has binary to escalate privileges.
- // For untrusted image, we drop mounting without reporting
- // INVALID_DOCKER_MOUNT messages to allow running untrusted
- // image in a sandbox.
- ret = 0;
+ // Disable mount volumes if image is not trusted.
+ if (check_trusted_image(command_config, conf) != 0) {
+ fprintf(ERRORFILE, "Disable mount volume for untrusted image\n");
+ // YARN will implicitly bind node manager local directory to
+ // docker image. This can create file system security holes,
+ // if docker container has binary to escalate privileges.
+ // For untrusted image, we drop mounting without reporting
+ // INVALID_DOCKER_MOUNT messages to allow running untrusted
+ // image in a sandbox.
+ ret = 0;
+ goto free_and_exit;
+ }
+ ret = normalize_mounts(permitted_ro_mounts, 1);
+ ret |= normalize_mounts(permitted_rw_mounts, 1);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Unable to find permitted docker mounts on disk\n");
+ ret = MOUNT_ACCESS_ERROR;
+ goto free_and_exit;
+ }
+ for (i = 0; values[i] != NULL; i++) {
+ mount_src = get_mount_source(values[i]);
+ if (mount_src == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", values[i]);
+ ret = INVALID_DOCKER_MOUNT;
goto free_and_exit;
}
- ret = normalize_mounts(permitted_ro_mounts, 1);
- ret |= normalize_mounts(permitted_rw_mounts, 1);
- if (ret != 0) {
- fprintf(ERRORFILE, "Unable to find permitted docker mounts on disk\n");
- ret = MOUNT_ACCESS_ERROR;
+ mount_type = get_mount_type(values[i]);
+ if (mount_type == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", values[i]);
+ ret = INVALID_DOCKER_MOUNT;
goto free_and_exit;
}
- for (i = 0; values[i] != NULL; i++) {
- mount_src = get_mount_source(values[i]);
- if (mount_src == NULL) {
- fprintf(ERRORFILE, "Invalid docker mount '%s'\n", values[i]);
- ret = INVALID_DOCKER_MOUNT;
- goto free_and_exit;
- }
- permitted_rw = check_mount_permitted((const char **) permitted_rw_mounts, mount_src);
- permitted_ro = check_mount_permitted((const char **) permitted_ro_mounts, mount_src);
- if (permitted_ro == -1 || permitted_rw == -1) {
- fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
- ret = INVALID_DOCKER_MOUNT;
- goto free_and_exit;
- }
+ permitted_rw = check_mount_permitted((const char **) permitted_rw_mounts, mount_src);
+ permitted_ro = check_mount_permitted((const char **) permitted_ro_mounts, mount_src);
+ if (permitted_ro == -1 || permitted_rw == -1) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_MOUNT;
+ goto free_and_exit;
+ }
+ if (strncmp("rw", mount_type, 2) == 0) {
// rw mount
- if (ro == 0) {
- if (permitted_rw == 0) {
- fprintf(ERRORFILE, "Invalid docker rw mount '%s', realpath=%s\n", values[i], mount_src);
+ if (permitted_rw == 0) {
+ fprintf(ERRORFILE, "Invalid docker rw mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_RW_MOUNT;
+ goto free_and_exit;
+ } else {
+ // determine if the user can modify the container-executor.cfg file
+ tmp_path_buffer[0] = normalize_mount(mount_src, 0);
+ // just re-use the function, flip the args to check if the container-executor path is in the requested
+ // mount point
+ ret = check_mount_permitted(tmp_path_buffer, container_executor_cfg_path);
+ free((void *) tmp_path_buffer[0]);
+ if (ret == 1) {
+ fprintf(ERRORFILE, "Attempting to mount a parent directory '%s' of container-executor.cfg as read-write\n",
+ values[i]);
ret = INVALID_DOCKER_RW_MOUNT;
goto free_and_exit;
- } else {
- // determine if the user can modify the container-executor.cfg file
- tmp_path_buffer[0] = normalize_mount(mount_src, 0);
- // just re-use the function, flip the args to check if the container-executor path is in the requested
- // mount point
- ret = check_mount_permitted(tmp_path_buffer, container_executor_cfg_path);
- free((void *) tmp_path_buffer[0]);
- if (ret == 1) {
- fprintf(ERRORFILE, "Attempting to mount a parent directory '%s' of container-executor.cfg as read-write\n",
- values[i]);
- ret = INVALID_DOCKER_RW_MOUNT;
- goto free_and_exit;
- }
}
}
- //ro mount
- if (ro != 0 && permitted_ro == 0 && permitted_rw == 0) {
+ } else {
+ // ro mount
+ if (permitted_ro == 0 && permitted_rw == 0) {
fprintf(ERRORFILE, "Invalid docker ro mount '%s', realpath=%s\n", values[i], mount_src);
ret = INVALID_DOCKER_RO_MOUNT;
goto free_and_exit;
}
+ }
- ret = add_to_args(args, "-v");
- if (ret != 0) {
- ret = BUFFER_TOO_SMALL;
- goto free_and_exit;
- }
+ if (strlen(mount_type) > 2) {
+ // overwrite separator between read mode and propagation option with ','
+ int mount_type_index = strlen(values[i]) - strlen(mount_type);
+ values[i][mount_type_index + 2] = ',';
+ }
- char *tmp_buffer = make_string("%s%s", values[i], (char *) ro_suffix);
- ret = add_to_args(args, tmp_buffer);
- free(tmp_buffer);
- if (ret != 0) {
- ret = BUFFER_TOO_SMALL;
- goto free_and_exit;
- }
- free(mount_src);
- mount_src = NULL;
+ ret = add_to_args(args, "-v");
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+
+ ret = add_to_args(args, values[i]);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
}
+ free(mount_src);
+ free(mount_type);
+ mount_src = NULL;
+ mount_type = NULL;
}
free_and_exit:
free(mount_src);
+ free(mount_type);
free_values(permitted_ro_mounts);
free_values(permitted_rw_mounts);
free_values(values);
@@ -1201,14 +1246,6 @@ free_and_exit:
return ret;
}
-static int add_ro_mounts(const struct configuration *command_config, const struct configuration *conf, args *args) {
- return add_mounts(command_config, conf, "ro-mounts", 1, args);
-}
-
-static int add_rw_mounts(const struct configuration *command_config, const struct configuration *conf, args *args) {
- return add_mounts(command_config, conf, "rw-mounts", 0, args);
-}
-
static int check_privileges(const char *user) {
int ngroups = 0;
gid_t *groups = NULL;
@@ -1427,12 +1464,7 @@ int get_docker_run_command(const char *command_file, const struct configuration
goto free_and_exit;
}
- ret = add_ro_mounts(&command_config, conf, args);
- if (ret != 0) {
- goto free_and_exit;
- }
-
- ret = add_rw_mounts(&command_config, conf, args);
+ ret = add_mounts(&command_config, conf, args);
if (ret != 0) {
goto free_and_exit;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23b8546a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index 007e737..7e18146 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -934,36 +934,42 @@ namespace ContainerExecutor {
}
- TEST_F(TestDockerUtil, test_add_rw_mounts) {
+ TEST_F(TestDockerUtil, test_add_mounts) {
struct configuration container_cfg, cmd_cfg;
struct args buff = ARGS_INITIAL_VALUE;
int ret = 0;
std::string container_executor_cfg_contents = "[docker]\n docker.trusted.registries=hadoop\n "
- "docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut\n "
+ "docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut,/usr/bin/awk\n "
"docker.allowed.ro-mounts=/etc/passwd";
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/var:/var", "-v /var:/var"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/var:/var:rw", "-v /var:/var:rw"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n rw-mounts=/var:/var", ""));
+ "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n mounts=/var:/var:rw", ""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/var/:/var/", "-v /var/:/var/"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/var/:/var/:rw", "-v /var/:/var/:rw"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/usr/bin/cut:/usr/bin/cut",
- "-v /usr/bin/cut:/usr/bin/cut"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/usr/bin/cut:/usr/bin/cut:rw",
+ "-v /usr/bin/cut:/usr/bin/cut:rw"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n rw-mounts=/lib:/lib",
+ "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n mounts=/lib:/lib:rw",
""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2",
- "-v /opt:/mydisk1 -v /var/log/:/mydisk2"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/opt:/mydisk1:rw,/var/log/:/mydisk2:rw",
+ "-v /opt:/mydisk1:rw -v /var/log/:/mydisk2:rw"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2",
+ "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n mounts=/opt:/mydisk1:rw,/var/log/:/mydisk2:rw",
""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n image=hadoop/image\n", ""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n", ""));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/usr/bin/awk:/awk:rw+shared,/etc/passwd:/etc/passwd:ro",
+ "-v /usr/bin/awk:/awk:rw,shared -v /etc/passwd:/etc/passwd:ro"));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/var:/var:ro+rprivate,/etc/passwd:/etc/passwd:ro+rshared",
+ "-v /var:/var:ro,rprivate -v /etc/passwd:/etc/passwd:ro,rshared"));
write_container_executor_cfg(container_executor_cfg_contents);
ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
if (ret != 0) {
@@ -984,7 +990,7 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
@@ -995,13 +1001,22 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/lib:/lib",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/lib:/lib:rw",
static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/usr/bin/:/usr/bin",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/usr/bin/:/usr/bin:rw",
static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/blah:/blah",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/blah:/blah:rw",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/tmp:/tmp:shared",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/lib:/lib",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/lib:/lib:other",
static_cast<int>(INVALID_DOCKER_MOUNT)));
std::vector<std::pair<std::string, int> >::const_iterator itr2;
@@ -1012,7 +1027,7 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_STREQ("", actual);
@@ -1024,14 +1039,14 @@ namespace ContainerExecutor {
// verify that you can't mount any directory in the container-executor.cfg path
char *ce_path = realpath("../etc/hadoop/container-executor.cfg", NULL);
while (strlen(ce_path) != 0) {
- std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=";
- cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop");
+ std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=";
+ cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop").append(":rw");
write_command_file(cmd_file_contents);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret) << " for input " << cmd_file_contents;
char *actual = flatten(&buff);
ASSERT_STREQ("", actual);
@@ -1046,7 +1061,7 @@ namespace ContainerExecutor {
free(ce_path);
free_configuration(&container_cfg);
- // For untrusted image, container add_rw_mounts will pass through
+ // For untrusted image, container add_mounts will pass through
// without mounting or report error code.
container_executor_cfg_contents = "[docker]\n";
write_container_executor_cfg(container_executor_cfg_contents);
@@ -1054,7 +1069,7 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ("", actual);
@@ -1073,26 +1088,26 @@ namespace ContainerExecutor {
"docker.allowed.ro-mounts=/etc/passwd,/etc/group";
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n ro-mounts=/var:/var", ""));
+ "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n mounts=/var:/var:ro", ""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n ro-mounts=/etc:/etc", ""));
+ "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n mounts=/etc:/etc:ro", ""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/var/:/var/", "-v /var/:/var/:ro"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/var/:/var/:ro", "-v /var/:/var/:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home:/home", "-v /home:/home:ro"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/home:/home:ro", "-v /home:/home:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home/:/home", "-v /home/:/home:ro"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/home/:/home:ro", "-v /home/:/home:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/usr/bin/cut:/usr/bin/cut",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/usr/bin/cut:/usr/bin/cut:ro",
"-v /usr/bin/cut:/usr/bin/cut:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc/group:/etc/group",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/etc/group:/etc/group:ro",
"-v /etc/group:/etc/group:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc/passwd:/etc/passwd",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/etc/passwd:/etc/passwd:ro",
"-v /etc/passwd:/etc/passwd:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/var/log:/mydisk1,/etc/passwd:/etc/passwd",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/var/log:/mydisk1:ro,/etc/passwd:/etc/passwd:ro",
"-v /var/log:/mydisk1:ro -v /etc/passwd:/etc/passwd:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n image=hadoop/image\n", ""));
@@ -1116,7 +1131,7 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- ret = add_ro_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
@@ -1127,10 +1142,10 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc:/etc",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/etc:/etc:ro",
static_cast<int>(INVALID_DOCKER_RO_MOUNT)));
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/blah:/blah",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/blah:/blah:ro",
static_cast<int>(INVALID_DOCKER_MOUNT)));
std::vector<std::pair<std::string, int> >::const_iterator itr2;
@@ -1141,7 +1156,7 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- ret = add_ro_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_STREQ("", actual);
@@ -1157,12 +1172,12 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home:/home");
+ write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/home:/home:ro");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- ret = add_ro_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_RO_MOUNT, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
@@ -1203,18 +1218,18 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN"
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN"
" --cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test hadoop/docker-image bash "
"test_script.sh arg1 arg2"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1225,18 +1240,18 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm --net=bridge -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN "
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN "
"--cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test hadoop/docker-image bash"
" test_script.sh arg1 arg2"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1247,24 +1262,24 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=root\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 -d --rm --net=bridge -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --privileged --cap-drop=ALL "
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --privileged --cap-drop=ALL "
"--cap-add=CHOWN --cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test hadoop/docker-image "
"bash test_script.sh arg1 arg2"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=root\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 -d --rm --net=bridge -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --privileged --cap-drop=ALL "
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --privileged --cap-drop=ALL "
"--cap-add=CHOWN --cap-add=SETUID --hostname=host-id "
"--device=/dev/test:/dev/test hadoop/docker-image bash test_script.sh arg1 arg2"));
@@ -1292,7 +1307,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1302,7 +1317,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/var/log:/var/log\n"
+ " mounts=/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/var/log:/var/log:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1312,7 +1327,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/bin:/bin,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/bin:/bin:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1322,7 +1337,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID,SETGID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1332,7 +1347,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/dev1:/dev/dev1\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1342,7 +1357,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n privileged=true\n net=host\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1432,18 +1447,18 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN"
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN"
" --cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test hadoop/docker-image bash "
"test_script.sh arg1 arg2"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1453,18 +1468,18 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm --net=bridge -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN "
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN "
"--cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test hadoop/docker-image bash"
" test_script.sh arg1 arg2"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1475,7 +1490,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23b8546a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 1d5c58d..0803660 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -389,7 +389,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -406,17 +406,16 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -440,7 +439,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(14, dockerCommands.size());
+ Assert.assertEquals(13, dockerCommands.size());
int counter = 0;
Assert.assertEquals("[docker-command-execution]",
dockerCommands.get(counter++));
@@ -456,18 +455,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert
.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -555,7 +553,7 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -574,18 +572,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert
.assertEquals(" net=" + allowedNetwork, dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -615,7 +612,7 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -634,18 +631,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert
.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -685,7 +681,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -706,16 +702,16 @@ public class TestDockerContainerRuntime {
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
Assert.assertEquals(
- " name=container_e11_1518975676334_14532816_01_000001",
- dockerCommands.get(counter++));
- Assert.assertEquals(" net=sdn1", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
+ " mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
dockerCommands.get(counter++));
Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
+ " name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
+ Assert.assertEquals(" net=sdn1", dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -754,18 +750,16 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
-
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=sdn2", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -803,7 +797,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -854,7 +848,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -871,18 +865,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
Assert.assertEquals(" pid=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -907,7 +900,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -1016,7 +1009,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1031,18 +1024,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
Assert.assertEquals(" privileged=true", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + submittingUser,
dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
@@ -1103,7 +1095,7 @@ public class TestDockerContainerRuntime {
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
- "source");
+ "/source");
try {
runtime.launchContainer(builder.build());
@@ -1133,7 +1125,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1150,19 +1142,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/test_local_dir/test_resource_file:test_mount:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(
- " ro-mounts=/test_filecache_dir:/test_filecache_dir,/"
- + "test_user_filecache_dir:/test_user_filecache_dir,"
- + "/test_local_dir/test_resource_file:test_mount",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -1189,7 +1179,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1206,20 +1196,18 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/test_local_dir/test_resource_file:test_mount1:ro,"
+ + "/test_local_dir/test_resource_file:test_mount2:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(
- " ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir,"
- + "/test_local_dir/test_resource_file:test_mount1,"
- + "/test_local_dir/test_resource_file:test_mount2",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -1235,7 +1223,8 @@ public class TestDockerContainerRuntime {
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
- "/tmp/foo:/tmp/foo:ro,/tmp/bar:/tmp/bar:rw");
+ "/tmp/foo:/tmp/foo:ro,/tmp/bar:/tmp/bar:rw,/tmp/baz:/tmp/baz," +
+ "/a:/a:shared,/b:/b:ro+shared,/c:/c:rw+rshared,/d:/d:private");
runtime.launchContainer(builder.build());
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
@@ -1245,7 +1234,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1262,19 +1251,19 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/tmp/foo:/tmp/foo:ro,"
+ + "/tmp/bar:/tmp/bar:rw,/tmp/baz:/tmp/baz:rw,/a:/a:rw+shared,"
+ + "/b:/b:ro+shared,/c:/c:rw+rshared,/d:/d:rw+private",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir,"
- + "/tmp/foo:/tmp/foo",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir,"
- + "/tmp/bar:/tmp/bar",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -1288,7 +1277,7 @@ public class TestDockerContainerRuntime {
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
- "source:target");
+ "/source:target:ro,/source:target:other,/source:target:rw");
try {
runtime.launchContainer(builder.build());
@@ -1306,7 +1295,7 @@ public class TestDockerContainerRuntime {
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
- "source:target:other");
+ "/source:target:other");
try {
runtime.launchContainer(builder.build());
@@ -1324,7 +1313,7 @@ public class TestDockerContainerRuntime {
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
- "s\0ource:target:ro");
+ "/s\0ource:target:ro");
try {
runtime.launchContainer(builder.build());
@@ -1352,7 +1341,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1369,18 +1358,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/tmp/foo:/tmp/foo:ro,/tmp/bar:/tmp/bar:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir,"
- + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -1420,7 +1408,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1437,18 +1425,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/tmp/foo:/tmp/foo:rw,/tmp/bar:/tmp/bar:rw",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir,"
- + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -2005,7 +1992,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -2022,18 +2009,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/source/path:/destination/path:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir,"
- + "/source/path:/destination/path",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
// Verify volume-driver is set to expected value.
@@ -2146,7 +2132,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -2165,17 +2151,16 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter++));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23b8546a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestNvidiaDockerV1CommandPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestNvidiaDockerV1CommandPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestNvidiaDockerV1CommandPlugin.java
index 7057847..c8b2eaf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestNvidiaDockerV1CommandPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestNvidiaDockerV1CommandPlugin.java
@@ -196,7 +196,7 @@ public class TestNvidiaDockerV1CommandPlugin {
// Volume driver should not be included by final commandline
Assert.assertFalse(newCommandLine.containsKey("volume-driver"));
Assert.assertTrue(newCommandLine.containsKey("devices"));
- Assert.assertTrue(newCommandLine.containsKey("ro-mounts"));
+ Assert.assertTrue(newCommandLine.containsKey("mounts"));
/* Test get docker volume command */
commandPlugin = new MyNvidiaDockerV1CommandPlugin(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23b8546a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index a2ef6fe..e35c906 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -304,7 +304,7 @@ environment variables in the application's environment:
| `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK` | Sets the network type to be used by the Docker container. It must be a valid value as determined by the yarn.nodemanager.runtime.linux.docker.allowed-container-networks property. |
| `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_PID_NAMESPACE` | Controls which PID namespace will be used by the Docker container. By default, each Docker container has its own PID namespace. To share the namespace of the host, the yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed property must be set to true. If the host PID namespace is allowed and this environment variable is set to host, the Docker container will share the host's PID namespace. No other value is allowed. |
| `YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER` | Controls whether the Docker container is a privileged container. In order to use privileged containers, the yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed property must be set to true, and the application owner must appear in the value of the yarn.nodemanager.runtime.linux.docker.privileged-containers.acl property. If this environment variable is set to true, a privileged Docker container will be used if allowed. No other value is allowed, so the environment variable should be left unset rather than setting it to false. |
-| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest:mode" and the mode must be "ro" (read-only) or "rw" (read-write) to specify the type of access being requested. The requested mounts will be validated by container-executor based on the values set in container-executor.cfg for docker.allowed.ro-mounts and docker.allowed.rw-mounts. |
+| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as `source:dest[:mode]` and the mode must be "ro" (read-only) or "rw" (read-write) to specify the type of access being requested. If neither is specified, read-write will be assumed. The mode may include a bind propagation option. In that case, the mode should either be of the form `[option]`, `rw+[option]`, or `ro+[option]`. Valid bind propagation options are shared, rshared, slave, rslave, private, and rprivate. The requested mounts will be validated by container-executor based on the values set in container-executor.cfg for `docker.allowed.ro-mounts` and `docker.allowed.rw-mounts`. |
| `YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL` | Allows a user to request delayed deletion of the Docker container on a per container basis. If true, Docker containers will not be removed until the duration defined by yarn.nodemanager.delete.debug-delay-sec has elapsed. Administrators can disable this feature through the yarn-site property yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed. This feature is disabled by default. When this feature is disabled or set to false, the container will be removed as soon as it exits. |
The first two are required. The remainder can be set as needed. While
@@ -347,10 +347,13 @@ supplied by the user must either match or be a child of the specified
directory.
The user supplied mount list is defined as a comma separated list in the form
-*source*:*destination*:*mode*. The source is the file or directory on the host.
-The destination is the path within the contatiner where the source will be bind
-mounted. The mode defines the mode the user expects for the mount, which can be
-ro (read-only) or rw (read-write).
+*source*:*destination* or *source*:*destination*:*mode*. The source is the file
+or directory on the host. The destination is the path within the container
+where the source will be bind mounted. The mode defines the mode the user
+expects for the mount, which can be ro (read-only) or rw (read-write). If not
+specified, rw is assumed. The mode may also include a bind propagation option
+ (shared, rshared, slave, rslave, private, or rprivate). In that case, the
+ mode should be of the form *option*, rw+*option*, or ro+*option*.
The following example outlines how to use this feature to mount the commonly
needed /sys/fs/cgroup directory into the container running on YARN.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] hadoop git commit: YARN-8506. Make
GetApplicationsRequestPBImpl thread safe. (wangda)
Posted by ey...@apache.org.
YARN-8506. Make GetApplicationsRequestPBImpl thread safe. (wangda)
Change-Id: I2c006965375823c83036e7f45f7163d13c0bdf90
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ad82ea5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ad82ea5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ad82ea5
Branch: refs/remotes/origin/branch-3.1
Commit: 8ad82ea5b5b0c87e5771dcaa5530d4fbe03c92ae
Parents: 242b5ac
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Jul 9 11:35:15 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Jul 9 11:35:15 2018 -0700
----------------------------------------------------------------------
.../impl/pb/GetApplicationsRequestPBImpl.java | 44 ++++++++++----------
1 file changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ad82ea5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
index ad009d6..bc6be80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
@@ -65,7 +65,7 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
viaProto = true;
}
- public GetApplicationsRequestProto getProto() {
+ public synchronized GetApplicationsRequestProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
@@ -175,13 +175,13 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public Set<String> getApplicationTypes() {
+ public synchronized Set<String> getApplicationTypes() {
initApplicationTypes();
return this.applicationTypes;
}
@Override
- public void setApplicationTypes(Set<String> applicationTypes) {
+ public synchronized void setApplicationTypes(Set<String> applicationTypes) {
maybeInitBuilder();
if (applicationTypes == null)
builder.clearApplicationTypes();
@@ -198,13 +198,13 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public Set<String> getApplicationTags() {
+ public synchronized Set<String> getApplicationTags() {
initApplicationTags();
return this.applicationTags;
}
@Override
- public void setApplicationTags(Set<String> tags) {
+ public synchronized void setApplicationTags(Set<String> tags) {
maybeInitBuilder();
if (tags == null || tags.isEmpty()) {
builder.clearApplicationTags();
@@ -219,7 +219,7 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public EnumSet<YarnApplicationState> getApplicationStates() {
+ public synchronized EnumSet<YarnApplicationState> getApplicationStates() {
initApplicationStates();
return this.applicationStates;
}
@@ -233,12 +233,12 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public ApplicationsRequestScope getScope() {
+ public synchronized ApplicationsRequestScope getScope() {
initScope();
return this.scope;
}
- public void setScope(ApplicationsRequestScope scope) {
+ public synchronized void setScope(ApplicationsRequestScope scope) {
maybeInitBuilder();
if (scope == null) {
builder.clearScope();
@@ -247,7 +247,7 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public void setApplicationStates(EnumSet<YarnApplicationState> applicationStates) {
+ public synchronized void setApplicationStates(EnumSet<YarnApplicationState> applicationStates) {
maybeInitBuilder();
if (applicationStates == null) {
builder.clearApplicationStates();
@@ -256,7 +256,7 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public void setApplicationStates(Set<String> applicationStates) {
+ public synchronized void setApplicationStates(Set<String> applicationStates) {
EnumSet<YarnApplicationState> appStates = null;
for (YarnApplicationState state : YarnApplicationState.values()) {
if (applicationStates.contains(
@@ -272,12 +272,12 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public Set<String> getUsers() {
+ public synchronized Set<String> getUsers() {
initUsers();
return this.users;
}
- public void setUsers(Set<String> users) {
+ public synchronized void setUsers(Set<String> users) {
maybeInitBuilder();
if (users == null) {
builder.clearUsers();
@@ -286,13 +286,13 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public Set<String> getQueues() {
+ public synchronized Set<String> getQueues() {
initQueues();
return this.queues;
}
@Override
- public void setQueues(Set<String> queues) {
+ public synchronized void setQueues(Set<String> queues) {
maybeInitBuilder();
if (queues == null) {
builder.clearQueues();
@@ -301,7 +301,7 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public long getLimit() {
+ public synchronized long getLimit() {
if (this.limit == Long.MAX_VALUE) {
GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder;
this.limit = p.hasLimit() ? p.getLimit() : Long.MAX_VALUE;
@@ -310,13 +310,13 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public void setLimit(long limit) {
+ public synchronized void setLimit(long limit) {
maybeInitBuilder();
this.limit = limit;
}
@Override
- public LongRange getStartRange() {
+ public synchronized LongRange getStartRange() {
if (this.start == null) {
GetApplicationsRequestProtoOrBuilder p = viaProto ? proto: builder;
if (p.hasStartBegin() || p.hasStartEnd()) {
@@ -329,12 +329,12 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public void setStartRange(LongRange range) {
+ public synchronized void setStartRange(LongRange range) {
this.start = range;
}
@Override
- public void setStartRange(long begin, long end)
+ public synchronized void setStartRange(long begin, long end)
throws IllegalArgumentException {
if (begin > end) {
throw new IllegalArgumentException("begin > end in range (begin, " +
@@ -344,7 +344,7 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public LongRange getFinishRange() {
+ public synchronized LongRange getFinishRange() {
if (this.finish == null) {
GetApplicationsRequestProtoOrBuilder p = viaProto ? proto: builder;
if (p.hasFinishBegin() || p.hasFinishEnd()) {
@@ -357,12 +357,12 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
}
@Override
- public void setFinishRange(LongRange range) {
+ public synchronized void setFinishRange(LongRange range) {
this.finish = range;
}
@Override
- public void setFinishRange(long begin, long end) {
+ public synchronized void setFinishRange(long begin, long end) {
if (begin > end) {
throw new IllegalArgumentException("begin > end in range (begin, " +
"end): (" + begin + ", " + end + ")");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] hadoop git commit: YARN-8546. Resource leak caused by a
reserved container being released more than once under async scheduling.
Contributed by Tao Yang.
Posted by ey...@apache.org.
YARN-8546. Resource leak caused by a reserved container being released more than once under async scheduling. Contributed by Tao Yang.
(Cherry-picked from commit 5be9f4a5d05c9cb99348719fe35626b1de3055db)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b89624a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b89624a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b89624a9
Branch: refs/remotes/origin/branch-3.1
Commit: b89624a943268e180e0e1532b3a394ff580a962c
Parents: 1396fa2
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Jul 25 17:35:27 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Jul 25 17:53:40 2018 +0800
----------------------------------------------------------------------
.../scheduler/common/fica/FiCaSchedulerApp.java | 15 ++++
.../TestCapacitySchedulerAsyncScheduling.java | 89 ++++++++++++++++++++
2 files changed, 104 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b89624a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 3b1b82c..9810e98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -361,6 +361,21 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
.isEmpty()) {
for (SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>
releaseContainer : allocation.getToRelease()) {
+ // Make sure to-release reserved containers are not outdated
+ if (releaseContainer.getRmContainer().getState()
+ == RMContainerState.RESERVED
+ && releaseContainer.getRmContainer() != releaseContainer
+ .getSchedulerNode().getReservedContainer()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to accept this proposal because "
+ + "it tries to release an outdated reserved container "
+ + releaseContainer.getRmContainer().getContainerId()
+ + " on node " + releaseContainer.getSchedulerNode().getNodeID()
+ + " whose reserved container is "
+ + releaseContainer.getSchedulerNode().getReservedContainer());
+ }
+ return false;
+ }
// Only consider non-reserved container (reserved container will
// not affect available resource of node) on the same node
if (releaseContainer.getRmContainer().getState()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b89624a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 338b9f9..c2c1519 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -685,6 +687,93 @@ public class TestCapacitySchedulerAsyncScheduling {
rm.stop();
}
+
+ @Test(timeout = 60000)
+ public void testReleaseOutdatedReservedContainer() throws Exception {
+ /*
+ * Submit a application, reserved container_02 on nm1,
+ * submit two allocate proposals which contain the same reserved
+ * container_02 as to-released container.
+ * First proposal should be accepted, second proposal should be rejected
+ * because it try to release an outdated reserved container
+ */
+ MockRM rm1 = new MockRM();
+ rm1.getRMContext().setNodeLabelManager(mgr);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+ MockNM nm3 = rm1.registerNode("h3:1234", 8 * GB);
+ rm1.drainEvents();
+
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ LeafQueue defaultQueue = (LeafQueue) cs.getQueue("default");
+ SchedulerNode sn1 = cs.getSchedulerNode(nm1.getNodeId());
+ SchedulerNode sn2 = cs.getSchedulerNode(nm2.getNodeId());
+ SchedulerNode sn3 = cs.getSchedulerNode(nm3.getNodeId());
+
+ // launch another app to queue, AM container should be launched in nm1
+ RMApp app1 = rm1.submitApp(4 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ Resource allocateResource = Resources.createResource(5 * GB);
+ am1.allocate("*", (int) allocateResource.getMemorySize(), 3, 0,
+ new ArrayList<ContainerId>(), "");
+ FiCaSchedulerApp schedulerApp1 =
+ cs.getApplicationAttempt(am1.getApplicationAttemptId());
+
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ Assert.assertEquals(1, schedulerApp1.getReservedContainers().size());
+ Assert.assertEquals(9 * GB,
+ defaultQueue.getQueueResourceUsage().getUsed().getMemorySize());
+
+ RMContainer reservedContainer =
+ schedulerApp1.getReservedContainers().get(0);
+ ResourceCommitRequest allocateFromSameReservedContainerProposal1 =
+ createAllocateFromReservedProposal(3, allocateResource, schedulerApp1,
+ sn2, sn1, cs.getRMContext(), reservedContainer);
+ boolean tryCommitResult = cs.tryCommit(cs.getClusterResource(),
+ allocateFromSameReservedContainerProposal1, true);
+ Assert.assertTrue(tryCommitResult);
+ ResourceCommitRequest allocateFromSameReservedContainerProposal2 =
+ createAllocateFromReservedProposal(4, allocateResource, schedulerApp1,
+ sn3, sn1, cs.getRMContext(), reservedContainer);
+ tryCommitResult = cs.tryCommit(cs.getClusterResource(),
+ allocateFromSameReservedContainerProposal2, true);
+ Assert.assertFalse("This proposal should be rejected because "
+ + "it try to release an outdated reserved container", tryCommitResult);
+
+ rm1.close();
+ }
+
+ private ResourceCommitRequest createAllocateFromReservedProposal(
+ int containerId, Resource allocateResource, FiCaSchedulerApp schedulerApp,
+ SchedulerNode allocateNode, SchedulerNode reservedNode,
+ RMContext rmContext, RMContainer reservedContainer) {
+ Container container = Container.newInstance(
+ ContainerId.newContainerId(schedulerApp.getApplicationAttemptId(), containerId),
+ allocateNode.getNodeID(), allocateNode.getHttpAddress(), allocateResource,
+ Priority.newInstance(0), null);
+ RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey
+ .create(ResourceRequest
+ .newInstance(Priority.newInstance(0), "*", allocateResource, 1)),
+ schedulerApp.getApplicationAttemptId(), allocateNode.getNodeID(), "user",
+ rmContext);
+ SchedulerContainer allocateContainer =
+ new SchedulerContainer(schedulerApp, allocateNode, rmContainer, "", true);
+ SchedulerContainer reservedSchedulerContainer =
+ new SchedulerContainer(schedulerApp, reservedNode, reservedContainer, "",
+ false);
+ List<SchedulerContainer> toRelease = new ArrayList<>();
+ toRelease.add(reservedSchedulerContainer);
+ ContainerAllocationProposal allocateFromReservedProposal =
+ new ContainerAllocationProposal(allocateContainer, toRelease, null,
+ NodeType.OFF_SWITCH, NodeType.OFF_SWITCH,
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, allocateResource);
+ List<ContainerAllocationProposal> allocateProposals = new ArrayList<>();
+ allocateProposals.add(allocateFromReservedProposal);
+ return new ResourceCommitRequest(allocateProposals, null, null);
+ }
+
private void keepNMHeartbeat(List<MockNM> mockNMs, int interval) {
if (nmHeartbeatThread != null) {
nmHeartbeatThread.setShouldStop();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] hadoop git commit: MAPREDUCE-7118. Distributed cache
conflicts breaks backwards compatability. (Jason Lowe via wangda)
Posted by ey...@apache.org.
MAPREDUCE-7118. Distributed cache conflicts breaks backwards compatability. (Jason Lowe via wangda)
Change-Id: I89ab4852b4ad305fec19812e8931c59d96581376
(cherry picked from commit b3b4d4ccb53fdf8dacc66e912822b34f8b3bf215)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23624c92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23624c92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23624c92
Branch: refs/remotes/origin/branch-3.1
Commit: 23624c9248b01794cae88f299bf97735ed09b6ce
Parents: 749fff5
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Jul 19 12:03:24 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Jul 19 14:26:40 2018 -0700
----------------------------------------------------------------------
.../mapreduce/v2/util/LocalResourceBuilder.java | 8 +++-----
.../hadoop/mapreduce/v2/util/TestMRApps.java | 20 ++++++++++++++++++--
2 files changed, 21 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23624c92/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java
index 48b157e..48cc29e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.InvalidJobConfException;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -144,10 +143,9 @@ class LocalResourceBuilder {
LocalResource orig = localResources.get(linkName);
if(orig != null && !orig.getResource().equals(URL.fromURI(p.toUri()))) {
- throw new InvalidJobConfException(
- getResourceDescription(orig.getType()) + orig.getResource()
- +
- " conflicts with " + getResourceDescription(type) + u);
+ LOG.warn(getResourceDescription(orig.getType()) + orig.getResource()
+ + " conflicts with " + getResourceDescription(type) + u);
+ continue;
}
Boolean sharedCachePolicy = sharedCacheUploadPolicies.get(u.toString());
sharedCachePolicy =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23624c92/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
index 3aadd63..c6a2874 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
@@ -360,7 +360,7 @@ public class TestMRApps {
}
@SuppressWarnings("deprecation")
- @Test(timeout = 120000, expected = InvalidJobConfException.class)
+ @Test(timeout = 120000)
public void testSetupDistributedCacheConflicts() throws Exception {
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
@@ -388,10 +388,18 @@ public class TestMRApps {
Map<String, LocalResource> localResources =
new HashMap<String, LocalResource>();
MRApps.setupDistributedCache(conf, localResources);
+
+ assertEquals(1, localResources.size());
+ LocalResource lr = localResources.get("something");
+ //Archive wins
+ assertNotNull(lr);
+ assertEquals(10l, lr.getSize());
+ assertEquals(10l, lr.getTimestamp());
+ assertEquals(LocalResourceType.ARCHIVE, lr.getType());
}
@SuppressWarnings("deprecation")
- @Test(timeout = 120000, expected = InvalidJobConfException.class)
+ @Test(timeout = 120000)
public void testSetupDistributedCacheConflictsFiles() throws Exception {
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
@@ -416,6 +424,14 @@ public class TestMRApps {
Map<String, LocalResource> localResources =
new HashMap<String, LocalResource>();
MRApps.setupDistributedCache(conf, localResources);
+
+ assertEquals(1, localResources.size());
+ LocalResource lr = localResources.get("something");
+ //First one wins
+ assertNotNull(lr);
+ assertEquals(10l, lr.getSize());
+ assertEquals(10l, lr.getTimestamp());
+ assertEquals(LocalResourceType.FILE, lr.getType());
}
@SuppressWarnings("deprecation")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] hadoop git commit: YARN-4606. CapacityScheduler: applications
could get starved because computation of #activeUsers considers pending apps.
Contributed by Manikandan R
Posted by ey...@apache.org.
YARN-4606. CapacityScheduler: applications could get starved because computation of #activeUsers considers pending apps. Contributed by Manikandan R
(cherry picked from commit 9485c9aee6e9bb935c3e6ae4da81d70b621781de)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/830ef12a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/830ef12a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/830ef12a
Branch: refs/remotes/origin/branch-3.1
Commit: 830ef12af830de5d54e51d2b1d16c56f5eb78e43
Parents: d2212c2
Author: Eric E Payne <er...@oath.com>
Authored: Wed Jul 25 16:22:04 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Wed Jul 25 16:30:30 2018 +0000
----------------------------------------------------------------------
.../scheduler/capacity/UsersManager.java | 27 +++-
.../capacity/TestCapacityScheduler.java | 128 +++++++++++++++++++
.../capacity/TestContainerAllocation.java | 43 +++++++
3 files changed, 197 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/830ef12a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index 747a488..83ee6c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -85,6 +85,7 @@ public class UsersManager implements AbstractUsersManager {
private final QueueMetrics metrics;
private AtomicInteger activeUsers = new AtomicInteger(0);
+ private AtomicInteger activeUsersWithOnlyPendingApps = new AtomicInteger(0);
private Map<String, Set<ApplicationId>> usersApplications =
new HashMap<String, Set<ApplicationId>>();
@@ -671,9 +672,23 @@ public class UsersManager implements AbstractUsersManager {
// update in local storage
userLimitPerSchedulingMode.put(schedulingMode, computedUserLimit);
+ computeNumActiveUsersWithOnlyPendingApps();
+
return userLimitPerSchedulingMode;
}
+ // This method is called within the lock.
+ private void computeNumActiveUsersWithOnlyPendingApps() {
+ int numPendingUsers = 0;
+ for (User user : users.values()) {
+ if ((user.getPendingApplications() > 0)
+ && (user.getActiveApplications() <= 0)) {
+ numPendingUsers++;
+ }
+ }
+ activeUsersWithOnlyPendingApps = new AtomicInteger(numPendingUsers);
+ }
+
private Resource computeUserLimit(String userName, Resource clusterResource,
String nodePartition, SchedulingMode schedulingMode, boolean activeUser) {
Resource partitionResource = labelManager.getResourceByLabel(nodePartition,
@@ -839,6 +854,11 @@ public class UsersManager implements AbstractUsersManager {
try {
this.writeLock.lock();
+ User userDesc = getUser(user);
+ if (userDesc != null && userDesc.getActiveApplications() <= 0) {
+ return;
+ }
+
Set<ApplicationId> userApps = usersApplications.get(user);
if (userApps == null) {
userApps = new HashSet<ApplicationId>();
@@ -893,7 +913,7 @@ public class UsersManager implements AbstractUsersManager {
@Override
public int getNumActiveUsers() {
- return activeUsers.get();
+ return activeUsers.get() + activeUsersWithOnlyPendingApps.get();
}
float sumActiveUsersTimesWeights() {
@@ -1090,4 +1110,9 @@ public class UsersManager implements AbstractUsersManager {
this.writeLock.unlock();
}
}
+
+ @VisibleForTesting
+ public int getNumActiveUsersWithOnlyPendingApps() {
+ return activeUsersWithOnlyPendingApps.get();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/830ef12a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 79cdcfe..8d948b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4978,4 +4978,132 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
Assert.assertEquals(AllocationState.QUEUE_SKIPPED,
ContainerAllocation.QUEUE_SKIPPED.getAllocationState());
}
+
+ @Test
+ public void testMoveAppWithActiveUsersWithOnlyPendingApps() throws Exception {
+
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ CapacitySchedulerConfiguration newConf =
+ new CapacitySchedulerConfiguration(conf);
+
+ // Define top-level queues
+ newConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[] { "a", "b" });
+
+ newConf.setCapacity(A, 50);
+ newConf.setCapacity(B, 50);
+
+ // Define 2nd-level queues
+ newConf.setQueues(A, new String[] { "a1" });
+ newConf.setCapacity(A1, 100);
+ newConf.setUserLimitFactor(A1, 2.0f);
+ newConf.setMaximumAMResourcePercentPerPartition(A1, "", 0.1f);
+
+ newConf.setQueues(B, new String[] { "b1" });
+ newConf.setCapacity(B1, 100);
+ newConf.setUserLimitFactor(B1, 2.0f);
+
+ LOG.info("Setup top-level queues a and b");
+
+ MockRM rm = new MockRM(newConf);
+ rm.start();
+
+ CapacityScheduler scheduler =
+ (CapacityScheduler) rm.getResourceScheduler();
+
+ MockNM nm1 = rm.registerNode("h1:1234", 16 * GB);
+
+ // submit an app
+ RMApp app = rm.submitApp(GB, "test-move-1", "u1", null, "a1");
+ MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1);
+
+ ApplicationAttemptId appAttemptId =
+ rm.getApplicationReport(app.getApplicationId())
+ .getCurrentApplicationAttemptId();
+
+ RMApp app2 = rm.submitApp(1 * GB, "app", "u2", null, "a1");
+ MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
+
+ RMApp app3 = rm.submitApp(1 * GB, "app", "u3", null, "a1");
+
+ RMApp app4 = rm.submitApp(1 * GB, "app", "u4", null, "a1");
+
+ // Each application asks 50 * 1GB containers
+ am1.allocate("*", 1 * GB, 50, null);
+ am2.allocate("*", 1 * GB, 50, null);
+
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+ // check preconditions
+ List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
+ assertEquals(4, appsInA1.size());
+ String queue =
+ scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue()
+ .getQueueName();
+ Assert.assertEquals("a1", queue);
+
+ List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
+ assertTrue(appsInA.contains(appAttemptId));
+ assertEquals(4, appsInA.size());
+
+ List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
+ assertTrue(appsInRoot.contains(appAttemptId));
+ assertEquals(4, appsInRoot.size());
+
+ List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
+ assertTrue(appsInB1.isEmpty());
+
+ List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
+ assertTrue(appsInB.isEmpty());
+
+ UsersManager um =
+ (UsersManager) scheduler.getQueue("a1").getAbstractUsersManager();
+
+ assertEquals(4, um.getNumActiveUsers());
+ assertEquals(2, um.getNumActiveUsersWithOnlyPendingApps());
+
+ // now move the app
+ scheduler.moveAllApps("a1", "b1");
+
+ //Triggering this event so that user limit computation can
+ //happen again
+ for (int i = 0; i < 10; i++) {
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ Thread.sleep(500);
+ }
+
+ // check postconditions
+ appsInB1 = scheduler.getAppsInQueue("b1");
+
+ assertEquals(4, appsInB1.size());
+ queue =
+ scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue()
+ .getQueueName();
+ Assert.assertEquals("b1", queue);
+
+ appsInB = scheduler.getAppsInQueue("b");
+ assertTrue(appsInB.contains(appAttemptId));
+ assertEquals(4, appsInB.size());
+
+ appsInRoot = scheduler.getAppsInQueue("root");
+ assertTrue(appsInRoot.contains(appAttemptId));
+ assertEquals(4, appsInRoot.size());
+
+ List<ApplicationAttemptId> oldAppsInA1 = scheduler.getAppsInQueue("a1");
+ assertEquals(0, oldAppsInA1.size());
+
+ UsersManager um_b1 =
+ (UsersManager) scheduler.getQueue("b1").getAbstractUsersManager();
+
+ assertEquals(2, um_b1.getNumActiveUsers());
+ assertEquals(2, um_b1.getNumActiveUsersWithOnlyPendingApps());
+
+ appsInB1 = scheduler.getAppsInQueue("b1");
+ assertEquals(4, appsInB1.size());
+ rm.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/830ef12a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 25e535a..b9bfc2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -941,4 +941,47 @@ public class TestContainerAllocation {
rm1.close();
}
+
+ @Test
+ public void testActiveUsersWithOnlyPendingApps() throws Exception {
+
+ CapacitySchedulerConfiguration newConf =
+ new CapacitySchedulerConfiguration(conf);
+ newConf.setMaximumAMResourcePercentPerPartition(
+ CapacitySchedulerConfiguration.ROOT + ".default", "", 0.2f);
+ MockRM rm1 = new MockRM(newConf);
+
+ rm1.getRMContext().setNodeLabelManager(mgr);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "u1", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ RMApp app2 = rm1.submitApp(1 * GB, "app", "u2", null, "default");
+ MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+ RMApp app3 = rm1.submitApp(1 * GB, "app", "u3", null, "default");
+
+ RMApp app4 = rm1.submitApp(1 * GB, "app", "u4", null, "default");
+
+ // Each application asks 50 * 1GB containers
+ am1.allocate("*", 1 * GB, 50, null);
+ am2.allocate("*", 1 * GB, 50, null);
+
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+ for (int i = 0; i < 10; i++) {
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ Thread.sleep(1000);
+ }
+ LeafQueue lq = (LeafQueue) cs.getQueue("default");
+ UsersManager um = (UsersManager) lq.getAbstractUsersManager();
+
+ Assert.assertEquals(4, um.getNumActiveUsers());
+ Assert.assertEquals(2, um.getNumActiveUsersWithOnlyPendingApps());
+ Assert.assertEquals(2, lq.getMetrics().getAppsPending());
+ rm1.close();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] hadoop git commit: YARN-8421: when moving app,
activeUsers is increased,
even though app does not have outstanding request. Contributed by Kyungwan Nam
Posted by ey...@apache.org.
YARN-8421: when moving app, activeUsers is increased, even though app does not have outstanding request. Contributed by Kyungwan Nam
(cherry picked from commit 937ef39b3ff90f72392b7a319e4346344db34e03)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a79e893
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a79e893
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a79e893
Branch: refs/remotes/origin/branch-3.1
Commit: 9a79e893f74ab97571d156b4f39a3b751aad240f
Parents: 677bbdc
Author: Eric E Payne <er...@oath.com>
Authored: Mon Jul 16 16:24:21 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Mon Jul 16 16:32:05 2018 +0000
----------------------------------------------------------------------
.../scheduler/AppSchedulingInfo.java | 4 +-
.../TestSchedulerApplicationAttempt.java | 44 ++++++++++++++++++++
2 files changed, 47 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a79e893/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 1efdd8b..8074f06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -578,7 +578,9 @@ public class AppSchedulingInfo {
newMetrics.moveAppTo(this);
abstractUsersManager.deactivateApplication(user, applicationId);
abstractUsersManager = newQueue.getAbstractUsersManager();
- abstractUsersManager.activateApplication(user, applicationId);
+ if (!schedulerKeys.isEmpty()) {
+ abstractUsersManager.activateApplication(user, applicationId);
+ }
this.queue = newQueue;
} finally {
this.writeLock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a79e893/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
index 17f9d23..c110b1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
@@ -58,6 +58,50 @@ public class TestSchedulerApplicationAttempt {
QueueMetrics.clearQueueMetrics();
DefaultMetricsSystem.shutdown();
}
+
+ @Test
+ public void testActiveUsersWhenMove() {
+ final String user = "user1";
+ Queue parentQueue = createQueue("parent", null);
+ Queue queue1 = createQueue("queue1", parentQueue);
+ Queue queue2 = createQueue("queue2", parentQueue);
+ Queue queue3 = createQueue("queue3", parentQueue);
+
+ ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getEpoch()).thenReturn(3L);
+ SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId,
+ user, queue1, queue1.getAbstractUsersManager(), rmContext);
+
+ // Resource request
+ Resource requestedResource = Resource.newInstance(1536, 2);
+ Priority requestedPriority = Priority.newInstance(2);
+ ResourceRequest request = ResourceRequest.newInstance(requestedPriority,
+ ResourceRequest.ANY, requestedResource, 1);
+ app.updateResourceRequests(Arrays.asList(request));
+
+ assertEquals(1, queue1.getAbstractUsersManager().getNumActiveUsers());
+ // move app from queue1 to queue2
+ app.move(queue2);
+ // Active user count has to decrease from queue1
+ assertEquals(0, queue1.getAbstractUsersManager().getNumActiveUsers());
+ // Increase the active user count in queue2 if the moved app has pending requests
+ assertEquals(1, queue2.getAbstractUsersManager().getNumActiveUsers());
+
+ // Allocated container
+ RMContainer container1 = createRMContainer(appAttId, 1, requestedResource);
+ app.liveContainers.put(container1.getContainerId(), container1);
+ SchedulerNode node = createNode();
+ app.appSchedulingInfo.allocate(NodeType.OFF_SWITCH, node,
+ toSchedulerKey(requestedPriority), container1.getContainer());
+
+ // Active user count has to decrease from queue2 due to app has NO pending requests
+ assertEquals(0, queue2.getAbstractUsersManager().getNumActiveUsers());
+ // move app from queue2 to queue3
+ app.move(queue3);
+ // Active user count in queue3 stays same if the moved app has NO pending requests
+ assertEquals(0, queue3.getAbstractUsersManager().getNumActiveUsers());
+ }
@Test
public void testMove() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] hadoop git commit: Fix potential FSImage corruption.
Contributed by Ekanth Sethuramalingam & Arpit Agarwal.
Posted by ey...@apache.org.
Fix potential FSImage corruption. Contributed by Ekanth Sethuramalingam & Arpit Agarwal.
(cherry picked from commit 0a1e922f3d8eca4e852be57124ec1bcafaadb289)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53c7d82d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53c7d82d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53c7d82d
Branch: refs/remotes/origin/branch-3.1
Commit: 53c7d82d539f1a4afcb37ebeaaa0a1a7c25fe942
Parents: 34f1dd0
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Mon Jul 16 18:20:24 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Mon Jul 16 18:29:43 2018 -0700
----------------------------------------------------------------------
.../server/namenode/AclEntryStatusFormat.java | 6 +-
.../namenode/INodeWithAdditionalFields.java | 4 +-
.../hdfs/server/namenode/XAttrFormat.java | 67 +++++++++++++-------
3 files changed, 49 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53c7d82d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
index 82aa214..2c5b23b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
@@ -38,7 +38,8 @@ import com.google.common.collect.ImmutableList;
* [1:3) -- the type of the entry (AclEntryType) <br>
* [3:6) -- the permission of the entry (FsAction) <br>
* [6:7) -- A flag to indicate whether Named entry or not <br>
- * [7:32) -- the name of the entry, which is an ID that points to a <br>
+ * [7:8) -- Reserved <br>
+ * [8:32) -- the name of the entry, which is an ID that points to a <br>
* string in the StringTableSection. <br>
*/
public enum AclEntryStatusFormat {
@@ -47,7 +48,8 @@ public enum AclEntryStatusFormat {
TYPE(SCOPE.BITS, 2),
PERMISSION(TYPE.BITS, 3),
NAMED_ENTRY_CHECK(PERMISSION.BITS, 1),
- NAME(NAMED_ENTRY_CHECK.BITS, 25);
+ RESERVED(NAMED_ENTRY_CHECK.BITS, 1),
+ NAME(RESERVED.BITS, 24);
private final LongBitFormat BITS;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53c7d82d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
index 9adcc3e..84d99e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
@@ -35,8 +35,8 @@ public abstract class INodeWithAdditionalFields extends INode
implements LinkedElement {
enum PermissionStatusFormat {
MODE(null, 16),
- GROUP(MODE.BITS, 25),
- USER(GROUP.BITS, 23);
+ GROUP(MODE.BITS, 24),
+ USER(GROUP.BITS, 24);
final LongBitFormat BITS;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53c7d82d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
index 7e704d0..f9f06db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
@@ -27,25 +27,56 @@ import org.apache.hadoop.hdfs.XAttrHelper;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Ints;
+import org.apache.hadoop.hdfs.util.LongBitFormat;
/**
* Class to pack XAttrs into byte[].<br>
* For each XAttr:<br>
* The first 4 bytes represents XAttr namespace and name<br>
* [0:3) - XAttr namespace<br>
- * [3:32) - The name of the entry, which is an ID that points to a
+ * [3:8) - Reserved<br>
+ * [8:32) - The name of the entry, which is an ID that points to a
* string in map<br>
* The following two bytes represents the length of XAttr value<br>
* The remaining bytes is the XAttr value<br>
*/
class XAttrFormat {
- private static final int XATTR_NAMESPACE_MASK = (1 << 3) - 1;
- private static final int XATTR_NAMESPACE_OFFSET = 29;
- private static final int XATTR_NAME_MASK = (1 << 29) - 1;
- private static final int XATTR_NAME_ID_MAX = 1 << 29;
+ private enum XAttrStatusFormat {
+
+ NAMESPACE(null, 3),
+ RESERVED(NAMESPACE.BITS, 5),
+ NAME(RESERVED.BITS, 24);
+
+ private final LongBitFormat BITS;
+
+ XAttrStatusFormat(LongBitFormat previous, int length) {
+ BITS = new LongBitFormat(name(), previous, length, 0);
+ }
+
+ static XAttr.NameSpace getNamespace(int xattrStatus) {
+ int ordinal = (int) NAMESPACE.BITS.retrieve(xattrStatus);
+ return XAttr.NameSpace.values()[ordinal];
+ }
+
+ static String getName(int xattrStatus) {
+ int id = (int) NAME.BITS.retrieve(xattrStatus);
+ return XAttrStorage.getName(id);
+ }
+
+ static int toInt(XAttr.NameSpace namespace, String name) {
+ long xattrStatusInt = 0;
+
+ xattrStatusInt = NAMESPACE.BITS
+ .combine(namespace.ordinal(), xattrStatusInt);
+ int nid = XAttrStorage.getNameSerialNumber(name);
+ xattrStatusInt = NAME.BITS
+ .combine(nid, xattrStatusInt);
+
+ return (int) xattrStatusInt;
+ }
+ }
+
private static final int XATTR_VALUE_LEN_MAX = 1 << 16;
- private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES =
- XAttr.NameSpace.values();
/**
* Unpack byte[] to XAttrs.
@@ -64,10 +95,8 @@ class XAttrFormat {
int v = Ints.fromBytes(attrs[i], attrs[i + 1],
attrs[i + 2], attrs[i + 3]);
i += 4;
- int ns = (v >> XATTR_NAMESPACE_OFFSET) & XATTR_NAMESPACE_MASK;
- int nid = v & XATTR_NAME_MASK;
- builder.setNameSpace(XATTR_NAMESPACE_VALUES[ns]);
- builder.setName(XAttrStorage.getName(nid));
+ builder.setNameSpace(XAttrStatusFormat.getNamespace(v));
+ builder.setName(XAttrStatusFormat.getName(v));
int vlen = ((0xff & attrs[i]) << 8) | (0xff & attrs[i + 1]);
i += 2;
if (vlen > 0) {
@@ -100,10 +129,8 @@ class XAttrFormat {
int v = Ints.fromBytes(attrs[i], attrs[i + 1],
attrs[i + 2], attrs[i + 3]);
i += 4;
- int ns = (v >> XATTR_NAMESPACE_OFFSET) & XATTR_NAMESPACE_MASK;
- int nid = v & XATTR_NAME_MASK;
- XAttr.NameSpace namespace = XATTR_NAMESPACE_VALUES[ns];
- String name = XAttrStorage.getName(nid);
+ XAttr.NameSpace namespace = XAttrStatusFormat.getNamespace(v);
+ String name = XAttrStatusFormat.getName(v);
int vlen = ((0xff & attrs[i]) << 8) | (0xff & attrs[i + 1]);
i += 2;
if (xAttr.getNameSpace() == namespace &&
@@ -134,15 +161,7 @@ class XAttrFormat {
ByteArrayOutputStream out = new ByteArrayOutputStream();
try {
for (XAttr a : xAttrs) {
- int nsOrd = a.getNameSpace().ordinal();
- Preconditions.checkArgument(nsOrd < 8, "Too many namespaces.");
- int nid = XAttrStorage.getNameSerialNumber(a.getName());
- Preconditions.checkArgument(nid < XATTR_NAME_ID_MAX,
- "Too large serial number of the xattr name");
-
- // big-endian
- int v = ((nsOrd & XATTR_NAMESPACE_MASK) << XATTR_NAMESPACE_OFFSET)
- | (nid & XATTR_NAME_MASK);
+ int v = XAttrStatusFormat.toInt(a.getNameSpace(), a.getName());
out.write(Ints.toByteArray(v));
int vlen = a.getValue() == null ? 0 : a.getValue().length;
Preconditions.checkArgument(vlen < XATTR_VALUE_LEN_MAX,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] hadoop git commit: Disable mounting cgroups by default
(miklos.szegedi@cloudera.com via rkanter)
Posted by ey...@apache.org.
Disable mounting cgroups by default (miklos.szegedi@cloudera.com via rkanter)
(cherry picked from commit 351cf87c92872d90f62c476f85ae4d02e485769c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27e2b4b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27e2b4b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27e2b4b3
Branch: refs/remotes/origin/branch-3.1
Commit: 27e2b4b36456ea5f42d38329dcc6bee0cb7b7ac0
Parents: d82edec
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Jun 7 17:09:34 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Jul 18 16:07:48 2018 -0700
----------------------------------------------------------------------
.../impl/container-executor.c | 54 ++++++++++++++------
.../impl/container-executor.h | 4 ++
.../main/native/container-executor/impl/main.c | 19 ++++---
.../src/site/markdown/NodeManagerCgroups.md | 2 +-
4 files changed, 55 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27e2b4b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 1b8842a..baf0e8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -73,6 +73,7 @@ static const char* DEFAULT_BANNED_USERS[] = {"yarn", "mapred", "hdfs", "bin", 0}
static const int DEFAULT_DOCKER_SUPPORT_ENABLED = 0;
static const int DEFAULT_TC_SUPPORT_ENABLED = 0;
+static const int DEFAULT_MOUNT_CGROUP_SUPPORT_ENABLED = 0;
static const char* PROC_PATH = "/proc";
@@ -482,6 +483,12 @@ int is_tc_support_enabled() {
DEFAULT_TC_SUPPORT_ENABLED, &executor_cfg);
}
+int is_mount_cgroups_support_enabled() {
+ return is_feature_enabled(MOUNT_CGROUP_SUPPORT_ENABLED_KEY,
+ DEFAULT_MOUNT_CGROUP_SUPPORT_ENABLED,
+ &executor_cfg);
+}
+
/**
* Utility function to concatenate argB to argA using the concat_pattern.
*/
@@ -2346,20 +2353,25 @@ void chown_dir_contents(const char *dir_path, uid_t uid, gid_t gid) {
DIR *dp;
struct dirent *ep;
- char *path_tmp = malloc(strlen(dir_path) + NAME_MAX + 2);
+ size_t len = strlen(dir_path) + NAME_MAX + 2;
+ char *path_tmp = malloc(len);
if (path_tmp == NULL) {
return;
}
- char *buf = stpncpy(path_tmp, dir_path, strlen(dir_path));
- *buf++ = '/';
-
dp = opendir(dir_path);
if (dp != NULL) {
while ((ep = readdir(dp)) != NULL) {
- stpncpy(buf, ep->d_name, strlen(ep->d_name));
- buf[strlen(ep->d_name)] = '\0';
- change_owner(path_tmp, uid, gid);
+ if (strcmp(ep->d_name, ".") != 0 &&
+ strcmp(ep->d_name, "..") != 0 &&
+ strstr(ep->d_name, "..") == NULL) {
+ int result = snprintf(path_tmp, len, "%s/%s", dir_path, ep->d_name);
+ if (result > 0 && result < len) {
+ change_owner(path_tmp, uid, gid);
+ } else {
+ fprintf(LOGFILE, "Ignored %s/%s due to length", dir_path, ep->d_name);
+ }
+ }
}
closedir(dp);
}
@@ -2383,11 +2395,16 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
char *mount_path = malloc(len);
char hier_path[EXECUTOR_PATH_MAX];
int result = 0;
- struct stat sb;
if (controller == NULL || mount_path == NULL) {
fprintf(LOGFILE, "Failed to mount cgroup controller; not enough memory\n");
result = OUT_OF_MEMORY;
+ goto cleanup;
+ }
+ if (hierarchy == NULL || strstr(hierarchy, "..") != NULL) {
+ fprintf(LOGFILE, "Unsupported cgroup hierarhy path detected.\n");
+ result = INVALID_COMMAND_PROVIDED;
+ goto cleanup;
}
if (get_kv_key(pair, controller, len) < 0 ||
get_kv_value(pair, mount_path, len) < 0) {
@@ -2395,13 +2412,10 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
pair);
result = -1;
} else {
- if (stat(mount_path, &sb) != 0) {
- // Create mount point, if it does not exist
- const mode_t mount_perms = S_IRWXU | S_IRGRP | S_IXGRP;
- if (mkdirs(mount_path, mount_perms) == 0) {
- fprintf(LOGFILE, "Failed to create cgroup mount point %s at %s\n",
- controller, mount_path);
- }
+ if (strstr(mount_path, "..") != NULL) {
+ fprintf(LOGFILE, "Unsupported cgroup mount path detected.\n");
+ result = INVALID_COMMAND_PROVIDED;
+ goto cleanup;
}
if (mount("none", mount_path, "cgroup", 0, controller) == 0) {
char *buf = stpncpy(hier_path, mount_path, strlen(mount_path));
@@ -2410,13 +2424,20 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
// create hierarchy as 0750 and chown to Hadoop NM user
const mode_t perms = S_IRWXU | S_IRGRP | S_IXGRP;
+ struct stat sb;
+ if (stat(hier_path, &sb) == 0 &&
+ (sb.st_uid != nm_uid || sb.st_gid != nm_gid)) {
+ fprintf(LOGFILE, "cgroup hierarchy %s already owned by another user %d\n", hier_path, sb.st_uid);
+ result = INVALID_COMMAND_PROVIDED;
+ goto cleanup;
+ }
if (mkdirs(hier_path, perms) == 0) {
change_owner(hier_path, nm_uid, nm_gid);
chown_dir_contents(hier_path, nm_uid, nm_gid);
}
} else {
fprintf(LOGFILE, "Failed to mount cgroup controller %s at %s - %s\n",
- controller, mount_path, strerror(errno));
+ controller, mount_path, strerror(errno));
// if controller is already mounted, don't stop trying to mount others
if (errno != EBUSY) {
result = -1;
@@ -2424,6 +2445,7 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
}
}
+cleanup:
free(controller);
free(mount_path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27e2b4b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 9136606..32e953d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -64,6 +64,7 @@ enum operations {
#define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users"
#define DOCKER_SUPPORT_ENABLED_KEY "feature.docker.enabled"
#define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled"
+#define MOUNT_CGROUP_SUPPORT_ENABLED_KEY "feature.mount-cgroup.enabled"
#define TMP_DIR "tmp"
extern struct passwd *user_detail;
@@ -238,6 +239,9 @@ int is_feature_enabled(const char* feature_key, int default_value,
/** Check if tc (traffic control) support is enabled in configuration. */
int is_tc_support_enabled();
+/** Check if cgroup mount support is enabled in configuration. */
+int is_mount_cgroups_support_enabled();
+
/**
* Run a batch of tc commands that modify interface configuration
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27e2b4b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 1ed3ce8..76fa39f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -270,14 +270,19 @@ static int validate_arguments(int argc, char **argv , int *operation) {
}
if (strcmp("--mount-cgroups", argv[1]) == 0) {
- if (argc < 4) {
- display_usage(stdout);
- return INVALID_ARGUMENT_NUMBER;
+ if (is_mount_cgroups_support_enabled()) {
+ if (argc < 4) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ optind++;
+ cmd_input.cgroups_hierarchy = argv[optind++];
+ *operation = MOUNT_CGROUPS;
+ return 0;
+ } else {
+ display_feature_disabled_message("mount cgroup");
+ return FEATURE_DISABLED;
}
- optind++;
- cmd_input.cgroups_hierarchy = argv[optind++];
- *operation = MOUNT_CGROUPS;
- return 0;
}
if (strcmp("--tc-modify-state", argv[1]) == 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27e2b4b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
index d362801..4a83dce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
@@ -50,7 +50,7 @@ YARN uses CGroups through a directory structure mounted into the file system by
| Option | Description |
|:---- |:---- |
| Discover CGroups mounted already | This should be used on newer systems like RHEL7 or Ubuntu16 or if the administrator mounts CGroups before YARN starts. Set `yarn.nodemanager.linux-container-executor.cgroups.mount` to false and leave other settings set to their defaults. YARN will locate the mount points in `/proc/mounts`. Common locations include `/sys/fs/cgroup` and `/cgroup`. The default location can vary depending on the Linux distribution in use.|
-| CGroups mounted by YARN | If the system does not have CGroups mounted or it is mounted to an inaccessible location then point `yarn.nodemanager.linux-container-executor.cgroups.mount-path` to an empty directory. Set `yarn.nodemanager.linux-container-executor.cgroups.mount` to true. A point to note here is that the container-executor binary will try to create and mount each subsystem as a subdirectory under this path. If `cpu` is already mounted somewhere with `cpuacct`, then the directory `cpu,cpuacct` will be created for the hierarchy.|
+| CGroups mounted by YARN | IMPORTANT: This option is deprecated due to security reasons with the `container-executor.cfg` option `feature.mount-cgroup.enabled=0` by default. Please mount cgroups before launching YARN.|
| CGroups mounted already or linked but not in `/proc/mounts` | If cgroups is accessible through lxcfs or simulated by another filesystem, then point `yarn.nodemanager.linux-container-executor.cgroups.mount-path` to your CGroups root directory. Set `yarn.nodemanager.linux-container-executor.cgroups.mount` to false. YARN tries to use this path first, before any CGroup mount point discovery. The path should have a subdirectory for each CGroup hierarchy named by the comma separated CGroup subsystems supported like `<path>/cpu,cpuacct`. Valid subsystem names are `cpu, cpuacct, cpuset, memory, net_cls, blkio, freezer, devices`.|
CGroups and security
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] hadoop git commit: YARN-8360. Improve YARN service restart
policy and node manager auto restart policy. Contributed by Suma Shivaprasad
Posted by ey...@apache.org.
YARN-8360. Improve YARN service restart policy and node manager auto restart policy.
Contributed by Suma Shivaprasad
(cherry picked from commit 84d7bf1eeff6b9418361afa4aa713e5e6f771365)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e665c0a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e665c0a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e665c0a9
Branch: refs/remotes/origin/branch-3.1
Commit: e665c0a9dd29248ac0cc22ec5a0c830ada13df60
Parents: 4f2a129
Author: Eric Yang <ey...@apache.org>
Authored: Mon Jul 23 12:57:01 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Mon Jul 23 12:59:37 2018 -0400
----------------------------------------------------------------------
.../service/component/AlwaysRestartPolicy.java | 5 ++
.../component/ComponentRestartPolicy.java | 2 +
.../service/component/NeverRestartPolicy.java | 5 ++
.../component/OnFailureRestartPolicy.java | 5 ++
.../provider/AbstractProviderService.java | 29 +++++----
.../hadoop/yarn/service/ServiceTestUtils.java | 2 +-
.../containerlaunch/TestAbstractLauncher.java | 66 ++++++++++++++++++++
7 files changed, 101 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e665c0a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/AlwaysRestartPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/AlwaysRestartPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/AlwaysRestartPolicy.java
index 704ab14..505120d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/AlwaysRestartPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/AlwaysRestartPolicy.java
@@ -79,4 +79,9 @@ public final class AlwaysRestartPolicy implements ComponentRestartPolicy {
@Override public boolean shouldTerminate(Component component) {
return false;
}
+
+ @Override public boolean allowContainerRetriesForInstance(
+ ComponentInstance componentInstance) {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e665c0a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentRestartPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentRestartPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentRestartPolicy.java
index 23b0fb9..c5adffe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentRestartPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentRestartPolicy.java
@@ -42,4 +42,6 @@ public interface ComponentRestartPolicy {
boolean shouldTerminate(Component component);
+ boolean allowContainerRetriesForInstance(ComponentInstance componentInstance);
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e665c0a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/NeverRestartPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/NeverRestartPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/NeverRestartPolicy.java
index ace1f89..cd44a58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/NeverRestartPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/NeverRestartPolicy.java
@@ -79,4 +79,9 @@ public final class NeverRestartPolicy implements ComponentRestartPolicy {
}
return true;
}
+
+ @Override public boolean allowContainerRetriesForInstance(
+ ComponentInstance componentInstance) {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e665c0a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/OnFailureRestartPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/OnFailureRestartPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/OnFailureRestartPolicy.java
index 39fba2a..b939ba0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/OnFailureRestartPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/OnFailureRestartPolicy.java
@@ -84,4 +84,9 @@ public final class OnFailureRestartPolicy implements ComponentRestartPolicy {
}
return true;
}
+
+ @Override public boolean allowContainerRetriesForInstance(
+ ComponentInstance componentInstance) {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e665c0a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
index 6d213c8..46e3a7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.ComponentRestartPolicy;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
@@ -116,18 +117,22 @@ public abstract class AbstractProviderService implements ProviderService,
public void buildContainerRetry(AbstractLauncher launcher,
Configuration yarnConf,
- ContainerLaunchService.ComponentLaunchContext compLaunchContext) {
+ ContainerLaunchService.ComponentLaunchContext compLaunchContext,
+ ComponentInstance instance) {
// By default retry forever every 30 seconds
- launcher.setRetryContext(
- YarnServiceConf.getInt(CONTAINER_RETRY_MAX,
- DEFAULT_CONTAINER_RETRY_MAX,
- compLaunchContext.getConfiguration(), yarnConf),
- YarnServiceConf.getInt(CONTAINER_RETRY_INTERVAL,
- DEFAULT_CONTAINER_RETRY_INTERVAL,
- compLaunchContext.getConfiguration(), yarnConf),
- YarnServiceConf.getLong(CONTAINER_FAILURES_VALIDITY_INTERVAL,
- DEFAULT_CONTAINER_FAILURES_VALIDITY_INTERVAL,
- compLaunchContext.getConfiguration(), yarnConf));
+
+ ComponentRestartPolicy restartPolicy = instance.getComponent()
+ .getRestartPolicyHandler();
+ if (restartPolicy.allowContainerRetriesForInstance(instance)) {
+ launcher.setRetryContext(YarnServiceConf
+ .getInt(CONTAINER_RETRY_MAX, DEFAULT_CONTAINER_RETRY_MAX,
+ compLaunchContext.getConfiguration(), yarnConf), YarnServiceConf
+ .getInt(CONTAINER_RETRY_INTERVAL, DEFAULT_CONTAINER_RETRY_INTERVAL,
+ compLaunchContext.getConfiguration(), yarnConf), YarnServiceConf
+ .getLong(CONTAINER_FAILURES_VALIDITY_INTERVAL,
+ DEFAULT_CONTAINER_FAILURES_VALIDITY_INTERVAL,
+ compLaunchContext.getConfiguration(), yarnConf));
+ }
}
public void buildContainerLaunchContext(AbstractLauncher launcher,
@@ -161,6 +166,6 @@ public abstract class AbstractProviderService implements ProviderService,
yarnConf, container, compLaunchContext, tokensForSubstitution);
// Setup container retry settings
- buildContainerRetry(launcher, yarnConf, compLaunchContext);
+ buildContainerRetry(launcher, yarnConf, compLaunchContext, instance);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e665c0a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index 3d1412d..170c20b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -115,7 +115,7 @@ public class ServiceTestUtils {
exampleApp.setName(serviceName);
exampleApp.setVersion("v1");
exampleApp.addComponent(
- createComponent("terminating-comp1", 2, "sleep " + "1000",
+ createComponent("terminating-comp1", 2, "sleep 1000",
Component.RestartPolicyEnum.NEVER, null));
exampleApp.addComponent(
createComponent("terminating-comp2", 2, "sleep 1000",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e665c0a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java
index f4f1a50..108078c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java
@@ -19,13 +19,33 @@
package org.apache.hadoop.yarn.service.containerlaunch;
import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.component.AlwaysRestartPolicy;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.NeverRestartPolicy;
+import org.apache.hadoop.yarn.service.component.OnFailureRestartPolicy;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.defaultImpl
+ .DefaultProviderService;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import static org.apache.hadoop.fi.FiConfig.getConfig;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf
+ .DEFAULT_CONTAINER_FAILURES_VALIDITY_INTERVAL;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf
+ .DEFAULT_CONTAINER_RETRY_INTERVAL;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf
+ .DEFAULT_CONTAINER_RETRY_MAX;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
/**
* Tests for {@link AbstractLauncher}.
@@ -51,4 +71,50 @@ public class TestAbstractLauncher {
Assert.assertEquals("s1:t1:ro,s2:t2:ro", dockerContainerMounts);
}
+
+ @Test
+ public void testContainerRetries() throws Exception {
+
+ DefaultProviderService providerService = new DefaultProviderService();
+ AbstractLauncher mockLauncher = mock(AbstractLauncher.class);
+ ContainerLaunchService.ComponentLaunchContext componentLaunchContext =
+ mock(ContainerLaunchService.ComponentLaunchContext.class);
+
+ ComponentInstance componentInstance = mock(ComponentInstance.class);
+
+ //Never Restart Policy
+ Component component = mock(Component.class);
+ when(componentInstance.getComponent()).thenReturn(component);
+
+ when(component.getRestartPolicyHandler()).thenReturn(NeverRestartPolicy
+ .getInstance());
+
+ providerService.buildContainerRetry(mockLauncher, getConfig(),
+ componentLaunchContext, componentInstance);
+ verifyZeroInteractions(mockLauncher);
+
+
+ //OnFailure restart policy
+ when(component.getRestartPolicyHandler()).thenReturn(OnFailureRestartPolicy
+ .getInstance());
+ when(componentLaunchContext.getConfiguration()).thenReturn(new
+ Configuration());
+ providerService.buildContainerRetry(mockLauncher, getConfig(),
+ componentLaunchContext, componentInstance);
+ verify(mockLauncher).setRetryContext(DEFAULT_CONTAINER_RETRY_MAX,
+ DEFAULT_CONTAINER_RETRY_INTERVAL,
+ DEFAULT_CONTAINER_FAILURES_VALIDITY_INTERVAL);
+
+ reset(mockLauncher);
+
+ //Always restart policy
+ when(component.getRestartPolicyHandler()).thenReturn(AlwaysRestartPolicy
+ .getInstance());
+ providerService.buildContainerRetry(mockLauncher, getConfig(),
+ componentLaunchContext, componentInstance);
+
+ verify(mockLauncher).setRetryContext(DEFAULT_CONTAINER_RETRY_MAX,
+ DEFAULT_CONTAINER_RETRY_INTERVAL,
+ DEFAULT_CONTAINER_FAILURES_VALIDITY_INTERVAL);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] hadoop git commit: YARN-8491.
TestServiceCLI#testEnableFastLaunch fail when umask is 077. Contributed by K
G Bakthavachalam.
Posted by ey...@apache.org.
YARN-8491. TestServiceCLI#testEnableFastLaunch fail when umask is 077. Contributed by K G Bakthavachalam.
(cherry picked from commit 52e1bc8539ce769f47743d8b2d318a54c3887ba0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2aaad400
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2aaad400
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2aaad400
Branch: refs/remotes/origin/branch-3.1
Commit: 2aaad4000ae2e7a33fb0753f51b89dfd3763f519
Parents: 9b4ead9
Author: bibinchundatt <bi...@apache.org>
Authored: Wed Jul 11 16:19:51 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Wed Jul 11 16:24:37 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/service/client/TestServiceCLI.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aaad400/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index 78a8198..363fe91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -121,12 +121,16 @@ public class TestServiceCLI {
basedir = new File("target", "apps");
basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath();
conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
+ fs = new SliderFileSystem(conf);
dependencyTarGzBaseDir = tmpFolder.getRoot();
+ fs.getFileSystem()
+ .setPermission(new Path(dependencyTarGzBaseDir.getAbsolutePath()),
+ new FsPermission("755"));
dependencyTarGz = getDependencyTarGz(dependencyTarGzBaseDir);
dependencyTarGzProp = DEPENDENCY_TARBALL_PATH + "=" + dependencyTarGz
.toString();
conf.set(DEPENDENCY_TARBALL_PATH, dependencyTarGz.toString());
- fs = new SliderFileSystem(conf);
+
if (basedir.exists()) {
FileUtils.deleteDirectory(basedir);
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] hadoop git commit: YARN-8577. Fix the broken anchor in SLS
site-doc. Contributed by Weiwei Yang.
Posted by ey...@apache.org.
YARN-8577. Fix the broken anchor in SLS site-doc. Contributed by Weiwei Yang.
(cherry picked from commit 3d3158cea4580eb2e3b2af635c3a7d30f4dbb873)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2212c20
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2212c20
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2212c20
Branch: refs/remotes/origin/branch-3.1
Commit: d2212c20c56944b974d10609f73387fd44df6ce1
Parents: 8e65057
Author: bibinchundatt <bi...@apache.org>
Authored: Wed Jul 25 16:19:14 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Wed Jul 25 19:00:12 2018 +0530
----------------------------------------------------------------------
.../hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2212c20/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
index 9df4998..e487267 100644
--- a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
+++ b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
@@ -27,7 +27,7 @@ YARN Scheduler Load Simulator (SLS)
* [Metrics](#Metrics)
* [Real-time Tracking](#Real-time_Tracking)
* [Offline Analysis](#Offline_Analysis)
- * [Synthetic Load Generator](#SynthGen)
+ * [Synthetic Load Generator](#Synthetic_Load_Generator)
* [Appendix](#Appendix)
* [Resources](#Resources)
* [SLS JSON input file format](#SLS_JSON_input_file_format)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] hadoop git commit: YARN-8545. Return allocated resource to RM
for failed container. Contributed by Chandni Singh
Posted by ey...@apache.org.
YARN-8545. Return allocated resource to RM for failed container.
Contributed by Chandni Singh
(cherry picked from commit 40fad32824d2f8f960c779d78357e62103453da0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/177f6045
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/177f6045
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/177f6045
Branch: refs/remotes/origin/branch-3.1
Commit: 177f6045ac4ae6e2dbae2e04da8c9cebb5da8748
Parents: 8e3807a
Author: Eric Yang <ey...@apache.org>
Authored: Thu Jul 26 18:22:57 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Jul 26 18:25:41 2018 -0400
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceScheduler.java | 3 +-
.../yarn/service/component/Component.java | 42 +++++++++++---------
.../component/instance/ComponentInstance.java | 21 +++++++---
.../instance/ComponentInstanceEvent.java | 2 +
.../containerlaunch/ContainerLaunchService.java | 12 ++++--
.../hadoop/yarn/service/MockServiceAM.java | 34 +++++++++++++++-
.../hadoop/yarn/service/TestServiceAM.java | 35 ++++++++++++++++
.../yarn/service/component/TestComponent.java | 3 +-
.../instance/TestComponentInstance.java | 26 ++++++------
9 files changed, 135 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177f6045/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index d3e8e4f..cfaf356 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -687,7 +687,8 @@ public class ServiceScheduler extends CompositeService {
}
ComponentEvent event =
new ComponentEvent(instance.getCompName(), CONTAINER_COMPLETED)
- .setStatus(status).setInstance(instance);
+ .setStatus(status).setInstance(instance)
+ .setContainerId(containerId);
dispatcher.getEventHandler().handle(event);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177f6045/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index a1ee796..aaa23da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.service.component;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ExecutionType;
@@ -518,10 +519,10 @@ public class Component implements EventHandler<ComponentEvent> {
private static class ContainerCompletedTransition extends BaseTransition {
@Override
public void transition(Component component, ComponentEvent event) {
-
+ Preconditions.checkNotNull(event.getContainerId());
component.updateMetrics(event.getStatus());
component.dispatcher.getEventHandler().handle(
- new ComponentInstanceEvent(event.getStatus().getContainerId(), STOP)
+ new ComponentInstanceEvent(event.getContainerId(), STOP)
.setStatus(event.getStatus()));
ComponentRestartPolicy restartPolicy =
@@ -784,28 +785,33 @@ public class Component implements EventHandler<ComponentEvent> {
}
private void updateMetrics(ContainerStatus status) {
- switch (status.getExitStatus()) {
- case SUCCESS:
- componentMetrics.containersSucceeded.incr();
- scheduler.getServiceMetrics().containersSucceeded.incr();
- return;
- case PREEMPTED:
- componentMetrics.containersPreempted.incr();
- scheduler.getServiceMetrics().containersPreempted.incr();
- break;
- case DISKS_FAILED:
- componentMetrics.containersDiskFailure.incr();
- scheduler.getServiceMetrics().containersDiskFailure.incr();
- break;
- default:
- break;
+ //when a container preparation fails while building launch context, then
+ //the container status may not exist.
+ if (status != null) {
+ switch (status.getExitStatus()) {
+ case SUCCESS:
+ componentMetrics.containersSucceeded.incr();
+ scheduler.getServiceMetrics().containersSucceeded.incr();
+ return;
+ case PREEMPTED:
+ componentMetrics.containersPreempted.incr();
+ scheduler.getServiceMetrics().containersPreempted.incr();
+ break;
+ case DISKS_FAILED:
+ componentMetrics.containersDiskFailure.incr();
+ scheduler.getServiceMetrics().containersDiskFailure.incr();
+ break;
+ default:
+ break;
+ }
}
// containersFailed include preempted, disks_failed etc.
componentMetrics.containersFailed.incr();
scheduler.getServiceMetrics().containersFailed.incr();
- if (Apps.shouldCountTowardsNodeBlacklisting(status.getExitStatus())) {
+ if (status != null && Apps.shouldCountTowardsNodeBlacklisting(
+ status.getExitStatus())) {
String host = scheduler.getLiveInstances().get(status.getContainerId())
.getNodeId().getHost();
failureTracker.incNodeFailure(host);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177f6045/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 529596d..44ae1e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -76,6 +76,8 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
Comparable<ComponentInstance> {
private static final Logger LOG =
LoggerFactory.getLogger(ComponentInstance.class);
+ private static final String FAILED_BEFORE_LAUNCH_DIAG =
+ "failed before launch";
private StateMachine<ComponentInstanceState, ComponentInstanceEventType,
ComponentInstanceEvent> stateMachine;
@@ -236,7 +238,8 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
@VisibleForTesting
static void handleComponentInstanceRelaunch(
- ComponentInstance compInstance, ComponentInstanceEvent event) {
+ ComponentInstance compInstance, ComponentInstanceEvent event,
+ boolean failureBeforeLaunch) {
Component comp = compInstance.getComponent();
// Do we need to relaunch the service?
@@ -252,8 +255,10 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
+ ": {} completed. Reinsert back to pending list and requested " +
"a new container." + System.lineSeparator() +
" exitStatus={}, diagnostics={}.",
- event.getContainerId(), event.getStatus().getExitStatus(),
- event.getStatus().getDiagnostics());
+ event.getContainerId(), failureBeforeLaunch ? null :
+ event.getStatus().getExitStatus(),
+ failureBeforeLaunch ? FAILED_BEFORE_LAUNCH_DIAG :
+ event.getStatus().getDiagnostics());
} else {
// When no relaunch, update component's #succeeded/#failed
// instances.
@@ -292,8 +297,8 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
Component comp = compInstance.component;
String containerDiag =
- compInstance.getCompInstanceId() + ": " + event.getStatus()
- .getDiagnostics();
+ compInstance.getCompInstanceId() + ": " + (failedBeforeLaunching ?
+ FAILED_BEFORE_LAUNCH_DIAG : event.getStatus().getDiagnostics());
compInstance.diagnostics.append(containerDiag + System.lineSeparator());
compInstance.cancelContainerStatusRetriever();
if (compInstance.getState().equals(ComponentInstanceState.UPGRADING)) {
@@ -307,6 +312,9 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
boolean shouldFailService = false;
final ServiceScheduler scheduler = comp.getScheduler();
+ scheduler.getAmRMClient().releaseAssignedContainer(
+ event.getContainerId());
+
// Check if it exceeds the failure threshold, but only if health threshold
// monitor is not enabled
if (!comp.isHealthThresholdMonitorEnabled()
@@ -347,7 +355,8 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
// According to component restart policy, handle container restart
// or finish the service (if all components finished)
- handleComponentInstanceRelaunch(compInstance, event);
+ handleComponentInstanceRelaunch(compInstance, event,
+ failedBeforeLaunching);
if (shouldFailService) {
scheduler.getTerminationHandler().terminate(-1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177f6045/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
index 707b034..889da6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service.component.instance;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.event.AbstractEvent;
@@ -32,6 +33,7 @@ public class ComponentInstanceEvent
public ComponentInstanceEvent(ContainerId containerId,
ComponentInstanceEventType componentInstanceEventType) {
super(componentInstanceEventType);
+ Preconditions.checkNotNull(containerId);
this.id = containerId;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177f6045/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
index 084c721..f674e0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -22,8 +22,11 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.service.ServiceContext;
import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.component.ComponentEventType;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
import org.apache.hadoop.yarn.service.provider.ProviderService;
import org.apache.hadoop.yarn.service.provider.ProviderFactory;
@@ -116,9 +119,12 @@ public class ContainerLaunchService extends AbstractService{
launcher.completeContainerLaunch(), true);
}
} catch (Exception e) {
- LOG.error(instance.getCompInstanceId()
- + ": Failed to launch container. ", e);
-
+ LOG.error("{}: Failed to launch container.",
+ instance.getCompInstanceId(), e);
+ ComponentEvent event = new ComponentEvent(instance.getCompName(),
+ ComponentEventType.CONTAINER_COMPLETED)
+ .setInstance(instance).setContainerId(container.getId());
+ context.scheduler.getDispatcher().getEventHandler().handle(event);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177f6045/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
index 4a75aef..729287c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
@@ -68,6 +68,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeoutException;
@@ -99,6 +100,8 @@ public class MockServiceAM extends ServiceMaster {
private Map<ContainerId, ContainerStatus> containerStatuses =
new ConcurrentHashMap<>();
+ private Set<ContainerId> releasedContainers = ConcurrentHashMap.newKeySet();
+
private Credentials amCreds;
public MockServiceAM(Service service) {
@@ -223,6 +226,13 @@ public class MockServiceAM extends ServiceMaster {
return response;
}
+ @Override
+ public synchronized void releaseAssignedContainer(
+ ContainerId containerId) {
+ releasedContainers.add(containerId);
+ super.releaseAssignedContainer(containerId);
+ }
+
@Override public void unregisterApplicationMaster(
FinalApplicationStatus appStatus, String appMessage,
String appTrackingUrl) {
@@ -288,7 +298,7 @@ public class MockServiceAM extends ServiceMaster {
}
/**
- *
+ * Creates a mock container and container ID and feeds to the component.
* @param service The service for the component
* @param id The id for the container
* @param compName The component to which the container is fed
@@ -297,6 +307,18 @@ public class MockServiceAM extends ServiceMaster {
public Container feedContainerToComp(Service service, int id,
String compName) {
ContainerId containerId = createContainerId(id);
+ return feedContainerToComp(service, containerId, compName);
+ }
+
+ /**
+ * Feeds the container to the component.
+ * @param service The service for the component
+ * @param containerId container id
+ * @param compName The component to which the container is fed
+ * @return
+ */
+ public Container feedContainerToComp(Service service, ContainerId containerId,
+ String compName) {
Container container = createContainer(containerId, compName);
synchronized (feedContainers) {
feedContainers.add(container);
@@ -423,4 +445,14 @@ public class MockServiceAM extends ServiceMaster {
}
return ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
}
+
+ /**
+ * Waits for the container to get released
+ * @param containerId ContainerId
+ */
+ public void waitForContainerToRelease(ContainerId containerId)
+ throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(() -> releasedContainers.contains(containerId),
+ 1000, 9990000);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177f6045/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
index e9478f0..21e93fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service;
+import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.io.FileUtils;
import org.apache.curator.test.TestingCluster;
@@ -391,4 +392,38 @@ public class TestServiceAM extends ServiceTestUtils{
.equals("newer.host"), 2000, 200000);
am.stop();
}
+
+ // Test to verify that the containers are released and the
+ // component instance is added to the pending queue when building the launch
+ // context fails.
+ @Test(timeout = 9990000)
+ public void testContainersReleasedWhenPreLaunchFails()
+ throws Exception {
+ ApplicationId applicationId = ApplicationId.newInstance(
+ System.currentTimeMillis(), 1);
+ Service exampleApp = new Service();
+ exampleApp.setId(applicationId.toString());
+ exampleApp.setVersion("v1");
+ exampleApp.setName("testContainersReleasedWhenPreLaunchFails");
+
+ Component compA = createComponent("compa", 1, "pwd");
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.TARBALL);
+ compA.artifact(artifact);
+ exampleApp.addComponent(compA);
+
+ MockServiceAM am = new MockServiceAM(exampleApp);
+ am.init(conf);
+ am.start();
+
+ ContainerId containerId = am.createContainerId(1);
+
+ // allocate a container
+ am.feedContainerToComp(exampleApp, containerId, "compa");
+ am.waitForContainerToRelease(containerId);
+
+ Assert.assertEquals(1,
+ am.getComponent("compa").getPendingInstances().size());
+ am.stop();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177f6045/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
index d7c15ec..a3b86a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
@@ -196,7 +196,8 @@ public class TestComponent {
org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE,
"successful", 0);
comp.handle(new ComponentEvent(comp.getName(),
- ComponentEventType.CONTAINER_COMPLETED).setStatus(containerStatus));
+ ComponentEventType.CONTAINER_COMPLETED).setStatus(containerStatus)
+ .setContainerId(instanceContainer.getId()));
componentInstance.handle(
new ComponentInstanceEvent(componentInstance.getContainer().getId(),
ComponentInstanceEventType.STOP).setStatus(containerStatus));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177f6045/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
index 26e8c93..95d774a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
@@ -245,7 +245,7 @@ public class TestComponentInstance {
comp.getAllComponentInstances().iterator().next();
ComponentInstance.handleComponentInstanceRelaunch(componentInstance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
verify(comp, never()).markAsSucceeded(any(ComponentInstance.class));
verify(comp, never()).markAsFailed(any(ComponentInstance.class));
@@ -262,7 +262,7 @@ public class TestComponentInstance {
componentInstance = comp.getAllComponentInstances().iterator().next();
containerStatus.setExitStatus(1);
ComponentInstance.handleComponentInstanceRelaunch(componentInstance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
verify(comp, never()).markAsSucceeded(any(ComponentInstance.class));
verify(comp, never()).markAsFailed(any(ComponentInstance.class));
verify(comp, times(1)).reInsertPendingInstance(
@@ -286,7 +286,7 @@ public class TestComponentInstance {
when(comp.getNumSucceededInstances()).thenReturn(new Long(1));
ComponentInstance.handleComponentInstanceRelaunch(componentInstance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
verify(comp, times(1)).markAsSucceeded(any(ComponentInstance.class));
verify(comp, never()).markAsFailed(any(ComponentInstance.class));
verify(comp, times(0)).reInsertPendingInstance(
@@ -304,7 +304,7 @@ public class TestComponentInstance {
when(comp.getNumFailedInstances()).thenReturn(new Long(1));
ComponentInstance.handleComponentInstanceRelaunch(componentInstance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
verify(comp, never()).markAsSucceeded(any(ComponentInstance.class));
verify(comp, times(1)).markAsFailed(any(ComponentInstance.class));
@@ -323,7 +323,7 @@ public class TestComponentInstance {
componentInstance = comp.getAllComponentInstances().iterator().next();
containerStatus.setExitStatus(1);
ComponentInstance.handleComponentInstanceRelaunch(componentInstance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
verify(comp, never()).markAsSucceeded(any(ComponentInstance.class));
verify(comp, never()).markAsFailed(any(ComponentInstance.class));
verify(comp, times(1)).reInsertPendingInstance(
@@ -340,7 +340,7 @@ public class TestComponentInstance {
componentInstance = comp.getAllComponentInstances().iterator().next();
containerStatus.setExitStatus(1);
ComponentInstance.handleComponentInstanceRelaunch(componentInstance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
verify(comp, never()).markAsSucceeded(any(ComponentInstance.class));
verify(comp, times(1)).markAsFailed(any(ComponentInstance.class));
verify(comp, times(0)).reInsertPendingInstance(
@@ -363,7 +363,7 @@ public class TestComponentInstance {
containerStatus.setExitStatus(1);
ComponentInstance commponentInstance = iter.next();
ComponentInstance.handleComponentInstanceRelaunch(commponentInstance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
verify(comp, never()).markAsSucceeded(any(ComponentInstance.class));
verify(comp, never()).markAsFailed(any(ComponentInstance.class));
@@ -404,7 +404,7 @@ public class TestComponentInstance {
when(component2Instance.getComponent().getNumFailedInstances())
.thenReturn(new Long(failed2Instances.size()));
ComponentInstance.handleComponentInstanceRelaunch(component2Instance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
}
Map<String, ComponentInstance> failed1Instances = new HashMap<>();
@@ -418,7 +418,7 @@ public class TestComponentInstance {
when(component1Instance.getComponent().getNumFailedInstances())
.thenReturn(new Long(failed1Instances.size()));
ComponentInstance.handleComponentInstanceRelaunch(component1Instance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
}
verify(comp, never()).markAsSucceeded(any(ComponentInstance.class));
@@ -458,7 +458,7 @@ public class TestComponentInstance {
when(component2Instance.getComponent().getNumSucceededInstances())
.thenReturn(new Long(succeeded2Instances.size()));
ComponentInstance.handleComponentInstanceRelaunch(component2Instance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
}
Map<String, ComponentInstance> succeeded1Instances = new HashMap<>();
@@ -471,7 +471,7 @@ public class TestComponentInstance {
when(component1Instance.getComponent().getNumSucceededInstances())
.thenReturn(new Long(succeeded1Instances.size()));
ComponentInstance.handleComponentInstanceRelaunch(component1Instance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
}
verify(comp, times(2)).markAsSucceeded(any(ComponentInstance.class));
@@ -500,7 +500,7 @@ public class TestComponentInstance {
for (ComponentInstance component2Instance : component2Instances) {
ComponentInstance.handleComponentInstanceRelaunch(component2Instance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
}
succeeded1Instances = new HashMap<>();
@@ -511,7 +511,7 @@ public class TestComponentInstance {
when(component1Instance.getComponent().getSucceededInstances())
.thenReturn(succeeded1Instances.values());
ComponentInstance.handleComponentInstanceRelaunch(component1Instance,
- componentInstanceEvent);
+ componentInstanceEvent, false);
}
verify(comp, times(2)).markAsSucceeded(any(ComponentInstance.class));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] hadoop git commit: HDFS-13729. Fix broken links to RBF
documentation. Contributed by Gabor Bota.
Posted by ey...@apache.org.
HDFS-13729. Fix broken links to RBF documentation. Contributed by Gabor Bota.
(cherry picked from commit 418cc7f3aeabedc57c94aa9d4c4248c1476ac90e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe256a98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe256a98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe256a98
Branch: refs/remotes/origin/branch-3.1
Commit: fe256a98ff7b8cfb2b7c5c43fe965720e371ce1a
Parents: f9fa3cb
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Jul 11 14:46:43 2018 -0400
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Jul 11 14:47:21 2018 -0400
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 4 ++--
.../hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md | 2 +-
hadoop-project/src/site/markdown/index.md.vm | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe256a98/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 9ed69bf..391b71b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -420,7 +420,7 @@ Runs a HDFS dfsadmin client.
Usage: `hdfs dfsrouter`
-Runs the DFS router. See [Router](./HDFSRouterFederation.html#Router) for more info.
+Runs the DFS router. See [Router](../hadoop-hdfs-rbf/HDFSRouterFederation.html#Router) for more info.
### `dfsrouteradmin`
@@ -449,7 +449,7 @@ Usage:
| `-nameservice` `disable` `enable` *nameservice* | Disable/enable a name service from the federation. If disabled, requests will not go to that name service. |
| `-getDisabledNameservices` | Get the name services that are disabled in the federation. |
-The commands for managing Router-based federation. See [Mount table management](./HDFSRouterFederation.html#Mount_table_management) for more info.
+The commands for managing Router-based federation. See [Mount table management](../hadoop-hdfs-rbf/HDFSRouterFederation.html#Mount_table_management) for more info.
### `diskbalancer`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe256a98/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md
index 01e7076..b8d5321 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md
@@ -38,7 +38,7 @@ is limited to creating a *read-only image* of a remote namespace that implements
to serve the image. Specifically, reads from a snapshot of a remote namespace are
supported. Adding a remote namespace to an existing/running namenode, refreshing the
remote snapshot, unmounting, and writes are not available in this release. One
-can use [ViewFs](./ViewFs.html) and [RBF](HDFSRouterFederation.html) to
+can use [ViewFs](./ViewFs.html) and [RBF](../hadoop-hdfs-rbf/HDFSRouterFederation.html) to
integrate namespaces with `PROVIDED` storage into an existing deployment.
Creating HDFS Clusters with `PROVIDED` Storage
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe256a98/hadoop-project/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/markdown/index.md.vm b/hadoop-project/src/site/markdown/index.md.vm
index 8b9cfda..438145a 100644
--- a/hadoop-project/src/site/markdown/index.md.vm
+++ b/hadoop-project/src/site/markdown/index.md.vm
@@ -225,7 +225,7 @@ cluster for existing HDFS clients.
See [HDFS-10467](https://issues.apache.org/jira/browse/HDFS-10467) and the
HDFS Router-based Federation
-[documentation](./hadoop-project-dist/hadoop-hdfs/HDFSRouterFederation.html) for
+[documentation](./hadoop-project-dist/hadoop-hdfs-rbf/HDFSRouterFederation.html) for
more details.
API-based configuration of Capacity Scheduler queue configuration
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] hadoop git commit: YARN-8511. When AM releases a container,
RM removes allocation tags before it is released by NM. (Weiwei Yang
via wangda)
Posted by ey...@apache.org.
YARN-8511. When AM releases a container, RM removes allocation tags before it is released by NM. (Weiwei Yang via wangda)
Change-Id: I6f9f409f2ef685b405cbff547dea9623bf3322d9
(cherry picked from commit 752dcce5f4cf0f6ebcb40a61f622f1a885c4bda7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44beab0b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44beab0b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44beab0b
Branch: refs/remotes/origin/branch-3.1
Commit: 44beab0b63cb853f2f711d9592fcd947241d112e
Parents: ac9155d
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Jul 16 10:54:41 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Jul 16 11:04:08 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 6 ++
.../yarn/sls/scheduler/RMNodeWrapper.java | 6 ++
.../rmcontainer/RMContainerImpl.java | 5 -
.../server/resourcemanager/rmnode/RMNode.java | 6 ++
.../resourcemanager/rmnode/RMNodeImpl.java | 5 +
.../scheduler/SchedulerNode.java | 15 +++
.../yarn/server/resourcemanager/MockNodes.java | 5 +
.../rmcontainer/TestRMContainerImpl.java | 16 ++-
.../scheduler/TestAbstractYarnScheduler.java | 104 +++++++++++++++++++
9 files changed, 162 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44beab0b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 0c99139..69946c8 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
@@ -219,6 +220,11 @@ public class NodeInfo {
}
@Override
+ public RMContext getRMContext() {
+ return null;
+ }
+
+ @Override
public Resource getPhysicalResource() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44beab0b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 78645e9..a96b790 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
@@ -207,6 +208,11 @@ public class RMNodeWrapper implements RMNode {
}
@Override
+ public RMContext getRMContext() {
+ return node.getRMContext();
+ }
+
+ @Override
public Resource getPhysicalResource() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44beab0b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index b5c8e7c..efac666 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -701,11 +701,6 @@ public class RMContainerImpl implements RMContainer {
@Override
public void transition(RMContainerImpl container, RMContainerEvent event) {
- // Notify AllocationTagsManager
- container.rmContext.getAllocationTagsManager().removeContainer(
- container.getNodeId(), container.getContainerId(),
- container.getAllocationTags());
-
RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
container.finishTime = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44beab0b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 872f2a6..68a780e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
/**
* Node managers information on available resources
@@ -189,4 +190,9 @@ public interface RMNode {
* @return a map of each allocation tag and its count.
*/
Map<String, Long> getAllocationTagsWithCount();
+
+ /**
+ * @return the RM context associated with this RM node.
+ */
+ RMContext getRMContext();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44beab0b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index b942afa..dfd93e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -1541,4 +1541,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
return context.getAllocationTagsManager()
.getAllocationTagsWithCount(getNodeID());
}
+
+ @Override
+ public RMContext getRMContext() {
+ return this.context;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44beab0b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index d5bfc57..59771fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
@@ -74,6 +75,7 @@ public abstract class SchedulerNode {
private final RMNode rmNode;
private final String nodeName;
+ private final RMContext rmContext;
private volatile Set<String> labels = null;
@@ -83,6 +85,7 @@ public abstract class SchedulerNode {
public SchedulerNode(RMNode node, boolean usePortForNodeName,
Set<String> labels) {
this.rmNode = node;
+ this.rmContext = node.getRMContext();
this.unallocatedResource = Resources.clone(node.getTotalCapability());
this.totalResource = Resources.clone(node.getTotalCapability());
if (usePortForNodeName) {
@@ -242,6 +245,18 @@ public abstract class SchedulerNode {
launchedContainers.remove(containerId);
Container container = info.container.getContainer();
+
+ // We remove allocation tags when a container is actually
+ // released on NM. This is to avoid running into situation
+ // when AM releases a container and NM has some delay to
+ // actually release it, then the tag can still be visible
+ // at RM so that RM can respect it during scheduling new containers.
+ if (rmContext != null && rmContext.getAllocationTagsManager() != null) {
+ rmContext.getAllocationTagsManager()
+ .removeContainer(container.getNodeId(),
+ container.getId(), container.getAllocationTags());
+ }
+
updateResourceForReleasedContainer(container);
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44beab0b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 84105d9..9041132 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -286,6 +286,11 @@ public class MockNodes {
}
@Override
+ public RMContext getRMContext() {
+ return null;
+ }
+
+ @Override
public Resource getPhysicalResource() {
return this.physicalResource;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44beab0b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 7a930cd..1115e8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -60,10 +60,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTags;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -401,6 +405,7 @@ public class TestRMContainerImpl {
Container container = BuilderUtils.newContainer(containerId, nodeId,
"host:3465", resource, priority, null);
+ container.setAllocationTags(ImmutableSet.of("mapper"));
ConcurrentMap<ApplicationId, RMApp> rmApps =
spy(new ConcurrentHashMap<ApplicationId, RMApp>());
RMApp rmApp = mock(RMApp.class);
@@ -423,11 +428,14 @@ public class TestRMContainerImpl {
true);
when(rmContext.getYarnConfiguration()).thenReturn(conf);
+ RMNode rmNode = new RMNodeImpl(nodeId, rmContext,
+ "localhost", 0, 0, null, Resource.newInstance(10240, 10), null);
+ SchedulerNode schedulerNode = new FiCaSchedulerNode(rmNode, false);
+
/* First container: ALLOCATED -> KILLED */
RMContainerImpl rmContainer = new RMContainerImpl(container,
SchedulerRequestKey.extractFrom(container), appAttemptId,
nodeId, "user", rmContext);
- rmContainer.setAllocationTags(ImmutableSet.of("mapper"));
Assert.assertEquals(0,
tagsManager.getNodeCardinalityByOp(nodeId,
@@ -437,6 +445,7 @@ public class TestRMContainerImpl {
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.START));
+ schedulerNode.allocateContainer(rmContainer);
Assert.assertEquals(1,
tagsManager.getNodeCardinalityByOp(nodeId,
@@ -446,6 +455,7 @@ public class TestRMContainerImpl {
rmContainer.handle(new RMContainerFinishedEvent(containerId, ContainerStatus
.newInstance(containerId, ContainerState.COMPLETE, "", 0),
RMContainerEventType.KILL));
+ schedulerNode.releaseContainer(container.getId(), true);
Assert.assertEquals(0,
tagsManager.getNodeCardinalityByOp(nodeId,
@@ -465,6 +475,7 @@ public class TestRMContainerImpl {
rmContainer.setAllocationTags(ImmutableSet.of("mapper"));
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.START));
+ schedulerNode.allocateContainer(rmContainer);
Assert.assertEquals(1,
tagsManager.getNodeCardinalityByOp(nodeId,
@@ -477,6 +488,7 @@ public class TestRMContainerImpl {
rmContainer.handle(new RMContainerFinishedEvent(containerId, ContainerStatus
.newInstance(containerId, ContainerState.COMPLETE, "", 0),
RMContainerEventType.FINISHED));
+ schedulerNode.releaseContainer(container.getId(), true);
Assert.assertEquals(0,
tagsManager.getNodeCardinalityByOp(nodeId,
@@ -496,6 +508,7 @@ public class TestRMContainerImpl {
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.START));
+ schedulerNode.allocateContainer(rmContainer);
Assert.assertEquals(1,
tagsManager.getNodeCardinalityByOp(nodeId,
@@ -511,6 +524,7 @@ public class TestRMContainerImpl {
rmContainer.handle(new RMContainerFinishedEvent(containerId, ContainerStatus
.newInstance(containerId, ContainerState.COMPLETE, "", 0),
RMContainerEventType.FINISHED));
+ schedulerNode.releaseContainer(container.getId(), true);
Assert.assertEquals(0,
tagsManager.getNodeCardinalityByOp(nodeId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44beab0b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index c0f8d39..ba409b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -27,9 +27,16 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -416,6 +423,103 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
}
}
+ @Test(timeout = 30000l)
+ public void testContainerReleaseWithAllocationTags() throws Exception {
+ // Currently only can be tested against capacity scheduler.
+ if (getSchedulerType().equals(SchedulerType.CAPACITY)) {
+ final String testTag1 = "some-tag";
+ final String testTag2 = "some-other-tag";
+ YarnConfiguration conf = getConf();
+ conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, "scheduler");
+ MockRM rm1 = new MockRM(conf);
+ rm1.start();
+ MockNM nm1 = new MockNM("127.0.0.1:1234",
+ 10240, rm1.getResourceTrackerService());
+ nm1.registerNode();
+ RMApp app1 =
+ rm1.submitApp(200, "name", "user", new HashMap<>(), false, "default",
+ -1, null, "Test", false, true);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // allocate 1 container with tag1
+ SchedulingRequest sr = SchedulingRequest
+ .newInstance(1l, Priority.newInstance(1),
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED),
+ Sets.newHashSet(testTag1),
+ ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)),
+ null);
+
+ // allocate 3 containers with tag2
+ SchedulingRequest sr1 = SchedulingRequest
+ .newInstance(2l, Priority.newInstance(1),
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED),
+ Sets.newHashSet(testTag2),
+ ResourceSizing.newInstance(3, Resource.newInstance(1024, 1)),
+ null);
+
+ AllocateRequest ar = AllocateRequest.newBuilder()
+ .schedulingRequests(Lists.newArrayList(sr, sr1)).build();
+ am1.allocate(ar);
+ nm1.nodeHeartbeat(true);
+
+ List<Container> allocated = new ArrayList<>();
+ while (allocated.size() < 4) {
+ AllocateResponse rsp = am1
+ .allocate(new ArrayList<>(), new ArrayList<>());
+ allocated.addAll(rsp.getAllocatedContainers());
+ nm1.nodeHeartbeat(true);
+ Thread.sleep(1000);
+ }
+
+ Assert.assertEquals(4, allocated.size());
+
+ Set<Container> containers = allocated.stream()
+ .filter(container -> container.getAllocationRequestId() == 1l)
+ .collect(Collectors.toSet());
+ Assert.assertNotNull(containers);
+ Assert.assertEquals(1, containers.size());
+ ContainerId cid = containers.iterator().next().getId();
+
+ // mock container start
+ rm1.getRMContext().getScheduler()
+ .getSchedulerNode(nm1.getNodeId()).containerStarted(cid);
+
+ // verifies the allocation is made with correct number of tags
+ Map<String, Long> nodeTags = rm1.getRMContext()
+ .getAllocationTagsManager()
+ .getAllocationTagsWithCount(nm1.getNodeId());
+ Assert.assertNotNull(nodeTags.get(testTag1));
+ Assert.assertEquals(1, nodeTags.get(testTag1).intValue());
+
+ // release a container
+ am1.allocate(new ArrayList<>(), Lists.newArrayList(cid));
+
+ // before NM confirms, the tag should still exist
+ nodeTags = rm1.getRMContext().getAllocationTagsManager()
+ .getAllocationTagsWithCount(nm1.getNodeId());
+ Assert.assertNotNull(nodeTags);
+ Assert.assertNotNull(nodeTags.get(testTag1));
+ Assert.assertEquals(1, nodeTags.get(testTag1).intValue());
+
+ // NM reports back that container is released
+ // RM should cleanup the tag
+ ContainerStatus cs = ContainerStatus.newInstance(cid,
+ ContainerState.COMPLETE, "", 0);
+ nm1.nodeHeartbeat(Lists.newArrayList(cs), true);
+
+ // Wait on condition
+ // 1) tag1 doesn't exist anymore
+ // 2) num of tag2 is still 3
+ GenericTestUtils.waitFor(() -> {
+ Map<String, Long> tags = rm1.getRMContext()
+ .getAllocationTagsManager()
+ .getAllocationTagsWithCount(nm1.getNodeId());
+ return tags.get(testTag1) == null &&
+ tags.get(testTag2).intValue() == 3;
+ }, 500, 3000);
+ }
+ }
+
@Test(timeout=60000)
public void testContainerReleasedByNode() throws Exception {
System.out.println("Starting testContainerReleasedByNode");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] hadoop git commit: HDFS-13475. RBF: Admin cannot enforce
Router enter SafeMode. Contributed by Chao Sun.
Posted by ey...@apache.org.
HDFS-13475. RBF: Admin cannot enforce Router enter SafeMode. Contributed by Chao Sun.
(cherry picked from commit 359ea4e18147af5677c6d88265e26de6b6c72999)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4898edf4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4898edf4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4898edf4
Branch: refs/remotes/origin/branch-3.1
Commit: 4898edf4f7fc83ab48cc2ed20bfe66ca0804699a
Parents: 9a79e89
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Jul 16 09:46:21 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Mon Jul 16 09:47:00 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/federation/router/Router.java | 7 +++
.../federation/router/RouterAdminServer.java | 32 ++++++++---
.../federation/router/RouterRpcServer.java | 26 +--------
.../router/RouterSafemodeService.java | 44 ++++++++++++---
.../federation/router/TestRouterAdminCLI.java | 7 ++-
.../federation/router/TestRouterSafemode.java | 58 ++++++++++++++++----
6 files changed, 121 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4898edf4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index df2a448..7e67daa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -665,4 +665,11 @@ public class Router extends CompositeService {
Collection<NamenodeHeartbeatService> getNamenodeHearbeatServices() {
return this.namenodeHeartbeatServices;
}
+
+ /**
+ * Get the Router safe mode service
+ */
+ RouterSafemodeService getSafemodeService() {
+ return this.safemodeService;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4898edf4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
index 139dfb8..8e23eca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Set;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -272,23 +273,37 @@ public class RouterAdminServer extends AbstractService
@Override
public EnterSafeModeResponse enterSafeMode(EnterSafeModeRequest request)
throws IOException {
- this.router.updateRouterState(RouterServiceState.SAFEMODE);
- this.router.getRpcServer().setSafeMode(true);
- return EnterSafeModeResponse.newInstance(verifySafeMode(true));
+ boolean success = false;
+ RouterSafemodeService safeModeService = this.router.getSafemodeService();
+ if (safeModeService != null) {
+ this.router.updateRouterState(RouterServiceState.SAFEMODE);
+ safeModeService.setManualSafeMode(true);
+ success = verifySafeMode(true);
+ }
+ return EnterSafeModeResponse.newInstance(success);
}
@Override
public LeaveSafeModeResponse leaveSafeMode(LeaveSafeModeRequest request)
throws IOException {
- this.router.updateRouterState(RouterServiceState.RUNNING);
- this.router.getRpcServer().setSafeMode(false);
- return LeaveSafeModeResponse.newInstance(verifySafeMode(false));
+ boolean success = false;
+ RouterSafemodeService safeModeService = this.router.getSafemodeService();
+ if (safeModeService != null) {
+ this.router.updateRouterState(RouterServiceState.RUNNING);
+ safeModeService.setManualSafeMode(false);
+ success = verifySafeMode(false);
+ }
+ return LeaveSafeModeResponse.newInstance(success);
}
@Override
public GetSafeModeResponse getSafeMode(GetSafeModeRequest request)
throws IOException {
- boolean isInSafeMode = this.router.getRpcServer().isInSafeMode();
+ boolean isInSafeMode = false;
+ RouterSafemodeService safeModeService = this.router.getSafemodeService();
+ if (safeModeService != null) {
+ isInSafeMode = safeModeService.isInSafeMode();
+ }
return GetSafeModeResponse.newInstance(isInSafeMode);
}
@@ -298,7 +313,8 @@ public class RouterAdminServer extends AbstractService
* @return
*/
private boolean verifySafeMode(boolean isInSafeMode) {
- boolean serverInSafeMode = this.router.getRpcServer().isInSafeMode();
+ Preconditions.checkNotNull(this.router.getSafemodeService());
+ boolean serverInSafeMode = this.router.getSafemodeService().isInSafeMode();
RouterServiceState currentState = this.router.getRouterState();
return (isInSafeMode && currentState == RouterServiceState.SAFEMODE
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4898edf4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 7031af7..027db8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -193,9 +193,6 @@ public class RouterRpcServer extends AbstractService
/** Interface to map global name space to HDFS subcluster name spaces. */
private final FileSubclusterResolver subclusterResolver;
- /** If we are in safe mode, fail requests as if a standby NN. */
- private volatile boolean safeMode;
-
/** Category of the operation that a thread is executing. */
private final ThreadLocal<OperationCategory> opCategory = new ThreadLocal<>();
@@ -456,7 +453,8 @@ public class RouterRpcServer extends AbstractService
return;
}
- if (safeMode) {
+ RouterSafemodeService safemodeService = router.getSafemodeService();
+ if (safemodeService != null && safemodeService.isInSafeMode()) {
// Throw standby exception, router is not available
if (rpcMonitor != null) {
rpcMonitor.routerFailureSafemode();
@@ -466,26 +464,6 @@ public class RouterRpcServer extends AbstractService
}
}
- /**
- * In safe mode all RPC requests will fail and return a standby exception.
- * The client will try another Router, similar to the client retry logic for
- * HA.
- *
- * @param mode True if enabled, False if disabled.
- */
- public void setSafeMode(boolean mode) {
- this.safeMode = mode;
- }
-
- /**
- * Check if the Router is in safe mode and cannot serve RPC calls.
- *
- * @return If the Router is in safe mode.
- */
- public boolean isInSafeMode() {
- return this.safeMode;
- }
-
@Override // ClientProtocol
public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4898edf4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java
index 5dfb356..877e1d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java
@@ -42,6 +42,23 @@ public class RouterSafemodeService extends PeriodicService {
/** Router to manage safe mode. */
private final Router router;
+ /**
+ * If we are in safe mode, fail requests as if a standby NN.
+ * Router can enter safe mode in two different ways:
+ * 1. upon start up: router enters this mode after service start, and will
+ * exit after certain time threshold;
+ * 2. via admin command: router enters this mode via admin command:
+ * dfsrouteradmin -safemode enter
+ * and exit after admin command:
+ * dfsrouteradmin -safemode leave
+ */
+
+ /** Whether Router is in safe mode */
+ private volatile boolean safeMode;
+
+ /** Whether the Router safe mode is set manually (i.e., via Router admin) */
+ private volatile boolean isSafeModeSetManually;
+
/** Interval in ms to wait post startup before allowing RPC requests. */
private long startupInterval;
/** Interval in ms after which the State Store cache is too stale. */
@@ -64,13 +81,28 @@ public class RouterSafemodeService extends PeriodicService {
}
/**
+ * Return whether the current Router is in safe mode.
+ */
+ boolean isInSafeMode() {
+ return this.safeMode;
+ }
+
+ /**
+ * Set the flag to indicate that the safe mode for this Router is set manually
+ * via the Router admin command.
+ */
+ void setManualSafeMode(boolean mode) {
+ this.safeMode = mode;
+ this.isSafeModeSetManually = mode;
+ }
+
+ /**
* Enter safe mode.
*/
private void enter() {
LOG.info("Entering safe mode");
enterSafeModeTime = now();
- RouterRpcServer rpcServer = router.getRpcServer();
- rpcServer.setSafeMode(true);
+ safeMode = true;
router.updateRouterState(RouterServiceState.SAFEMODE);
}
@@ -87,8 +119,7 @@ public class RouterSafemodeService extends PeriodicService {
} else {
routerMetrics.setSafeModeTime(timeInSafemode);
}
- RouterRpcServer rpcServer = router.getRpcServer();
- rpcServer.setSafeMode(false);
+ safeMode = false;
router.updateRouterState(RouterServiceState.RUNNING);
}
@@ -131,17 +162,16 @@ public class RouterSafemodeService extends PeriodicService {
this.startupInterval - delta);
return;
}
- RouterRpcServer rpcServer = router.getRpcServer();
StateStoreService stateStore = router.getStateStore();
long cacheUpdateTime = stateStore.getCacheUpdateTime();
boolean isCacheStale = (now - cacheUpdateTime) > this.staleInterval;
// Always update to indicate our cache was updated
if (isCacheStale) {
- if (!rpcServer.isInSafeMode()) {
+ if (!safeMode) {
enter();
}
- } else if (rpcServer.isInSafeMode()) {
+ } else if (safeMode && !isSafeModeSetManually) {
// Cache recently updated, leave safe mode
leave();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4898edf4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
index 7e04e61..5207f00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
@@ -82,6 +82,7 @@ public class TestRouterAdminCLI {
.stateStore()
.admin()
.rpc()
+ .safemode()
.build();
cluster.addRouterOverrides(conf);
@@ -501,13 +502,13 @@ public class TestRouterAdminCLI {
public void testManageSafeMode() throws Exception {
// ensure the Router become RUNNING state
waitState(RouterServiceState.RUNNING);
- assertFalse(routerContext.getRouter().getRpcServer().isInSafeMode());
+ assertFalse(routerContext.getRouter().getSafemodeService().isInSafeMode());
assertEquals(0, ToolRunner.run(admin,
new String[] {"-safemode", "enter"}));
// verify state
assertEquals(RouterServiceState.SAFEMODE,
routerContext.getRouter().getRouterState());
- assertTrue(routerContext.getRouter().getRpcServer().isInSafeMode());
+ assertTrue(routerContext.getRouter().getSafemodeService().isInSafeMode());
System.setOut(new PrintStream(out));
assertEquals(0, ToolRunner.run(admin,
@@ -519,7 +520,7 @@ public class TestRouterAdminCLI {
// verify state
assertEquals(RouterServiceState.RUNNING,
routerContext.getRouter().getRouterState());
- assertFalse(routerContext.getRouter().getRpcServer().isInSafeMode());
+ assertFalse(routerContext.getRouter().getSafemodeService().isInSafeMode());
out.reset();
assertEquals(0, ToolRunner.run(admin,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4898edf4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
index f16ceb5..9c1aeb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
@@ -28,14 +28,17 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.tools.federation.RouterAdmin;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -60,12 +63,12 @@ public class TestRouterSafemode {
// 2 sec startup standby
conf.setTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION,
TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS);
- // 1 sec cache refresh
+ // 200 ms cache refresh
conf.setTimeDuration(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
- TimeUnit.SECONDS.toMillis(1), TimeUnit.MILLISECONDS);
- // 2 sec post cache update before entering safemode (2 intervals)
+ 200, TimeUnit.MILLISECONDS);
+ // 1 sec post cache update before entering safemode (2 intervals)
conf.setTimeDuration(DFS_ROUTER_SAFEMODE_EXPIRATION,
- TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS);
+ TimeUnit.SECONDS.toMillis(1), TimeUnit.MILLISECONDS);
conf.set(RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
conf.set(RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
@@ -77,6 +80,7 @@ public class TestRouterSafemode {
// RPC + State Store + Safe Mode only
conf = new RouterConfigBuilder(conf)
.rpc()
+ .admin()
.safemode()
.stateStore()
.metrics()
@@ -118,7 +122,7 @@ public class TestRouterSafemode {
public void testRouterExitSafemode()
throws InterruptedException, IllegalStateException, IOException {
- assertTrue(router.getRpcServer().isInSafeMode());
+ assertTrue(router.getSafemodeService().isInSafeMode());
verifyRouter(RouterServiceState.SAFEMODE);
// Wait for initial time in milliseconds
@@ -129,7 +133,7 @@ public class TestRouterSafemode {
TimeUnit.SECONDS.toMillis(1), TimeUnit.MILLISECONDS);
Thread.sleep(interval);
- assertFalse(router.getRpcServer().isInSafeMode());
+ assertFalse(router.getSafemodeService().isInSafeMode());
verifyRouter(RouterServiceState.RUNNING);
}
@@ -138,7 +142,7 @@ public class TestRouterSafemode {
throws IllegalStateException, IOException, InterruptedException {
// Verify starting state
- assertTrue(router.getRpcServer().isInSafeMode());
+ assertTrue(router.getSafemodeService().isInSafeMode());
verifyRouter(RouterServiceState.SAFEMODE);
// We should be in safe mode for DFS_ROUTER_SAFEMODE_EXTENSION time
@@ -157,7 +161,7 @@ public class TestRouterSafemode {
Thread.sleep(interval1);
// Running
- assertFalse(router.getRpcServer().isInSafeMode());
+ assertFalse(router.getSafemodeService().isInSafeMode());
verifyRouter(RouterServiceState.RUNNING);
// Disable cache
@@ -167,12 +171,12 @@ public class TestRouterSafemode {
long interval2 =
conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXPIRATION,
TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS) +
- conf.getTimeDuration(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
+ 2 * conf.getTimeDuration(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
TimeUnit.SECONDS.toMillis(1), TimeUnit.MILLISECONDS);
Thread.sleep(interval2);
// Safemode
- assertTrue(router.getRpcServer().isInSafeMode());
+ assertTrue(router.getSafemodeService().isInSafeMode());
verifyRouter(RouterServiceState.SAFEMODE);
}
@@ -180,7 +184,7 @@ public class TestRouterSafemode {
public void testRouterRpcSafeMode()
throws IllegalStateException, IOException {
- assertTrue(router.getRpcServer().isInSafeMode());
+ assertTrue(router.getSafemodeService().isInSafeMode());
verifyRouter(RouterServiceState.SAFEMODE);
// If the Router is in Safe Mode, we should get a SafeModeException
@@ -194,6 +198,38 @@ public class TestRouterSafemode {
assertTrue("We should have thrown a safe mode exception", exception);
}
+ @Test
+ public void testRouterManualSafeMode() throws Exception {
+ InetSocketAddress adminAddr = router.getAdminServerAddress();
+ conf.setSocketAddr(RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, adminAddr);
+ RouterAdmin admin = new RouterAdmin(conf);
+
+ assertTrue(router.getSafemodeService().isInSafeMode());
+ verifyRouter(RouterServiceState.SAFEMODE);
+
+ // Wait until the Router exit start up safe mode
+ long interval = conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION,
+ TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS) + 300;
+ Thread.sleep(interval);
+ verifyRouter(RouterServiceState.RUNNING);
+
+ // Now enter safe mode via Router admin command - it should work
+ assertEquals(0, ToolRunner.run(admin, new String[] {"-safemode", "enter"}));
+ verifyRouter(RouterServiceState.SAFEMODE);
+
+ // Wait for update interval of the safe mode service, it should still in
+ // safe mode.
+ interval = 2 * conf.getTimeDuration(
+ DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, TimeUnit.SECONDS.toMillis(1),
+ TimeUnit.MILLISECONDS);
+ Thread.sleep(interval);
+ verifyRouter(RouterServiceState.SAFEMODE);
+
+ // Exit safe mode via admin command
+ assertEquals(0, ToolRunner.run(admin, new String[] {"-safemode", "leave"}));
+ verifyRouter(RouterServiceState.RUNNING);
+ }
+
private void verifyRouter(RouterServiceState status)
throws IllegalStateException, IOException {
assertEquals(status, router.getRouterState());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] hadoop git commit: HDFS-13448. HDFS Block Placement - Ignore
Locality for First Block Replica (Contributed by BELUGA BEHR via Daniel
Templeton)
Posted by ey...@apache.org.
HDFS-13448. HDFS Block Placement - Ignore Locality for First Block Replica
(Contributed by BELUGA BEHR via Daniel Templeton)
Change-Id: I965d1cfa642ad24296038b83e3d5c9983545267d
(cherry picked from commit 849c45db187224095b13fe297a4d7377fbb9d2cd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00c476ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00c476ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00c476ab
Branch: refs/remotes/origin/branch-3.1
Commit: 00c476abd8f1d34414b646219856859477558458
Parents: 7e7792d
Author: Daniel Templeton <te...@apache.org>
Authored: Tue Jul 24 15:34:19 2018 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Tue Jul 24 16:12:43 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/CreateFlag.java | 9 ++-
.../org/apache/hadoop/hdfs/AddBlockFlag.java | 11 ++-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 3 +
.../hadoop/hdfs/DistributedFileSystem.java | 11 +++
.../src/main/proto/ClientNamenodeProtocol.proto | 1 +
.../BlockPlacementPolicyDefault.java | 4 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 30 +++++---
.../server/namenode/TestFSDirWriteFileOp.java | 79 ++++++++++++++++++++
8 files changed, 134 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c476ab/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
index 383d65a..c3e088b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
@@ -116,7 +116,14 @@ public enum CreateFlag {
* Enforce the file to be a replicated file, no matter what its parent
* directory's replication or erasure coding policy is.
*/
- SHOULD_REPLICATE((short) 0x80);
+ SHOULD_REPLICATE((short) 0x80),
+
+ /**
+ * Advise that the first block replica NOT take into account DataNode
+ * locality. The first block replica should be placed randomly within the
+ * cluster. Subsequent block replicas should follow DataNode locality rules.
+ */
+ IGNORE_CLIENT_LOCALITY((short) 0x100);
private final short mode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c476ab/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/AddBlockFlag.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/AddBlockFlag.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/AddBlockFlag.java
index 6a0805b..b0686d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/AddBlockFlag.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/AddBlockFlag.java
@@ -36,7 +36,16 @@ public enum AddBlockFlag {
*
* @see CreateFlag#NO_LOCAL_WRITE
*/
- NO_LOCAL_WRITE((short) 0x01);
+ NO_LOCAL_WRITE((short) 0x01),
+
+ /**
+ * Advise that the first block replica NOT take into account DataNode
+ * locality. The first block replica should be placed randomly within the
+ * cluster. Subsequent block replicas should follow DataNode locality rules.
+ *
+ * @see CreateFlag#IGNORE_CLIENT_LOCALITY
+ */
+ IGNORE_CLIENT_LOCALITY((short) 0x02);
private final short mode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c476ab/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 9734752..e977054 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -201,6 +201,9 @@ public class DFSOutputStream extends FSOutputSummer
if (flag.contains(CreateFlag.NO_LOCAL_WRITE)) {
this.addBlockFlags.add(AddBlockFlag.NO_LOCAL_WRITE);
}
+ if (flag.contains(CreateFlag.IGNORE_CLIENT_LOCALITY)) {
+ this.addBlockFlags.add(AddBlockFlag.IGNORE_CLIENT_LOCALITY);
+ }
if (progress != null) {
DFSClient.LOG.debug("Set non-null progress callback on DFSOutputStream "
+"{}", src);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c476ab/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 82cdd8c..3519c60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -3205,6 +3205,17 @@ public class DistributedFileSystem extends FileSystem
return this;
}
+ /**
+ * Advise that the first block replica be written without regard to the
+ * client locality.
+ *
+ * @see CreateFlag for the details.
+ */
+ public HdfsDataOutputStreamBuilder ignoreClientLocality() {
+ getFlags().add(CreateFlag.IGNORE_CLIENT_LOCALITY);
+ return this;
+ }
+
@VisibleForTesting
@Override
protected EnumSet<CreateFlag> getFlags() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c476ab/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 0f5ce94..e51aeda 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -167,6 +167,7 @@ message AbandonBlockResponseProto { // void response
enum AddBlockFlagProto {
NO_LOCAL_WRITE = 1; // avoid writing to local node.
+ IGNORE_CLIENT_LOCALITY = 2; // write to a random node
}
message AddBlockRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c476ab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 25c7cfc..e2a9c55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -275,7 +275,9 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
if (avoidLocalNode) {
results = new ArrayList<>(chosenStorage);
Set<Node> excludedNodeCopy = new HashSet<>(excludedNodes);
- excludedNodeCopy.add(writer);
+ if (writer != null) {
+ excludedNodeCopy.add(writer);
+ }
localNode = chooseTarget(numOfReplicas, writer,
excludedNodeCopy, blocksize, maxNodesPerRack, results,
avoidStaleNodes, storagePolicy,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c476ab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 8f34e1c..95b1dae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -270,19 +270,27 @@ class FSDirWriteFileOp {
BlockManager bm, String src, DatanodeInfo[] excludedNodes,
String[] favoredNodes, EnumSet<AddBlockFlag> flags,
ValidateAddBlockResult r) throws IOException {
- Node clientNode = bm.getDatanodeManager()
- .getDatanodeByHost(r.clientMachine);
- if (clientNode == null) {
- clientNode = getClientNode(bm, r.clientMachine);
- }
+ Node clientNode = null;
- Set<Node> excludedNodesSet = null;
- if (excludedNodes != null) {
- excludedNodesSet = new HashSet<>(excludedNodes.length);
- Collections.addAll(excludedNodesSet, excludedNodes);
+ boolean ignoreClientLocality = (flags != null
+ && flags.contains(AddBlockFlag.IGNORE_CLIENT_LOCALITY));
+
+ // If client locality is ignored, clientNode remains 'null' to indicate
+ if (!ignoreClientLocality) {
+ clientNode = bm.getDatanodeManager().getDatanodeByHost(r.clientMachine);
+ if (clientNode == null) {
+ clientNode = getClientNode(bm, r.clientMachine);
+ }
}
- List<String> favoredNodesList = (favoredNodes == null) ? null
- : Arrays.asList(favoredNodes);
+
+ Set<Node> excludedNodesSet =
+ (excludedNodes == null) ? new HashSet<>()
+ : new HashSet<>(Arrays.asList(excludedNodes));
+
+ List<String> favoredNodesList =
+ (favoredNodes == null) ? Collections.emptyList()
+ : Arrays.asList(favoredNodes);
+
// choose targets for the new block to be allocated.
return bm.chooseTarget4NewBlock(src, r.numTargets, clientNode,
excludedNodesSet, r.blockSize,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c476ab/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirWriteFileOp.java
new file mode 100644
index 0000000..762fa61
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirWriteFileOp.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyByte;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anySet;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.EnumSet;
+
+import org.apache.hadoop.hdfs.AddBlockFlag;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.ValidateAddBlockResult;
+import org.apache.hadoop.net.Node;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+public class TestFSDirWriteFileOp {
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testIgnoreClientLocality() throws IOException {
+ ValidateAddBlockResult addBlockResult =
+ new ValidateAddBlockResult(1024L, 3, (byte) 0x01, null, null, null);
+
+ EnumSet<AddBlockFlag> addBlockFlags =
+ EnumSet.of(AddBlockFlag.IGNORE_CLIENT_LOCALITY);
+
+ BlockManager bmMock = mock(BlockManager.class);
+
+ ArgumentCaptor<Node> nodeCaptor = ArgumentCaptor.forClass(Node.class);
+
+ when(bmMock.chooseTarget4NewBlock(anyString(), anyInt(), any(), anySet(),
+ anyLong(), anyList(), anyByte(), any(), any(), any())).thenReturn(null);
+
+ FSDirWriteFileOp.chooseTargetForNewBlock(bmMock, "localhost", null, null,
+ addBlockFlags, addBlockResult);
+
+ // There should be no other interactions with the block manager when the
+ // IGNORE_CLIENT_LOCALITY is passed in because there is no need to discover
+ // the local node requesting the new block
+ verify(bmMock, times(1)).chooseTarget4NewBlock(anyString(), anyInt(),
+ nodeCaptor.capture(), anySet(), anyLong(), anyList(), anyByte(), any(),
+ any(), any());
+
+ verifyNoMoreInteractions(bmMock);
+
+ assertNull(
+ "Source node was assigned a value. Expected 'null' value because "
+ + "chooseTarget was flagged to ignore source node locality",
+ nodeCaptor.getValue());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] hadoop git commit: YARN-8518. test-container-executor
test_is_empty() is broken (Jim_Brennan via rkanter)
Posted by ey...@apache.org.
YARN-8518. test-container-executor test_is_empty() is broken (Jim_Brennan via rkanter)
(cherry picked from commit 1bc106a738a6ce4f7ed025d556bb44c1ede022e3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dfa71428
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dfa71428
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dfa71428
Branch: refs/remotes/origin/branch-3.1
Commit: dfa71428ea19835ba84d97f98ca78ec02790a209
Parents: 1c7d916
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Jul 12 16:38:46 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Jul 18 16:07:48 2018 -0700
----------------------------------------------------------------------
.../container-executor/test/test-container-executor.c | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfa71428/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index a199d84..5607823 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -1203,19 +1203,23 @@ void test_trim_function() {
free(trimmed);
}
+int is_empty(char *name);
+
void test_is_empty() {
printf("\nTesting is_empty function\n");
if (is_empty("/")) {
printf("FAIL: / should not be empty\n");
exit(1);
}
- if (is_empty("/tmp/2938rf2983hcqnw8ud/noexist")) {
- printf("FAIL: /tmp/2938rf2983hcqnw8ud/noexist should not exist\n");
+ char *noexist = TEST_ROOT "/noexist";
+ if (is_empty(noexist)) {
+ printf("%s should not exist\n", noexist);
exit(1);
}
- mkdir("/tmp/2938rf2983hcqnw8ud/emptydir", S_IRWXU);
- if (!is_empty("/tmp/2938rf2983hcqnw8ud/emptydir")) {
- printf("FAIL: /tmp/2938rf2983hcqnw8ud/emptydir be empty\n");
+ char *emptydir = TEST_ROOT "/emptydir";
+ mkdir(emptydir, S_IRWXU);
+ if (!is_empty(emptydir)) {
+ printf("FAIL: %s should be empty\n", emptydir);
exit(1);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] hadoop git commit: YARN-7748.
TestContainerResizing.testIncreaseContainerUnreservedWhenApplicationCompleted
fails due to multiple container fail events. Contributed by Weiwei Yang.
Posted by ey...@apache.org.
YARN-7748. TestContainerResizing.testIncreaseContainerUnreservedWhenApplicationCompleted fails due to multiple container fail events. Contributed by Weiwei Yang.
(cherry picked from commit 35ce6eb1f526ce3db7e015fb1761eee15604100c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4488fd82
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4488fd82
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4488fd82
Branch: refs/remotes/origin/branch-3.1
Commit: 4488fd8295011b37f683c964ae2012fe1b6a4044
Parents: a684a2e
Author: Sunil G <su...@apache.org>
Authored: Tue Jul 24 22:20:06 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Jul 24 22:21:15 2018 +0530
----------------------------------------------------------------------
.../scheduler/capacity/TestContainerResizing.java | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4488fd82/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
index eacbf6e..307d5ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
@@ -58,7 +59,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica
.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -740,11 +740,14 @@ public class TestContainerResizing {
@Test
public void testIncreaseContainerUnreservedWhenApplicationCompleted()
throws Exception {
+ // Disable relaunch app attempt on failure, in order to check
+ // resource usages for current app only.
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
/**
* Similar to testIncreaseContainerUnreservedWhenContainerCompleted, when
* application finishes, reserved increase container should be cancelled
*/
- MockRM rm1 = new MockRM() {
+ MockRM rm1 = new MockRM(conf) {
@Override
public RMNodeLabelsManager createNodeLabelManager() {
return mgr;
@@ -807,9 +810,14 @@ public class TestContainerResizing {
Assert.assertEquals(6 * GB,
app.getAppAttemptResourceUsage().getReserved().getMemorySize());
- // Kill the application
- cs.handle(new AppAttemptRemovedSchedulerEvent(am1.getApplicationAttemptId(),
- RMAppAttemptState.KILLED, false));
+ // Kill the application by killing the AM container
+ ContainerId amContainer =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ cs.killContainer(cs.getRMContainer(amContainer));
+ rm1.waitForState(am1.getApplicationAttemptId(),
+ RMAppAttemptState.FAILED);
+ rm1.waitForState(am1.getApplicationAttemptId().getApplicationId(),
+ RMAppState.FAILED);
/* Check statuses after reservation satisfied */
// Increase request should be unreserved
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] hadoop git commit: HADOOP-15598. DataChecksum calculate
checksum is contented on hashtable synchronization. Contributed by Prasanth
Jayachandran.
Posted by ey...@apache.org.
HADOOP-15598. DataChecksum calculate checksum is contented on hashtable synchronization. Contributed by Prasanth Jayachandran.
(cherry picked from commit 0c7a578927032d5d1ef3469283d7d1fb7dee2a56)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34f1dd03
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34f1dd03
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34f1dd03
Branch: refs/remotes/origin/branch-3.1
Commit: 34f1dd03ee0c31173eb3a0b88a5fad7796a30c24
Parents: 44beab0
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Jul 16 11:32:45 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Jul 16 11:33:43 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/util/NativeCrc32.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34f1dd03/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java
index 0669b0a..3142df2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java
@@ -28,12 +28,12 @@ import com.google.common.annotations.VisibleForTesting;
* natively.
*/
class NativeCrc32 {
-
+ private static final boolean isSparc = System.getProperty("os.arch").toLowerCase().startsWith("sparc");
/**
* Return true if the JNI-based native CRC extensions are available.
*/
public static boolean isAvailable() {
- if (System.getProperty("os.arch").toLowerCase().startsWith("sparc")) {
+ if (isSparc) {
return false;
} else {
return NativeCodeLoader.isNativeCodeLoaded();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] hadoop git commit: YARN-6966. NodeManager metrics may return
wrong negative values when NM restart. (Szilard Nemeth via Haibo Chen)
Posted by ey...@apache.org.
YARN-6966. NodeManager metrics may return wrong negative values when NM restart. (Szilard Nemeth via Haibo Chen)
(cherry picked from commit 9d3c39e9dd88b8f32223c01328581bb68507d415)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e7792dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e7792dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e7792dd
Branch: refs/remotes/origin/branch-3.1
Commit: 7e7792dd7b4d97a10af1dd583fc65214b4b9c009
Parents: 4488fd8
Author: Haibo Chen <ha...@apache.org>
Authored: Mon Jul 23 11:06:44 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Tue Jul 24 12:50:43 2018 -0700
----------------------------------------------------------------------
.../containermanager/ContainerManagerImpl.java | 2 +-
.../scheduler/ContainerScheduler.java | 16 ++++--
.../recovery/NMLeveldbStateStoreService.java | 32 ++++++-----
.../recovery/NMNullStateStoreService.java | 2 +-
.../recovery/NMStateStoreService.java | 3 +-
.../BaseContainerManagerTest.java | 2 +-
.../TestContainerManagerRecovery.java | 57 ++++++++++++++++++++
.../TestContainerSchedulerRecovery.java | 46 +++++++++++-----
.../metrics/TestNodeManagerMetrics.java | 4 +-
.../recovery/NMMemoryStateStoreService.java | 16 +++++-
.../TestNMLeveldbStateStoreService.java | 21 +++++++-
11 files changed, 163 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 26d06aa..ce240bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -496,7 +496,7 @@ public class ContainerManagerImpl extends CompositeService implements
Container container = new ContainerImpl(getConfig(), dispatcher,
launchContext, credentials, metrics, token, context, rcs);
context.getContainers().put(token.getContainerID(), container);
- containerScheduler.recoverActiveContainer(container, rcs.getStatus());
+ containerScheduler.recoverActiveContainer(container, rcs);
app.handle(new ApplicationContainerInitEvent(container));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 57368ab..e6b0f46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -41,6 +41,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService
+ .RecoveredContainerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -229,11 +232,11 @@ public class ContainerScheduler extends AbstractService implements
* @param rcs Recovered Container status
*/
public void recoverActiveContainer(Container container,
- RecoveredContainerStatus rcs) {
+ RecoveredContainerState rcs) {
ExecutionType execType =
container.getContainerTokenIdentifier().getExecutionType();
- if (rcs == RecoveredContainerStatus.QUEUED
- || rcs == RecoveredContainerStatus.PAUSED) {
+ if (rcs.getStatus() == RecoveredContainerStatus.QUEUED
+ || rcs.getStatus() == RecoveredContainerStatus.PAUSED) {
if (execType == ExecutionType.GUARANTEED) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
} else if (execType == ExecutionType.OPPORTUNISTIC) {
@@ -244,10 +247,15 @@ public class ContainerScheduler extends AbstractService implements
"UnKnown execution type received " + container.getContainerId()
+ ", execType " + execType);
}
- } else if (rcs == RecoveredContainerStatus.LAUNCHED) {
+ } else if (rcs.getStatus() == RecoveredContainerStatus.LAUNCHED) {
runningContainers.put(container.getContainerId(), container);
utilizationTracker.addContainerResources(container);
}
+ if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED
+ && rcs.getCapability() != null) {
+ metrics.launchedContainer();
+ metrics.allocateContainer(rcs.getCapability());
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 6f643b0..44f5e18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.fusesource.leveldbjni.JniDBFactory;
@@ -237,7 +238,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
iter.seek(bytes(CONTAINERS_KEY_PREFIX));
while (iter.hasNext()) {
- Entry<byte[],byte[]> entry = iter.peekNext();
+ Entry<byte[], byte[]> entry = iter.peekNext();
String key = asString(entry.getKey());
if (!key.startsWith(CONTAINERS_KEY_PREFIX)) {
break;
@@ -299,6 +300,10 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
if (suffix.equals(CONTAINER_REQUEST_KEY_SUFFIX)) {
rcs.startRequest = new StartContainerRequestPBImpl(
StartContainerRequestProto.parseFrom(entry.getValue()));
+ ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
+ .newContainerTokenIdentifier(rcs.startRequest.getContainerToken());
+ rcs.capability = new ResourcePBImpl(
+ containerTokenIdentifier.getProto().getResource());
} else if (suffix.equals(CONTAINER_VERSION_KEY_SUFFIX)) {
rcs.version = Integer.parseInt(asString(entry.getValue()));
} else if (suffix.equals(CONTAINER_START_TIME_KEY_SUFFIX)) {
@@ -382,24 +387,25 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
LOG.debug("storeContainer: containerId= " + idStr
+ ", startRequest= " + startRequest);
}
- String keyRequest = getContainerKey(idStr, CONTAINER_REQUEST_KEY_SUFFIX);
- String keyVersion = getContainerVersionKey(idStr);
- String keyStartTime =
+ final String keyVersion = getContainerVersionKey(idStr);
+ final String keyRequest =
+ getContainerKey(idStr, CONTAINER_REQUEST_KEY_SUFFIX);
+ final StartContainerRequestProto startContainerRequest =
+ ((StartContainerRequestPBImpl) startRequest).getProto();
+
+ final String keyStartTime =
getContainerKey(idStr, CONTAINER_START_TIME_KEY_SUFFIX);
+ final String startTimeValue = Long.toString(startTime);
+
try {
- WriteBatch batch = db.createWriteBatch();
- try {
- batch.put(bytes(keyRequest),
- ((StartContainerRequestPBImpl) startRequest).getProto().
- toByteArray());
- batch.put(bytes(keyStartTime), bytes(Long.toString(startTime)));
+ try (WriteBatch batch = db.createWriteBatch()) {
+ batch.put(bytes(keyRequest), startContainerRequest.toByteArray());
+ batch.put(bytes(keyStartTime), bytes(startTimeValue));
if (containerVersion != 0) {
batch.put(bytes(keyVersion),
- bytes(Integer.toString(containerVersion)));
+ bytes(Integer.toString(containerVersion)));
}
db.write(batch);
- } finally {
- batch.close();
}
} catch (DBException e) {
markStoreUnHealthy(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
index f217f2f..dfad9cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
@@ -73,7 +73,7 @@ public class NMNullStateStoreService extends NMStateStoreService {
@Override
public void storeContainer(ContainerId containerId, int version,
- long startTime, StartContainerRequest startRequest) throws IOException {
+ long startTime, StartContainerRequest startRequest) {
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
index 0ea0ef3..70decdb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
@@ -416,7 +416,8 @@ public abstract class NMStateStoreService extends AbstractService {
* @throws IOException
*/
public abstract void storeContainer(ContainerId containerId,
- int containerVersion, long startTime, StartContainerRequest startRequest)
+ int containerVersion, long startTime,
+ StartContainerRequest startRequest)
throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index b31601c..493aa4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -107,7 +107,7 @@ public abstract class BaseContainerManagerTest {
protected static File remoteLogDir;
protected static File tmpDir;
- protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
+ protected NodeManagerMetrics metrics = NodeManagerMetrics.create();
public BaseContainerManagerTest() throws UnsupportedFileSystemException {
localFS = FileContext.getLocalFSFileContext();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index 0a834af..a144adf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -106,6 +107,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerScheduler;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.TestNodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
@@ -401,6 +403,61 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
}
@Test
+ public void testNodeManagerMetricsRecovery() throws Exception {
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
+
+ NMStateStoreService stateStore = new NMMemoryStateStoreService();
+ stateStore.init(conf);
+ stateStore.start();
+ Context context = createContext(conf, stateStore);
+ ContainerManagerImpl cm = createContainerManager(context, delSrvc);
+ cm.init(conf);
+ cm.start();
+ metrics.addResource(Resource.newInstance(10240, 8));
+
+ // add an application by starting a container
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId cid = ContainerId.newContainerId(attemptId, 1);
+ Map<String, String> containerEnv = Collections.emptyMap();
+ Map<String, ByteBuffer> serviceData = Collections.emptyMap();
+ Map<String, LocalResource> localResources = Collections.emptyMap();
+ List<String> commands = Arrays.asList("sleep 60s".split(" "));
+ ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
+ localResources, containerEnv, commands, serviceData,
+ null, null);
+ StartContainersResponse startResponse = startContainer(context, cm, cid,
+ clc, null, ContainerType.TASK);
+ assertTrue(startResponse.getFailedRequests().isEmpty());
+ assertEquals(1, context.getApplications().size());
+ Application app = context.getApplications().get(appId);
+ assertNotNull(app);
+
+ // make sure the container reaches RUNNING state
+ waitForNMContainerState(cm, cid,
+ org.apache.hadoop.yarn.server.nodemanager
+ .containermanager.container.ContainerState.RUNNING);
+ TestNodeManagerMetrics.checkMetrics(1, 0, 0, 0, 0, 1, 1, 1, 9, 1, 7);
+
+ // restart and verify metrics could be recovered
+ cm.stop();
+ DefaultMetricsSystem.shutdown();
+ metrics = NodeManagerMetrics.create();
+ metrics.addResource(Resource.newInstance(10240, 8));
+ TestNodeManagerMetrics.checkMetrics(0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8);
+ context = createContext(conf, stateStore);
+ cm = createContainerManager(context, delSrvc);
+ cm.init(conf);
+ cm.start();
+ assertEquals(1, context.getApplications().size());
+ app = context.getApplications().get(appId);
+ assertNotNull(app);
+ TestNodeManagerMetrics.checkMetrics(1, 0, 0, 0, 0, 1, 1, 1, 9, 1, 7);
+ cm.stop();
+ }
+
+ @Test
public void testContainerResizeRecovery() throws Exception {
conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
index 2ae8b97..6b3ac67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.doNothing;
@@ -31,6 +32,8 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService
+ .RecoveredContainerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
import org.junit.After;
import org.junit.Before;
@@ -71,6 +74,13 @@ public class TestContainerSchedulerRecovery {
private ContainerScheduler spy;
+ private RecoveredContainerState createRecoveredContainerState(
+ RecoveredContainerStatus status) {
+ RecoveredContainerState mockState = mock(RecoveredContainerState.class);
+ when(mockState.getStatus()).thenReturn(status);
+ return mockState;
+ }
+
@Before public void setUp() throws Exception {
MockitoAnnotations.initMocks(this);
spy = spy(tempContainerScheduler);
@@ -94,7 +104,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.QUEUED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.QUEUED);
when(token.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
@@ -113,7 +124,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.QUEUED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.QUEUED);
when(token.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
@@ -132,7 +144,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.PAUSED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.PAUSED);
when(token.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
@@ -151,7 +164,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.PAUSED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.PAUSED);
when(token.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
@@ -170,7 +184,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.LAUNCHED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.LAUNCHED);
when(token.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
@@ -189,7 +204,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.LAUNCHED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.LAUNCHED);
when(token.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
@@ -208,7 +224,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.REQUESTED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.REQUESTED);
when(token.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
@@ -227,7 +244,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.REQUESTED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.REQUESTED);
when(token.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
@@ -246,7 +264,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.COMPLETED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.COMPLETED);
when(token.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
@@ -265,7 +284,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.COMPLETED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.COMPLETED);
when(token.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
@@ -284,7 +304,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.QUEUED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.QUEUED);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
@@ -302,7 +323,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- RecoveredContainerStatus rcs = RecoveredContainerStatus.PAUSED;
+ RecoveredContainerState rcs =
+ createRecoveredContainerState(RecoveredContainerStatus.PAUSED);
when(container.getContainerTokenIdentifier()).thenReturn(token);
spy.recoverActiveContainer(container, rcs);
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
index d21e7ad..c5f80ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
@@ -113,8 +113,8 @@ public class TestNodeManagerMetrics {
assertGauge("AvailableVCores", 19, rb);
}
- private void checkMetrics(int launched, int completed, int failed, int killed,
- int initing, int running, int allocatedGB,
+ public static void checkMetrics(int launched, int completed, int failed,
+ int killed, int initing, int running, int allocatedGB,
int allocatedContainers, int availableGB, int allocatedVCores,
int availableVCores) {
MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
index b67d11f..c5428d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
@@ -45,6 +46,9 @@ import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
+
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+
public class NMMemoryStateStoreService extends NMStateStoreService {
private Map<ApplicationId, ContainerManagerApplicationProto> apps;
private Map<ContainerId, RecoveredContainerState> containerStates;
@@ -132,11 +136,19 @@ public class NMMemoryStateStoreService extends NMStateStoreService {
@Override
public synchronized void storeContainer(ContainerId containerId,
- int version, long startTime, StartContainerRequest startRequest)
- throws IOException {
+ int version, long startTime, StartContainerRequest startRequest) {
RecoveredContainerState rcs = new RecoveredContainerState();
rcs.startRequest = startRequest;
rcs.version = version;
+ try {
+ ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
+ .newContainerTokenIdentifier(startRequest.getContainerToken());
+ rcs.capability =
+ new ResourcePBImpl(containerTokenIdentifier.getProto().getResource());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
rcs.setStartTime(startTime);
containerStates.put(containerId, rcs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e7792dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index 265b3e6..c8c07d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -238,7 +238,9 @@ public class TestNMLeveldbStateStoreService {
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 4);
ContainerId containerId = ContainerId.newContainerId(appAttemptId, 5);
- StartContainerRequest containerReq = createContainerRequest(containerId);
+ Resource containerResource = Resource.newInstance(1024, 2);
+ StartContainerRequest containerReq =
+ createContainerRequest(containerId, containerResource);
// store a container and verify recovered
long containerStartTime = System.currentTimeMillis();
@@ -260,6 +262,7 @@ public class TestNMLeveldbStateStoreService {
assertEquals(false, rcs.getKilled());
assertEquals(containerReq, rcs.getStartRequest());
assertTrue(rcs.getDiagnostics().isEmpty());
+ assertEquals(containerResource, rcs.getCapability());
// store a new container record without StartContainerRequest
ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 6);
@@ -279,6 +282,7 @@ public class TestNMLeveldbStateStoreService {
assertEquals(false, rcs.getKilled());
assertEquals(containerReq, rcs.getStartRequest());
assertTrue(rcs.getDiagnostics().isEmpty());
+ assertEquals(containerResource, rcs.getCapability());
// launch the container, add some diagnostics, and verify recovered
StringBuilder diags = new StringBuilder();
@@ -294,6 +298,7 @@ public class TestNMLeveldbStateStoreService {
assertEquals(false, rcs.getKilled());
assertEquals(containerReq, rcs.getStartRequest());
assertEquals(diags.toString(), rcs.getDiagnostics());
+ assertEquals(containerResource, rcs.getCapability());
// pause the container, and verify recovered
stateStore.storeContainerPaused(containerId);
@@ -395,7 +400,17 @@ public class TestNMLeveldbStateStoreService {
}
private StartContainerRequest createContainerRequest(
+ ContainerId containerId, Resource res) {
+ return createContainerRequestInternal(containerId, res);
+ }
+
+ private StartContainerRequest createContainerRequest(
ContainerId containerId) {
+ return createContainerRequestInternal(containerId, null);
+ }
+
+ private StartContainerRequest createContainerRequestInternal(ContainerId
+ containerId, Resource res) {
LocalResource lrsrc = LocalResource.newInstance(
URL.newInstance("hdfs", "somehost", 12345, "/some/path/to/rsrc"),
LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 123L,
@@ -421,6 +436,10 @@ public class TestNMLeveldbStateStoreService {
localResources, env, containerCmds, serviceData, containerTokens,
acls);
Resource containerRsrc = Resource.newInstance(1357, 3);
+
+ if (res != null) {
+ containerRsrc = res;
+ }
ContainerTokenIdentifier containerTokenId =
new ContainerTokenIdentifier(containerId, "host", "user",
containerRsrc, 9876543210L, 42, 2468, Priority.newInstance(7),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] hadoop git commit: HDFS-13733. RBF: Add Web UI configurations
and descriptions to RBF document. Contributed by Takanobu Asanuma.
Posted by ey...@apache.org.
HDFS-13733. RBF: Add Web UI configurations and descriptions to RBF document. Contributed by Takanobu Asanuma.
(cherry picked from commit 1af87df242c4286474961078d306a5692f85debc)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/228508a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/228508a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/228508a0
Branch: refs/remotes/origin/branch-3.1
Commit: 228508a0eef7015e5e03dec0be71afd12c311d6d
Parents: 53c7d82
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Jul 17 10:45:08 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Tue Jul 17 10:47:07 2018 +0800
----------------------------------------------------------------------
.../src/site/markdown/HDFSRouterFederation.md | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228508a0/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
index 73e0f4a..c5bf5e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
@@ -330,6 +330,18 @@ The administration server to manage the Mount Table.
| dfs.federation.router.admin-bind-host | 0.0.0.0 | The actual address the RPC admin server will bind to. |
| dfs.federation.router.admin.handler.count | 1 | The number of server threads for the router to handle RPC requests from admin. |
+### HTTP Server
+
+The HTTP Server to provide Web UI and the HDFS REST interface ([WebHDFS](../hadoop-hdfs/WebHDFS.html)) for the clients. The default URL is "`http://router_host:50071`".
+
+| Property | Default | Description|
+|:---- |:---- |:---- |
+| dfs.federation.router.http.enable | `true` | If `true`, the HTTP service to handle client requests in the router is enabled. |
+| dfs.federation.router.http-address | 0.0.0.0:50071 | HTTP address that handles the web requests to the Router. |
+| dfs.federation.router.http-bind-host | 0.0.0.0 | The actual address the HTTP server will bind to. |
+| dfs.federation.router.https-address | 0.0.0.0:50072 | HTTPS address that handles the web requests to the Router. |
+| dfs.federation.router.https-bind-host | 0.0.0.0 | The actual address the HTTPS server will bind to. |
+
### State Store
The connection to the State Store and the internal caching at the Router.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] hadoop git commit: HDFS-13524. Occasional "All datanodes are
bad" error in TestLargeBlock#testLargeBlockSize. Contributed by Siyao Meng.
Posted by ey...@apache.org.
HDFS-13524. Occasional "All datanodes are bad" error in TestLargeBlock#testLargeBlockSize. Contributed by Siyao Meng.
(cherry picked from commit 88b2794244d19b6432253eb649a375e5bcdcf964)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac9155d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac9155d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac9155d6
Branch: refs/remotes/origin/branch-3.1
Commit: ac9155d6cb174ba19a8cf64b7d0142e147248e04
Parents: 4898edf
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Jul 16 10:51:23 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Jul 16 10:52:56 2018 -0700
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac9155d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java
index a37da35..ec7a077 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java
@@ -50,6 +50,7 @@ public class TestLargeBlock {
// should we verify the data read back from the file? (slow)
static final boolean verifyData = true;
static final byte[] pattern = { 'D', 'E', 'A', 'D', 'B', 'E', 'E', 'F'};
+ static final int numDatanodes = 3;
// creates a file
static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl,
@@ -158,7 +159,7 @@ public class TestLargeBlock {
* timeout here.
* @throws IOException in case of errors
*/
- @Test (timeout = 900000)
+ @Test (timeout = 1800000)
public void testLargeBlockSize() throws IOException {
final long blockSize = 2L * 1024L * 1024L * 1024L + 512L; // 2GB + 512B
runTest(blockSize);
@@ -175,7 +176,8 @@ public class TestLargeBlock {
final long fileSize = blockSize + 1L;
Configuration conf = new Configuration();
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(numDatanodes).build();
FileSystem fs = cluster.getFileSystem();
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] hadoop git commit: YARN-8434. Update federation documentation
of Nodemanager configurations. Contributed by Bibin A Chundatt.
Posted by ey...@apache.org.
YARN-8434. Update federation documentation of Nodemanager configurations. Contributed by Bibin A Chundatt.
(cherry picked from commit 4523cc5637bc3558aa5796150b358ca8471773bb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/677bbdcd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/677bbdcd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/677bbdcd
Branch: refs/remotes/origin/branch-3.1
Commit: 677bbdcdc439e4bc06cdb34b44fe1d06ab03b1fa
Parents: 7cbb959
Author: bibinchundatt <bi...@apache.org>
Authored: Sun Jul 15 13:53:53 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Sun Jul 15 13:57:14 2018 +0530
----------------------------------------------------------------------
.../hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/677bbdcd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index 953f826..aeb7677 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -267,7 +267,6 @@ These are extra configurations that should appear in the **conf/yarn-site.xml**
|:---- |:---- |
| `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled. |
| `yarn.nodemanager.amrmproxy.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor` | A comma-separated list of interceptors to be run at the amrmproxy. For federation the last step in the pipeline should be the FederationInterceptor. |
-| `yarn.client.failover-proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.|
Optional:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] hadoop git commit: HADOOP-15593. Fixed NPE in UGI
spawnAutoRenewalThreadForUserCreds. Contributed by Gabor Bota
Posted by ey...@apache.org.
HADOOP-15593. Fixed NPE in UGI spawnAutoRenewalThreadForUserCreds.
Contributed by Gabor Bota
(cherry picked from commit 77721f39e26b630352a1f4087524a3fbd21ff06e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a869bd97
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a869bd97
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a869bd97
Branch: refs/remotes/origin/branch-3.1
Commit: a869bd970e832c4d770b3cee6257225260f4d235
Parents: 177f604
Author: Eric Yang <ey...@apache.org>
Authored: Thu Jul 26 18:35:36 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Jul 26 18:47:58 2018 -0400
----------------------------------------------------------------------
.../hadoop/security/UserGroupInformation.java | 179 ++++++++++++-------
.../security/TestUserGroupInformation.java | 38 ++++
2 files changed, 148 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a869bd97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 33a876f..c44ef72 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -40,6 +40,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Date;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.Iterator;
@@ -850,81 +851,121 @@ public class UserGroupInformation {
}
//spawn thread only if we have kerb credentials
- Thread t = new Thread(new Runnable() {
+ KerberosTicket tgt = getTGT();
+ if (tgt == null) {
+ return;
+ }
+ String cmd = conf.get("hadoop.kerberos.kinit.command", "kinit");
+ long nextRefresh = getRefreshTime(tgt);
+ Thread t =
+ new Thread(new AutoRenewalForUserCredsRunnable(tgt, cmd, nextRefresh));
+ t.setDaemon(true);
+ t.setName("TGT Renewer for " + getUserName());
+ t.start();
+ }
+
+ @VisibleForTesting
+ class AutoRenewalForUserCredsRunnable implements Runnable {
+ private KerberosTicket tgt;
+ private RetryPolicy rp;
+ private String kinitCmd;
+ private long nextRefresh;
+ private boolean runRenewalLoop = true;
+
+ AutoRenewalForUserCredsRunnable(KerberosTicket tgt, String kinitCmd,
+ long nextRefresh){
+ this.tgt = tgt;
+ this.kinitCmd = kinitCmd;
+ this.nextRefresh = nextRefresh;
+ this.rp = null;
+ }
+
+ public void setRunRenewalLoop(boolean runRenewalLoop) {
+ this.runRenewalLoop = runRenewalLoop;
+ }
- @Override
- public void run() {
- String cmd = conf.get("hadoop.kerberos.kinit.command", "kinit");
- KerberosTicket tgt = getTGT();
- if (tgt == null) {
+ @Override
+ public void run() {
+ do {
+ try {
+ long now = Time.now();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Current time is " + now);
+ LOG.debug("Next refresh is " + nextRefresh);
+ }
+ if (now < nextRefresh) {
+ Thread.sleep(nextRefresh - now);
+ }
+ String output = Shell.execCommand(kinitCmd, "-R");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Renewed ticket. kinit output: {}", output);
+ }
+ reloginFromTicketCache();
+ tgt = getTGT();
+ if (tgt == null) {
+ LOG.warn("No TGT after renewal. Aborting renew thread for " +
+ getUserName());
+ return;
+ }
+ nextRefresh = Math.max(getRefreshTime(tgt),
+ now + kerberosMinSecondsBeforeRelogin);
+ metrics.renewalFailures.set(0);
+ rp = null;
+ } catch (InterruptedException ie) {
+ LOG.warn("Terminating renewal thread");
return;
- }
- long nextRefresh = getRefreshTime(tgt);
- RetryPolicy rp = null;
- while (true) {
+ } catch (IOException ie) {
+ metrics.renewalFailuresTotal.incr();
+ final long now = Time.now();
+
+ if (tgt.isDestroyed()) {
+ LOG.error("TGT is destroyed. Aborting renew thread for {}.",
+ getUserName());
+ return;
+ }
+
+ long tgtEndTime;
+ // As described in HADOOP-15593 we need to handle the case when
+ // tgt.getEndTime() throws NPE because of JDK issue JDK-8147772
+ // NPE is only possible if this issue is not fixed in the JDK
+ // currently used
try {
- long now = Time.now();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Current time is " + now);
- LOG.debug("Next refresh is " + nextRefresh);
- }
- if (now < nextRefresh) {
- Thread.sleep(nextRefresh - now);
- }
- Shell.execCommand(cmd, "-R");
- if (LOG.isDebugEnabled()) {
- LOG.debug("renewed ticket");
- }
- reloginFromTicketCache();
- tgt = getTGT();
- if (tgt == null) {
- LOG.warn("No TGT after renewal. Aborting renew thread for " +
- getUserName());
- return;
- }
- nextRefresh = Math.max(getRefreshTime(tgt),
- now + kerberosMinSecondsBeforeRelogin);
- metrics.renewalFailures.set(0);
- rp = null;
- } catch (InterruptedException ie) {
- LOG.warn("Terminating renewal thread");
+ tgtEndTime = tgt.getEndTime().getTime();
+ } catch (NullPointerException npe) {
+ LOG.error("NPE thrown while getting KerberosTicket endTime. "
+ + "Aborting renew thread for {}.", getUserName());
+ return;
+ }
+
+ LOG.warn("Exception encountered while running the renewal "
+ + "command for {}. (TGT end time:{}, renewalFailures: {},"
+ + "renewalFailuresTotal: {})", getUserName(), tgtEndTime,
+ metrics.renewalFailures.value(),
+ metrics.renewalFailuresTotal.value(), ie);
+ if (rp == null) {
+ // Use a dummy maxRetries to create the policy. The policy will
+ // only be used to get next retry time with exponential back-off.
+ // The final retry time will be later limited within the
+ // tgt endTime in getNextTgtRenewalTime.
+ rp = RetryPolicies.exponentialBackoffRetry(Long.SIZE - 2,
+ kerberosMinSecondsBeforeRelogin, TimeUnit.MILLISECONDS);
+ }
+ try {
+ nextRefresh = getNextTgtRenewalTime(tgtEndTime, now, rp);
+ } catch (Exception e) {
+ LOG.error("Exception when calculating next tgt renewal time", e);
+ return;
+ }
+ metrics.renewalFailures.incr();
+ // retry until close enough to tgt endTime.
+ if (now > nextRefresh) {
+ LOG.error("TGT is expired. Aborting renew thread for {}.",
+ getUserName());
return;
- } catch (IOException ie) {
- metrics.renewalFailuresTotal.incr();
- final long tgtEndTime = tgt.getEndTime().getTime();
- LOG.warn("Exception encountered while running the renewal "
- + "command for {}. (TGT end time:{}, renewalFailures: {},"
- + "renewalFailuresTotal: {})", getUserName(), tgtEndTime,
- metrics.renewalFailures, metrics.renewalFailuresTotal, ie);
- final long now = Time.now();
- if (rp == null) {
- // Use a dummy maxRetries to create the policy. The policy will
- // only be used to get next retry time with exponential back-off.
- // The final retry time will be later limited within the
- // tgt endTime in getNextTgtRenewalTime.
- rp = RetryPolicies.exponentialBackoffRetry(Long.SIZE - 2,
- kerberosMinSecondsBeforeRelogin, TimeUnit.MILLISECONDS);
- }
- try {
- nextRefresh = getNextTgtRenewalTime(tgtEndTime, now, rp);
- } catch (Exception e) {
- LOG.error("Exception when calculating next tgt renewal time", e);
- return;
- }
- metrics.renewalFailures.incr();
- // retry until close enough to tgt endTime.
- if (now > nextRefresh) {
- LOG.error("TGT is expired. Aborting renew thread for {}.",
- getUserName());
- return;
- }
}
}
- }
- });
- t.setDaemon(true);
- t.setName("TGT Renewer for " + getUserName());
- t.start();
+ } while (runRenewalLoop);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a869bd97/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index 9477990..011e930 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -47,6 +47,7 @@ import org.slf4j.event.Level;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
+import javax.security.auth.kerberos.KerberosTicket;
import javax.security.auth.kerberos.KeyTab;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;
@@ -61,6 +62,7 @@ import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.ConcurrentModificationException;
import java.util.Date;
+import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -88,7 +90,10 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
public class TestUserGroupInformation {
@@ -1211,4 +1216,37 @@ public class TestUserGroupInformation {
barrier.await();
assertSame(testUgi1.getSubject(), blockingLookup.get().getSubject());
}
+
+ @Test
+ public void testKerberosTicketIsDestroyedChecked() throws Exception {
+ // Create UserGroupInformation
+ GenericTestUtils.setLogLevel(UserGroupInformation.LOG, Level.DEBUG);
+ Set<User> users = new HashSet<>();
+ users.add(new User("Foo"));
+ Subject subject =
+ new Subject(true, users, new HashSet<>(), new HashSet<>());
+ UserGroupInformation ugi = spy(new UserGroupInformation(subject));
+
+ // throw IOException in the middle of the autoRenewalForUserCreds
+ doThrow(new IOException()).when(ugi).reloginFromTicketCache();
+
+ // Create and destroy the KerberosTicket, so endTime will be null
+ Date d = new Date();
+ KerberosPrincipal kp = new KerberosPrincipal("Foo");
+ KerberosTicket tgt = spy(new KerberosTicket(new byte[]{}, kp, kp, new
+ byte[]{}, 0, null, d, d, d, d, null));
+ tgt.destroy();
+
+ // run AutoRenewalForUserCredsRunnable with this
+ UserGroupInformation.AutoRenewalForUserCredsRunnable userCredsRunnable =
+ ugi.new AutoRenewalForUserCredsRunnable(tgt,
+ Boolean.toString(Boolean.TRUE), 100);
+
+ // Set the runnable to not to run in a loop
+ userCredsRunnable.setRunRenewalLoop(false);
+ // there should be no exception when calling this
+ userCredsRunnable.run();
+ // isDestroyed should be called at least once
+ Mockito.verify(tgt, atLeastOnce()).isDestroyed();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] hadoop git commit: HADOOP-15384. distcp numListstatusThreads
option doesn't get to -delete scan. Contributed by Steve Loughran.
Posted by ey...@apache.org.
HADOOP-15384. distcp numListstatusThreads option doesn't get to -delete scan.
Contributed by Steve Loughran.
(cherry picked from commit ca8b80bf59c0570bb9172208d3a6c993a6854514)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d54241e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d54241e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d54241e9
Branch: refs/remotes/origin/branch-3.1
Commit: d54241e9c995c8f63e6a6317599b858f486763ce
Parents: 8ad82ea
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jul 10 10:50:40 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jul 10 10:50:40 2018 +0100
----------------------------------------------------------------------
.../java/org/apache/hadoop/tools/DistCpOptions.java | 5 ++++-
.../org/apache/hadoop/tools/mapred/CopyCommitter.java | 13 +++++++++++--
.../tools/contract/AbstractContractDistCpTest.java | 2 +-
3 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d54241e9/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index ea99016..cff04eb 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -387,7 +387,10 @@ public final class DistCpOptions {
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.TRACK_MISSING,
String.valueOf(trackPath));
}
-
+ if (numListstatusThreads > 0) {
+ DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.NUM_LISTSTATUS_THREADS,
+ Integer.toString(numListstatusThreads));
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d54241e9/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
index 07eacb0..38106fa 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
@@ -392,6 +392,9 @@ public class CopyCommitter extends FileOutputCommitter {
Path sourceListing = new Path(conf.get(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH));
FileSystem clusterFS = sourceListing.getFileSystem(conf);
Path sortedSourceListing = DistCpUtils.sortListing(conf, sourceListing);
+ long sourceListingCompleted = System.currentTimeMillis();
+ LOG.info("Source listing completed in {}",
+ formatDuration(sourceListingCompleted - listingStart));
// Similarly, create the listing of target-files. Sort alphabetically.
Path targetListing = new Path(sourceListing.getParent(), "targetListing.seq");
@@ -409,8 +412,8 @@ public class CopyCommitter extends FileOutputCommitter {
// Walk both source and target file listings.
// Delete all from target that doesn't also exist on source.
long deletionStart = System.currentTimeMillis();
- LOG.info("Listing completed in {}",
- formatDuration(deletionStart - listingStart));
+ LOG.info("Destination listing completed in {}",
+ formatDuration(deletionStart - sourceListingCompleted));
long deletedEntries = 0;
long filesDeleted = 0;
@@ -545,9 +548,15 @@ public class CopyCommitter extends FileOutputCommitter {
// Set up options to be the same from the CopyListing.buildListing's
// perspective, so to collect similar listings as when doing the copy
//
+ // thread count is picked up from the job
+ int threads = conf.getInt(DistCpConstants.CONF_LABEL_LISTSTATUS_THREADS,
+ DistCpConstants.DEFAULT_LISTSTATUS_THREADS);
+ LOG.info("Scanning destination directory {} with thread count: {}",
+ targetFinalPath, threads);
DistCpOptions options = new DistCpOptions.Builder(targets, resultNonePath)
.withOverwrite(overwrite)
.withSyncFolder(syncFolder)
+ .withNumListstatusThreads(threads)
.build();
DistCpContext distCpContext = new DistCpContext(options);
distCpContext.setTargetPathExists(targetPathExists);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d54241e9/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
index a5e0a03..1458991 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
@@ -572,7 +572,7 @@ public abstract class AbstractContractDistCpTest
private DistCpOptions buildWithStandardOptions(
DistCpOptions.Builder builder) {
return builder
- .withNumListstatusThreads(8)
+ .withNumListstatusThreads(DistCpOptions.MAX_NUM_LISTSTATUS_THREADS)
.build();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] hadoop git commit: HDFS-13723. Occasional "Should be
different group" error in TestRefreshUserMappings#testGroupMappingRefresh.
Contributed by Siyao Meng.
Posted by ey...@apache.org.
HDFS-13723. Occasional "Should be different group" error in TestRefreshUserMappings#testGroupMappingRefresh. Contributed by Siyao Meng.
(cherry picked from commit 162228e8db937d4bdb9cf61d15ed555f1c96368f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9fa3cb1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9fa3cb1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9fa3cb1
Branch: refs/remotes/origin/branch-3.1
Commit: f9fa3cb1570af6497fdb51f47a457268fbea6bd5
Parents: 99e98bf
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Wed Jul 11 10:02:08 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Wed Jul 11 10:04:07 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/security/Groups.java | 5 ++++-
.../hadoop/security/TestRefreshUserMappings.java | 19 +++++++++++++------
2 files changed, 17 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9fa3cb1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index ad09865..63ec9a5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -73,7 +73,8 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Evolving
public class Groups {
- private static final Logger LOG = LoggerFactory.getLogger(Groups.class);
+ @VisibleForTesting
+ static final Logger LOG = LoggerFactory.getLogger(Groups.class);
private final GroupMappingServiceProvider impl;
@@ -308,6 +309,7 @@ public class Groups {
*/
@Override
public List<String> load(String user) throws Exception {
+ LOG.debug("GroupCacheLoader - load.");
TraceScope scope = null;
Tracer tracer = Tracer.curThreadTracer();
if (tracer != null) {
@@ -346,6 +348,7 @@ public class Groups {
public ListenableFuture<List<String>> reload(final String key,
List<String> oldValue)
throws Exception {
+ LOG.debug("GroupCacheLoader - reload (async).");
if (!reloadGroupsInBackground) {
return super.reload(key, oldValue);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9fa3cb1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
index f511eb1..0e7dfc3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
@@ -45,6 +45,8 @@ import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.slf4j.event.Level;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -93,6 +95,8 @@ public class TestRefreshUserMappings {
FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0");
cluster = new MiniDFSCluster.Builder(config).build();
cluster.waitActive();
+
+ GenericTestUtils.setLogLevel(Groups.LOG, Level.DEBUG);
}
@After
@@ -114,21 +118,24 @@ public class TestRefreshUserMappings {
String [] args = new String[]{"-refreshUserToGroupsMappings"};
Groups groups = Groups.getUserToGroupsMappingService(config);
String user = UserGroupInformation.getCurrentUser().getUserName();
- System.out.println("first attempt:");
+
+ System.out.println("First attempt:");
List<String> g1 = groups.getGroups(user);
String [] str_groups = new String [g1.size()];
g1.toArray(str_groups);
System.out.println(Arrays.toString(str_groups));
- System.out.println("second attempt, should be same:");
+ System.out.println("Second attempt, should be the same:");
List<String> g2 = groups.getGroups(user);
g2.toArray(str_groups);
System.out.println(Arrays.toString(str_groups));
for(int i=0; i<g2.size(); i++) {
assertEquals("Should be same group ", g1.get(i), g2.get(i));
}
+
+ // Test refresh command
admin.run(args);
- System.out.println("third attempt(after refresh command), should be different:");
+ System.out.println("Third attempt(after refresh command), should be different:");
List<String> g3 = groups.getGroups(user);
g3.toArray(str_groups);
System.out.println(Arrays.toString(str_groups));
@@ -137,9 +144,9 @@ public class TestRefreshUserMappings {
g1.get(i).equals(g3.get(i)));
}
- // test time out
- Thread.sleep(groupRefreshTimeoutSec*1100);
- System.out.println("fourth attempt(after timeout), should be different:");
+ // Test timeout
+ Thread.sleep(groupRefreshTimeoutSec * 1500);
+ System.out.println("Fourth attempt(after timeout), should be different:");
List<String> g4 = groups.getGroups(user);
g4.toArray(str_groups);
System.out.println(Arrays.toString(str_groups));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] hadoop git commit: YARN-8544. [DS] AM registration fails when
hadoop authorization is enabled. Contributed by Bibin A Chundatt.
Posted by ey...@apache.org.
YARN-8544. [DS] AM registration fails when hadoop authorization is enabled. Contributed by Bibin A Chundatt.
(cherry picked from commit 84612788339392fcda1aef0e27c43f5c6b2a19e5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0710107f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0710107f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0710107f
Branch: refs/remotes/origin/branch-3.1
Commit: 0710107f8d931d63627d356d6100885696cc8736
Parents: 5aca058
Author: bibinchundatt <bi...@apache.org>
Authored: Tue Jul 24 13:09:17 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Tue Jul 24 13:11:31 2018 +0530
----------------------------------------------------------------------
.../src/main/conf/hadoop-policy.xml | 20 ++++++++++++++++
.../dev-support/findbugs-exclude.xml | 4 ++++
.../hadoop/yarn/conf/YarnConfiguration.java | 7 ++++++
.../yarn/conf/TestYarnConfigurationFields.java | 4 ++++
.../nodemanager/amrmproxy/AMRMProxyService.java | 8 +++++++
.../collectormanager/NMCollectorService.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 2 +-
.../localizer/ResourceLocalizationService.java | 2 +-
.../security/authorize/NMPolicyProvider.java | 25 ++++++++++++++++++--
.../security/authorize/RMPolicyProvider.java | 3 +++
10 files changed, 72 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0710107f/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
index cf3dd1f..bd7c111 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
@@ -242,4 +242,24 @@
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
+
+ <property>
+ <name>security.applicationmaster-nodemanager.applicationmaster.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ApplicationMasterProtocol, used by the Nodemanager
+ and ApplicationMasters to communicate.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.distributedscheduling.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for DistributedSchedulingAMProtocol, used by the Nodemanager
+ and Resourcemanager to communicate.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0710107f/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 5841361..7d40c70 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -461,6 +461,10 @@
<Bug pattern="DC_DOUBLECHECK" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider"/>
+ <Bug pattern="DC_DOUBLECHECK" />
+ </Match>
<!-- ApplicationClassLoader is deprecated and moved to hadoop-common; ignore
warning on the identical name as it should be removed later -->
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0710107f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 586fabf..f7fd4fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2202,6 +2202,9 @@ public class YarnConfiguration extends Configuration {
public static final String
YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL =
"security.applicationmaster.protocol.acl";
+ public static final String
+ YARN_SECURITY_SERVICE_AUTHORIZATION_DISTRIBUTEDSCHEDULING_PROTOCOL =
+ "security.distributedscheduling.protocol.acl";
public static final String
YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL =
@@ -2218,6 +2221,10 @@ public class YarnConfiguration extends Configuration {
YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL =
"security.collector-nodemanager.protocol.acl";
+ public static final String
+ YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_NODEMANAGER_PROTOCOL =
+ "security.applicationmaster-nodemanager.applicationmaster.protocol.acl";
+
/** No. of milliseconds to wait between sending a SIGTERM and SIGKILL
* to a running container */
public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0710107f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index b9ba543..9249ed4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -68,6 +68,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
configurationPropsToSkipCompare.add(YarnConfiguration
.YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL);
+ configurationPropsToSkipCompare.add(YarnConfiguration
+ .YARN_SECURITY_SERVICE_AUTHORIZATION_DISTRIBUTEDSCHEDULING_PROTOCOL);
+ configurationPropsToSkipCompare.add(YarnConfiguration
+ .YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_NODEMANAGER_PROTOCOL);
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
configurationPropsToSkipCompare
.add(YarnConfiguration.RM_RESERVATION_SYSTEM_MAX_PERIODICITY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0710107f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
index 86fbb72..02ff432 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
@@ -70,6 +70,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredAMRMProxyState;
import org.apache.hadoop.yarn.server.nodemanager.scheduler.DistributedScheduler;
+import org.apache.hadoop.yarn.server.nodemanager.security.authorize
+ .NMPolicyProvider;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
@@ -169,6 +171,12 @@ public class AMRMProxyService extends CompositeService implements
listenerEndpoint, serverConf, this.secretManager,
numWorkerThreads);
+ if (conf
+ .getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
+ false)) {
+ this.server.refreshServiceAcl(conf, NMPolicyProvider.getInstance());
+ }
+
this.server.start();
LOG.info("AMRMProxyService listening on address: "
+ this.server.getListenerAddress());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0710107f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index 4648a65..f07ef85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -87,7 +87,7 @@ public class NMCollectorService extends CompositeService implements
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
- server.refreshServiceAcl(conf, new NMPolicyProvider());
+ server.refreshServiceAcl(conf, NMPolicyProvider.getInstance());
}
server.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0710107f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index ad63720..26d06aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -610,7 +610,7 @@ public class ContainerManagerImpl extends CompositeService implements
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- refreshServiceAcls(conf, new NMPolicyProvider());
+ refreshServiceAcls(conf, NMPolicyProvider.getInstance());
}
String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0710107f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index ddae2ae..12251a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -409,7 +409,7 @@ public class ResourceLocalizationService extends CompositeService
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- server.refreshServiceAcl(conf, new NMPolicyProvider());
+ server.refreshServiceAcl(conf, NMPolicyProvider.getInstance());
}
return server;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0710107f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/authorize/NMPolicyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/authorize/NMPolicyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/authorize/NMPolicyProvider.java
index 7b28659..c8986f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/authorize/NMPolicyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/authorize/NMPolicyProvider.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.Service;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB;
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB;
@@ -32,7 +33,24 @@ import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocolPB;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class NMPolicyProvider extends PolicyProvider {
-
+
+ private static NMPolicyProvider nmPolicyProvider = null;
+
+ private NMPolicyProvider() {}
+
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public static NMPolicyProvider getInstance() {
+ if (nmPolicyProvider == null) {
+ synchronized(NMPolicyProvider.class) {
+ if (nmPolicyProvider == null) {
+ nmPolicyProvider = new NMPolicyProvider();
+ }
+ }
+ }
+ return nmPolicyProvider;
+ }
+
private static final Service[] NODE_MANAGER_SERVICES =
new Service[] {
new Service(YarnConfiguration.
@@ -43,7 +61,10 @@ public class NMPolicyProvider extends PolicyProvider {
LocalizationProtocolPB.class),
new Service(YarnConfiguration.
YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL,
- CollectorNodemanagerProtocolPB.class)
+ CollectorNodemanagerProtocolPB.class),
+ new Service(YarnConfiguration.
+ YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_NODEMANAGER_PROTOCOL,
+ ApplicationMasterProtocolPB.class),
};
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0710107f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java
index 8c5efa1..b56ca23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB;
import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB;
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocolPB;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
import org.apache.hadoop.yarn.server.api.ResourceTrackerPB;
@@ -67,6 +68,8 @@ public class RMPolicyProvider extends PolicyProvider {
new Service(
YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL,
ApplicationMasterProtocolPB.class),
+ new Service(YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_DISTRIBUTEDSCHEDULING_PROTOCOL,
+ DistributedSchedulingAMProtocolPB.class),
new Service(
YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL,
ResourceManagerAdministrationProtocolPB.class),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] hadoop git commit: HDFS-12837. Intermittent failure in
TestReencryptionWithKMS.
Posted by ey...@apache.org.
HDFS-12837. Intermittent failure in TestReencryptionWithKMS.
(cherry picked from commit b37074be5ab35c238e18bb9c3b89db6d7f8d0986)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5d98755
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5d98755
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5d98755
Branch: refs/remotes/origin/branch-3.1
Commit: d5d987550a2aa089761440f1e64be59eb446c2a7
Parents: fe256a9
Author: Xiao Chen <xi...@apache.org>
Authored: Wed Jul 11 20:54:37 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Wed Jul 11 21:03:45 2018 -0700
----------------------------------------------------------------------
.../server/namenode/ReencryptionHandler.java | 4 +-
.../hdfs/server/namenode/TestReencryption.java | 61 +++++++++++---------
2 files changed, 37 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d98755/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
index 5b52c82..b92fe9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
@@ -616,7 +616,9 @@ public class ReencryptionHandler implements Runnable {
while (shouldPauseForTesting) {
LOG.info("Sleeping in the re-encrypt handler for unit test.");
synchronized (reencryptionHandler) {
- reencryptionHandler.wait(30000);
+ if (shouldPauseForTesting) {
+ reencryptionHandler.wait(30000);
+ }
}
LOG.info("Continuing re-encrypt handler after pausing.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d98755/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
index 7685f31..2222647 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
@@ -67,6 +67,7 @@ import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -207,8 +208,7 @@ public class TestReencryption {
ZoneReencryptionStatus zs = it.next();
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
assertNotEquals(fei0.getEzKeyVersionName(), zs.getEzKeyVersionName());
assertEquals(fei1.getEzKeyVersionName(), zs.getEzKeyVersionName());
assertEquals(10, zs.getFilesReencrypted());
@@ -600,14 +600,27 @@ public class TestReencryption {
final ZoneReencryptionStatus zs = it.next();
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
if (fei != null) {
assertNotEquals(fei.getEzKeyVersionName(), zs.getEzKeyVersionName());
}
assertEquals(expectedFiles, zs.getFilesReencrypted());
}
+ /**
+ * Verify the zone status' completion time is larger than 0, and is no less
+ * than submission time.
+ */
+ private void verifyZoneCompletionTime(final ZoneReencryptionStatus zs) {
+ assertNotNull(zs);
+ assertTrue("Completion time should be positive. " + zs.getCompletionTime(),
+ zs.getCompletionTime() > 0);
+ assertTrue("Completion time " + zs.getCompletionTime()
+ + " should be no less than submission time "
+ + zs.getSubmissionTime(),
+ zs.getCompletionTime() >= zs.getSubmissionTime());
+ }
+
@Test
public void testReencryptLoadedFromFsimage() throws Exception {
/*
@@ -1476,7 +1489,7 @@ public class TestReencryption {
}
@Override
- public void reencryptEncryptedKeys() throws IOException {
+ public synchronized void reencryptEncryptedKeys() throws IOException {
if (exceptionCount > 0) {
exceptionCount--;
try {
@@ -1537,8 +1550,7 @@ public class TestReencryption {
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
assertTrue(zs.isCanceled());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
assertEquals(0, zs.getFilesReencrypted());
assertTrue(getUpdater().isRunning());
@@ -1560,8 +1572,7 @@ public class TestReencryption {
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
assertFalse(zs.isCanceled());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
assertEquals(10, zs.getFilesReencrypted());
}
@@ -1579,8 +1590,7 @@ public class TestReencryption {
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
assertTrue(zs.isCanceled());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
assertEquals(0, zs.getFilesReencrypted());
// verify re-encryption works after restart.
@@ -1592,8 +1602,7 @@ public class TestReencryption {
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
assertFalse(zs.isCanceled());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
assertEquals(10, zs.getFilesReencrypted());
}
@@ -1679,8 +1688,7 @@ public class TestReencryption {
ZoneReencryptionStatus zs = it.next();
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
assertEquals(10, zs.getFilesReencrypted());
}
@@ -1736,7 +1744,7 @@ public class TestReencryption {
}
@Override
- public void reencryptEncryptedKeys() throws IOException {
+ public synchronized void reencryptEncryptedKeys() throws IOException {
if (exceptionCount > 0) {
--exceptionCount;
throw new IOException("Injected KMS failure");
@@ -1772,8 +1780,7 @@ public class TestReencryption {
ZoneReencryptionStatus zs = it.next();
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
assertEquals(5, zs.getFilesReencrypted());
assertEquals(5, zs.getNumReencryptionFailures());
}
@@ -1788,7 +1795,8 @@ public class TestReencryption {
}
@Override
- public void reencryptUpdaterProcessOneTask() throws IOException {
+ public synchronized void reencryptUpdaterProcessOneTask()
+ throws IOException {
if (exceptionCount > 0) {
--exceptionCount;
throw new IOException("Injected process task failure");
@@ -1824,8 +1832,7 @@ public class TestReencryption {
ZoneReencryptionStatus zs = it.next();
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
assertEquals(5, zs.getFilesReencrypted());
assertEquals(1, zs.getNumReencryptionFailures());
}
@@ -1841,7 +1848,8 @@ public class TestReencryption {
}
@Override
- public void reencryptUpdaterProcessCheckpoint() throws IOException {
+ public synchronized void reencryptUpdaterProcessCheckpoint()
+ throws IOException {
if (exceptionCount > 0) {
--exceptionCount;
throw new IOException("Injected process checkpoint failure");
@@ -1877,8 +1885,7 @@ public class TestReencryption {
ZoneReencryptionStatus zs = it.next();
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
assertEquals(10, zs.getFilesReencrypted());
assertEquals(1, zs.getNumReencryptionFailures());
}
@@ -1893,7 +1900,8 @@ public class TestReencryption {
}
@Override
- public void reencryptUpdaterProcessOneTask() throws IOException {
+ public synchronized void reencryptUpdaterProcessOneTask()
+ throws IOException {
if (exceptionCount > 0) {
--exceptionCount;
throw new RetriableException("Injected process task failure");
@@ -1930,8 +1938,7 @@ public class TestReencryption {
ZoneReencryptionStatus zs = it.next();
assertEquals(zone.toString(), zs.getZoneName());
assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
- assertTrue(zs.getCompletionTime() > 0);
- assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ verifyZoneCompletionTime(zs);
assertEquals(10, zs.getFilesReencrypted());
assertEquals(0, zs.getNumReencryptionFailures());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] hadoop git commit: YARN-8330. Improved publishing ALLOCATED
events to ATS. Contributed by Suma Shivaprasad
Posted by ey...@apache.org.
YARN-8330. Improved publishing ALLOCATED events to ATS.
Contributed by Suma Shivaprasad
(cherry picked from commit f93ecf5c1e0b3db27424963814fc01ec43eb76e0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e3807af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e3807af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e3807af
Branch: refs/remotes/origin/branch-3.1
Commit: 8e3807afe007150f3f0ea0ac3a6c48913e31965b
Parents: 964f345
Author: Eric Yang <ey...@apache.org>
Authored: Wed Jul 25 18:49:30 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Jul 25 18:51:42 2018 -0400
----------------------------------------------------------------------
.../rmcontainer/RMContainerImpl.java | 64 +++++++++++---------
.../rmcontainer/TestRMContainerImpl.java | 11 +++-
2 files changed, 43 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e3807af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index efac666..945e7cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -244,23 +244,13 @@ public class RMContainerImpl implements RMContainer {
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
- saveNonAMContainerMetaInfo = rmContext.getYarnConfiguration().getBoolean(
- YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
- YarnConfiguration
- .DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO);
+ saveNonAMContainerMetaInfo =
+ shouldPublishNonAMContainerEventstoATS(rmContext);
if (container.getId() != null) {
rmContext.getRMApplicationHistoryWriter().containerStarted(this);
}
- // If saveNonAMContainerMetaInfo is true, store system metrics for all
- // containers. If false, and if this container is marked as the AM, metrics
- // will still be published for this container, but that calculation happens
- // later.
- if (saveNonAMContainerMetaInfo && null != container.getId()) {
- rmContext.getSystemMetricsPublisher().containerCreated(
- this, this.creationTime);
- }
if (this.container != null) {
this.allocationTags = this.container.getAllocationTags();
}
@@ -590,8 +580,12 @@ public class RMContainerImpl implements RMContainer {
container.getNodeId(), container.getContainerId(),
container.getAllocationTags());
- container.eventHandler.handle(new RMAppAttemptEvent(
- container.appAttemptId, RMAppAttemptEventType.CONTAINER_ALLOCATED));
+ container.eventHandler.handle(
+ new RMAppAttemptEvent(container.appAttemptId,
+ RMAppAttemptEventType.CONTAINER_ALLOCATED));
+
+ publishNonAMContainerEventstoATS(container);
+
}
}
@@ -610,9 +604,11 @@ public class RMContainerImpl implements RMContainer {
// Tell the app
container.eventHandler.handle(new RMAppRunningOnNodeEvent(container
.getApplicationAttemptId().getApplicationId(), container.nodeId));
+
+ publishNonAMContainerEventstoATS(container);
}
}
-
+
private static final class ContainerAcquiredWhileRunningTransition extends
BaseTransition {
@@ -718,17 +714,12 @@ public class RMContainerImpl implements RMContainer {
container);
boolean saveNonAMContainerMetaInfo =
- container.rmContext.getYarnConfiguration().getBoolean(
- YarnConfiguration
- .APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
- YarnConfiguration
- .DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO);
+ shouldPublishNonAMContainerEventstoATS(container.rmContext);
if (saveNonAMContainerMetaInfo || container.isAMContainer()) {
container.rmContext.getSystemMetricsPublisher().containerFinished(
container, container.finishTime);
}
-
}
private static void updateAttemptMetrics(RMContainerImpl container) {
@@ -754,6 +745,29 @@ public class RMContainerImpl implements RMContainer {
}
}
+ private static boolean shouldPublishNonAMContainerEventstoATS(
+ RMContext rmContext) {
+ return rmContext.getYarnConfiguration().getBoolean(
+ YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
+ YarnConfiguration
+ .DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO);
+ }
+
+ private static void publishNonAMContainerEventstoATS(
+ RMContainerImpl rmContainer) {
+ boolean saveNonAMContainerMetaInfo = shouldPublishNonAMContainerEventstoATS(
+ rmContainer.rmContext);
+
+ // If saveNonAMContainerMetaInfo is true, store system metrics for all
+ // containers. If false, and if this container is marked as the AM, metrics
+ // will still be published for this container, but that calculation happens
+ // later.
+ if (saveNonAMContainerMetaInfo && null != rmContainer.container.getId()) {
+ rmContainer.rmContext.getSystemMetricsPublisher().containerCreated(
+ rmContainer, rmContainer.creationTime);
+ }
+ }
+
private static final class KillTransition extends FinishedTransition {
@Override
@@ -884,13 +898,5 @@ public class RMContainerImpl implements RMContainer {
if (containerId != null) {
rmContext.getRMApplicationHistoryWriter().containerStarted(this);
}
- // If saveNonAMContainerMetaInfo is true, store system metrics for all
- // containers. If false, and if this container is marked as the AM, metrics
- // will still be published for this container, but that calculation happens
- // later.
- if (saveNonAMContainerMetaInfo && null != container.getId()) {
- rmContext.getSystemMetricsPublisher().containerCreated(
- this, this.creationTime);
- }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e3807af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 1115e8c..bb6591b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -135,7 +135,6 @@ public class TestRMContainerImpl {
assertEquals(priority,
rmContainer.getAllocatedSchedulerKey().getPriority());
verify(writer).containerStarted(any(RMContainer.class));
- verify(publisher).containerCreated(any(RMContainer.class), anyLong());
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.START));
@@ -150,6 +149,8 @@ public class TestRMContainerImpl {
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
+ verify(publisher, times(2)).containerCreated(any(RMContainer.class),
+ anyLong());
assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
@@ -240,22 +241,25 @@ public class TestRMContainerImpl {
assertEquals(priority,
rmContainer.getAllocatedSchedulerKey().getPriority());
verify(writer).containerStarted(any(RMContainer.class));
- verify(publisher).containerCreated(any(RMContainer.class), anyLong());
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.START));
drainDispatcher.await();
assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
+ verify(publisher).containerCreated(any(RMContainer.class), anyLong());
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.ACQUIRED));
drainDispatcher.await();
assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());
+ verify(publisher, times(2)).containerCreated(any(RMContainer.class),
+ anyLong());
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
+
assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
@@ -340,7 +344,8 @@ public class TestRMContainerImpl {
// RMContainer should be publishing system metrics for all containers.
// Since there is 1 AM container and 1 non-AM container, there should be 2
// container created events and 2 container finished events.
- verify(publisher, times(2)).containerCreated(any(RMContainer.class), anyLong());
+ verify(publisher, times(4)).containerCreated(any(RMContainer.class),
+ anyLong());
verify(publisher, times(2)).containerFinished(any(RMContainer.class), anyLong());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] hadoop git commit: YARN-8473. Containers being launched as
app tears down can leave containers in NEW state. Contributed by Jason Lowe.
Posted by ey...@apache.org.
YARN-8473. Containers being launched as app tears down can leave containers in NEW state. Contributed by Jason Lowe.
(cherry picked from commit 705e2c1f7cba51496b0d019ecedffbe5fb55c28b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f10491e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f10491e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f10491e
Branch: refs/remotes/origin/branch-3.1
Commit: 6f10491e646ca69d05b7c676ba92e17e9a64c3b7
Parents: d54241e
Author: Sunil G <su...@apache.org>
Authored: Tue Jul 10 20:11:47 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Jul 10 20:12:47 2018 +0530
----------------------------------------------------------------------
.../application/ApplicationImpl.java | 36 ++++++++++---
.../application/TestApplication.java | 53 ++++++++++++++++----
2 files changed, 71 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f10491e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 39be7a7..6d84fb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -211,6 +211,9 @@ public class ApplicationImpl implements Application {
private static final ContainerDoneTransition CONTAINER_DONE_TRANSITION =
new ContainerDoneTransition();
+ private static final InitContainerTransition INIT_CONTAINER_TRANSITION =
+ new InitContainerTransition();
+
private static StateMachineFactory<ApplicationImpl, ApplicationState,
ApplicationEventType, ApplicationEvent> stateMachineFactory =
new StateMachineFactory<ApplicationImpl, ApplicationState,
@@ -221,12 +224,12 @@ public class ApplicationImpl implements Application {
ApplicationEventType.INIT_APPLICATION, new AppInitTransition())
.addTransition(ApplicationState.NEW, ApplicationState.NEW,
ApplicationEventType.INIT_CONTAINER,
- new InitContainerTransition())
+ INIT_CONTAINER_TRANSITION)
// Transitions from INITING state
.addTransition(ApplicationState.INITING, ApplicationState.INITING,
ApplicationEventType.INIT_CONTAINER,
- new InitContainerTransition())
+ INIT_CONTAINER_TRANSITION)
.addTransition(ApplicationState.INITING,
EnumSet.of(ApplicationState.FINISHING_CONTAINERS_WAIT,
ApplicationState.APPLICATION_RESOURCES_CLEANINGUP),
@@ -249,7 +252,7 @@ public class ApplicationImpl implements Application {
.addTransition(ApplicationState.RUNNING,
ApplicationState.RUNNING,
ApplicationEventType.INIT_CONTAINER,
- new InitContainerTransition())
+ INIT_CONTAINER_TRANSITION)
.addTransition(ApplicationState.RUNNING,
ApplicationState.RUNNING,
ApplicationEventType.APPLICATION_CONTAINER_FINISHED,
@@ -270,6 +273,10 @@ public class ApplicationImpl implements Application {
new AppFinishTransition())
.addTransition(ApplicationState.FINISHING_CONTAINERS_WAIT,
ApplicationState.FINISHING_CONTAINERS_WAIT,
+ ApplicationEventType.INIT_CONTAINER,
+ INIT_CONTAINER_TRANSITION)
+ .addTransition(ApplicationState.FINISHING_CONTAINERS_WAIT,
+ ApplicationState.FINISHING_CONTAINERS_WAIT,
EnumSet.of(
ApplicationEventType.APPLICATION_LOG_HANDLING_INITED,
ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED,
@@ -286,6 +293,10 @@ public class ApplicationImpl implements Application {
new AppCompletelyDoneTransition())
.addTransition(ApplicationState.APPLICATION_RESOURCES_CLEANINGUP,
ApplicationState.APPLICATION_RESOURCES_CLEANINGUP,
+ ApplicationEventType.INIT_CONTAINER,
+ INIT_CONTAINER_TRANSITION)
+ .addTransition(ApplicationState.APPLICATION_RESOURCES_CLEANINGUP,
+ ApplicationState.APPLICATION_RESOURCES_CLEANINGUP,
EnumSet.of(
ApplicationEventType.APPLICATION_LOG_HANDLING_INITED,
ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED,
@@ -300,9 +311,14 @@ public class ApplicationImpl implements Application {
ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED,
ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED),
new AppLogsAggregatedTransition())
+ .addTransition(ApplicationState.FINISHED,
+ ApplicationState.FINISHED,
+ ApplicationEventType.INIT_CONTAINER,
+ INIT_CONTAINER_TRANSITION)
.addTransition(ApplicationState.FINISHED, ApplicationState.FINISHED,
EnumSet.of(
ApplicationEventType.APPLICATION_LOG_HANDLING_INITED,
+ ApplicationEventType.APPLICATION_CONTAINER_FINISHED,
ApplicationEventType.FINISH_APPLICATION))
// create the topology tables
.installTopology();
@@ -445,8 +461,9 @@ public class ApplicationImpl implements Application {
app.containers.put(container.getContainerId(), container);
LOG.info("Adding " + container.getContainerId()
+ " to application " + app.toString());
-
- switch (app.getApplicationState()) {
+
+ ApplicationState appState = app.getApplicationState();
+ switch (appState) {
case RUNNING:
app.dispatcher.getEventHandler().handle(new ContainerInitEvent(
container.getContainerId()));
@@ -456,8 +473,13 @@ public class ApplicationImpl implements Application {
// these get queued up and sent out in AppInitDoneTransition
break;
default:
- assert false : "Invalid state for InitContainerTransition: " +
- app.getApplicationState();
+ LOG.warn("Killing {} because {} is in state {}",
+ container.getContainerId(), app, appState);
+ app.dispatcher.getEventHandler().handle(new ContainerKillEvent(
+ container.getContainerId(),
+ ContainerExitStatus.KILLED_AFTER_APP_COMPLETION,
+ "Application no longer running.\n"));
+ break;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f10491e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
index c8f28e2..cbe19ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
@@ -360,35 +360,66 @@ public class TestApplication {
}
}
-//TODO Re-work after Application transitions are changed.
-// @Test
+ @Test
@SuppressWarnings("unchecked")
- public void testStartContainerAfterAppFinished() {
+ public void testStartContainerAfterAppRunning() {
WrappedApplication wa = null;
try {
- wa = new WrappedApplication(5, 314159265358979L, "yak", 3);
+ wa = new WrappedApplication(5, 314159265358979L, "yak", 4);
wa.initApplication();
- wa.initContainer(-1);
+ wa.initContainer(0);
assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
wa.applicationInited();
assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
- reset(wa.localizerBus);
- wa.containerFinished(0);
- wa.containerFinished(1);
- wa.containerFinished(2);
assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
- assertEquals(0, wa.app.getContainers().size());
+ assertEquals(1, wa.app.getContainers().size());
wa.appFinished();
+ verify(wa.containerBus).handle(
+ argThat(new ContainerKillMatcher(wa.containers.get(0)
+ .getContainerId())));
+ assertEquals(ApplicationState.FINISHING_CONTAINERS_WAIT,
+ wa.app.getApplicationState());
+
+ wa.initContainer(1);
+ verify(wa.containerBus).handle(
+ argThat(new ContainerKillMatcher(wa.containers.get(1)
+ .getContainerId())));
+ assertEquals(ApplicationState.FINISHING_CONTAINERS_WAIT,
+ wa.app.getApplicationState());
+ wa.containerFinished(1);
+ assertEquals(ApplicationState.FINISHING_CONTAINERS_WAIT,
+ wa.app.getApplicationState());
+
+ wa.containerFinished(0);
assertEquals(ApplicationState.APPLICATION_RESOURCES_CLEANINGUP,
wa.app.getApplicationState());
verify(wa.localizerBus).handle(
refEq(new ApplicationLocalizationEvent(
- LocalizationEventType.DESTROY_APPLICATION_RESOURCES, wa.app)));
+ LocalizationEventType.DESTROY_APPLICATION_RESOURCES,
+ wa.app), "timestamp"));
+
+ wa.initContainer(2);
+ verify(wa.containerBus).handle(
+ argThat(new ContainerKillMatcher(wa.containers.get(2)
+ .getContainerId())));
+ assertEquals(ApplicationState.APPLICATION_RESOURCES_CLEANINGUP,
+ wa.app.getApplicationState());
+ wa.containerFinished(2);
+ assertEquals(ApplicationState.APPLICATION_RESOURCES_CLEANINGUP,
+ wa.app.getApplicationState());
wa.appResourcesCleanedup();
assertEquals(ApplicationState.FINISHED, wa.app.getApplicationState());
+
+ wa.initContainer(3);
+ verify(wa.containerBus).handle(
+ argThat(new ContainerKillMatcher(wa.containers.get(3)
+ .getContainerId())));
+ assertEquals(ApplicationState.FINISHED, wa.app.getApplicationState());
+ wa.containerFinished(3);
+ assertEquals(ApplicationState.FINISHED, wa.app.getApplicationState());
} finally {
if (wa != null)
wa.finished();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] hadoop git commit: HADOOP-15610. Fixed pylint version for
Hadoop docker image. Contributed by Jack Bearden
Posted by ey...@apache.org.
HADOOP-15610. Fixed pylint version for Hadoop docker image.
Contributed by Jack Bearden
(cherry picked from commit ba1ab08fdae96ad7c9c4f4bf8672abd741b7f758)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a607c02f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a607c02f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a607c02f
Branch: refs/remotes/origin/branch-3.1
Commit: a607c02f15f6346d06dd2830737037f8556c7d89
Parents: dfa7142
Author: Eric Yang <ey...@apache.org>
Authored: Wed Jul 18 20:09:43 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Jul 18 20:11:39 2018 -0400
----------------------------------------------------------------------
dev-support/docker/Dockerfile | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a607c02f/dev-support/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 6d05fe1..e1aa00f 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -151,9 +151,10 @@ RUN apt-get -q update && apt-get -q install -y shellcheck
RUN apt-get -q update && apt-get -q install -y bats
####
-# Install pylint (always want latest)
+# Install pylint at fixed version (2.0.0 removed python2 support)
+# https://github.com/PyCQA/pylint/issues/2294
####
-RUN pip2 install pylint
+RUN pip2 install pylint==1.9.2
####
# Install dateutil.parser
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] hadoop git commit: HADOOP-15614.
TestGroupsCaching.testExceptionOnBackgroundRefreshHandled reliably fails.
Contributed by Weiwei Yang.
Posted by ey...@apache.org.
HADOOP-15614. TestGroupsCaching.testExceptionOnBackgroundRefreshHandled reliably fails. Contributed by Weiwei Yang.
(cherry picked from commit ccf2db7fc2688d262df3309007cb12a4dfedc179)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a147098c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a147098c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a147098c
Branch: refs/remotes/origin/branch-3.1
Commit: a147098c4fab435f1c8962e1fa5b22bf6a3b84f0
Parents: a607c02
Author: Kihwal Lee <ki...@apache.org>
Authored: Thu Jul 19 11:19:19 2018 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Thu Jul 19 11:19:19 2018 -0500
----------------------------------------------------------------------
.../apache/hadoop/security/TestGroupsCaching.java | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a147098c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
index 46e36b3..bba8152 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
@@ -561,23 +561,28 @@ public class TestGroupsCaching {
// Then expire that entry
timer.advance(4 * 1000);
+ // Pause the getGroups operation and this will delay the cache refresh
+ FakeGroupMapping.pause();
// Now get the cache entry - it should return immediately
// with the old value and the cache will not have completed
// a request to getGroups yet.
assertEquals(groups.getGroups("me").size(), 2);
assertEquals(startingRequestCount, FakeGroupMapping.getRequestCount());
+ // Resume the getGroups operation and the cache can get refreshed
+ FakeGroupMapping.resume();
- // Now sleep for a short time and re-check the request count. It should have
- // increased, but the exception means the cache will not have updated
- Thread.sleep(50);
+ // Now wait for the refresh done, because of the exception, we expect
+ // a onFailure callback gets called and the counter for failure is 1
+ waitForGroupCounters(groups, 0, 0, 0, 1);
FakeGroupMapping.setThrowException(false);
assertEquals(startingRequestCount + 1, FakeGroupMapping.getRequestCount());
assertEquals(groups.getGroups("me").size(), 2);
- // Now sleep another short time - the 3rd call to getGroups above
- // will have kicked off another refresh that updates the cache
- Thread.sleep(50);
+ // Now the 3rd call to getGroups above will have kicked off
+ // another refresh that updates the cache, since it no longer gives
+ // exception, we now expect the counter for success is 1.
+ waitForGroupCounters(groups, 0, 0, 1, 1);
assertEquals(startingRequestCount + 2, FakeGroupMapping.getRequestCount());
assertEquals(groups.getGroups("me").size(), 3);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] hadoop git commit: HADOOP-15547/ WASB: improve listStatus
performance. Contributed by Thomas Marquardt.
Posted by ey...@apache.org.
HADOOP-15547/ WASB: improve listStatus performance.
Contributed by Thomas Marquardt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/749fff57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/749fff57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/749fff57
Branch: refs/remotes/origin/branch-3.1
Commit: 749fff577ed9afb4ef8a54b8948f74be083cc620
Parents: 76b8beb
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jul 19 12:29:21 2018 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Jul 19 12:29:21 2018 -0700
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 10 +
hadoop-tools/hadoop-azure/pom.xml | 12 +
.../fs/azure/AzureNativeFileSystemStore.java | 182 ++++-----
.../apache/hadoop/fs/azure/FileMetadata.java | 77 ++--
.../hadoop/fs/azure/NativeAzureFileSystem.java | 376 ++++++++-----------
.../hadoop/fs/azure/NativeFileSystemStore.java | 15 +-
.../apache/hadoop/fs/azure/PartialListing.java | 61 ---
.../hadoop/fs/azure/ITestListPerformance.java | 196 ++++++++++
8 files changed, 514 insertions(+), 415 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749fff57/hadoop-tools/hadoop-azure/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-azure/dev-support/findbugs-exclude.xml
index cde1734..38de35e 100644
--- a/hadoop-tools/hadoop-azure/dev-support/findbugs-exclude.xml
+++ b/hadoop-tools/hadoop-azure/dev-support/findbugs-exclude.xml
@@ -47,4 +47,14 @@
<Bug pattern="WMI_WRONG_MAP_ITERATOR" />
<Priority value="2" />
</Match>
+
+ <!-- FileMetadata is used internally for storing metadata but also
+ subclasses FileStatus to reduce allocations when listing a large number
+ of files. When it is returned to an external caller as a FileStatus, the
+ extra metadata is no longer useful and we want the equals and hashCode
+ methods of FileStatus to be used. -->
+ <Match>
+ <Class name="org.apache.hadoop.fs.azure.FileMetadata" />
+ <Bug pattern="EQ_DOESNT_OVERRIDE_EQUALS" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749fff57/hadoop-tools/hadoop-azure/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index 978e89e..7265df9 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -43,6 +43,8 @@
<fs.azure.scale.test.huge.partitionsize>unset</fs.azure.scale.test.huge.partitionsize>
<!-- Timeout in seconds for scale tests.-->
<fs.azure.scale.test.timeout>7200</fs.azure.scale.test.timeout>
+ <fs.azure.scale.test.list.performance.threads>10</fs.azure.scale.test.list.performance.threads>
+ <fs.azure.scale.test.list.performance.files>1000</fs.azure.scale.test.list.performance.files>
</properties>
<build>
@@ -298,6 +300,8 @@
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
<fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+ <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
+ <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
</systemPropertyVariables>
<includes>
<include>**/Test*.java</include>
@@ -326,6 +330,8 @@
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
<fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+ <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
+ <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
</systemPropertyVariables>
<includes>
<include>**/TestRollingWindowAverage*.java</include>
@@ -367,6 +373,8 @@
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
<fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+ <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
+ <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
</systemPropertyVariables>
<!-- Some tests cannot run in parallel. Tests that cover -->
<!-- access to the root directory must run in isolation -->
@@ -412,6 +420,8 @@
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
<fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+ <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
+ <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
</systemPropertyVariables>
<includes>
<include>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</include>
@@ -454,6 +464,8 @@
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+ <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
+ <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
</systemPropertyVariables>
<forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
<trimStackTrace>false</trimStackTrace>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749fff57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 9396a51..74bb035 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -30,7 +30,6 @@ import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.security.InvalidKeyException;
-import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.EnumSet;
@@ -128,6 +127,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// computed as min(2*cpu,8)
private static final String KEY_CONCURRENT_CONNECTION_VALUE_OUT = "fs.azure.concurrentRequestCount.out";
+ private static final String HADOOP_BLOCK_SIZE_PROPERTY_NAME = "fs.azure.block.size";
private static final String KEY_STREAM_MIN_READ_SIZE = "fs.azure.read.request.size";
private static final String KEY_STORAGE_CONNECTION_TIMEOUT = "fs.azure.storage.timeout";
private static final String KEY_WRITE_BLOCK_SIZE = "fs.azure.write.request.size";
@@ -252,6 +252,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// Default block sizes
public static final int DEFAULT_DOWNLOAD_BLOCK_SIZE = 4 * 1024 * 1024;
public static final int DEFAULT_UPLOAD_BLOCK_SIZE = 4 * 1024 * 1024;
+ public static final long DEFAULT_HADOOP_BLOCK_SIZE = 512 * 1024 * 1024L;
private static final int DEFAULT_INPUT_STREAM_VERSION = 2;
@@ -313,6 +314,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
private boolean tolerateOobAppends = DEFAULT_READ_TOLERATE_CONCURRENT_APPEND;
+ private long hadoopBlockSize = DEFAULT_HADOOP_BLOCK_SIZE;
private int downloadBlockSizeBytes = DEFAULT_DOWNLOAD_BLOCK_SIZE;
private int uploadBlockSizeBytes = DEFAULT_UPLOAD_BLOCK_SIZE;
private int inputStreamVersion = DEFAULT_INPUT_STREAM_VERSION;
@@ -740,6 +742,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
KEY_STREAM_MIN_READ_SIZE, DEFAULT_DOWNLOAD_BLOCK_SIZE);
this.uploadBlockSizeBytes = sessionConfiguration.getInt(
KEY_WRITE_BLOCK_SIZE, DEFAULT_UPLOAD_BLOCK_SIZE);
+ this.hadoopBlockSize = sessionConfiguration.getLong(
+ HADOOP_BLOCK_SIZE_PROPERTY_NAME, DEFAULT_HADOOP_BLOCK_SIZE);
this.inputStreamVersion = sessionConfiguration.getInt(
KEY_INPUT_STREAM_VERSION, DEFAULT_INPUT_STREAM_VERSION);
@@ -1234,7 +1238,14 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
return false;
}
-
+ /**
+ * Returns the file block size. This is a fake value used for integration
+ * of the Azure store with Hadoop.
+ */
+ @Override
+ public long getHadoopBlockSize() {
+ return hadoopBlockSize;
+ }
/**
* This should be called from any method that does any modifications to the
@@ -2066,7 +2077,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// The key refers to root directory of container.
// Set the modification time for root to zero.
return new FileMetadata(key, 0, defaultPermissionNoBlobMetadata(),
- BlobMaterialization.Implicit);
+ BlobMaterialization.Implicit, hadoopBlockSize);
}
CloudBlobWrapper blob = getBlobReference(key);
@@ -2086,7 +2097,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
if (retrieveFolderAttribute(blob)) {
LOG.debug("{} is a folder blob.", key);
return new FileMetadata(key, properties.getLastModified().getTime(),
- getPermissionStatus(blob), BlobMaterialization.Explicit);
+ getPermissionStatus(blob), BlobMaterialization.Explicit, hadoopBlockSize);
} else {
LOG.debug("{} is a normal blob.", key);
@@ -2095,7 +2106,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
key, // Always return denormalized key with metadata.
getDataLength(blob, properties),
properties.getLastModified().getTime(),
- getPermissionStatus(blob));
+ getPermissionStatus(blob), hadoopBlockSize);
}
} catch(StorageException e){
if (!NativeAzureFileSystemHelper.isFileNotFoundException(e)) {
@@ -2129,7 +2140,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
BlobProperties properties = blob.getProperties();
return new FileMetadata(key, properties.getLastModified().getTime(),
- getPermissionStatus(blob), BlobMaterialization.Implicit);
+ getPermissionStatus(blob), BlobMaterialization.Implicit, hadoopBlockSize);
}
}
@@ -2178,46 +2189,13 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
}
@Override
- public PartialListing list(String prefix, final int maxListingCount,
+ public FileMetadata[] list(String prefix, final int maxListingCount,
final int maxListingDepth) throws IOException {
- return list(prefix, maxListingCount, maxListingDepth, null);
- }
-
- @Override
- public PartialListing list(String prefix, final int maxListingCount,
- final int maxListingDepth, String priorLastKey) throws IOException {
- return list(prefix, PATH_DELIMITER, maxListingCount, maxListingDepth,
- priorLastKey);
+ return listInternal(prefix, maxListingCount, maxListingDepth);
}
- @Override
- public PartialListing listAll(String prefix, final int maxListingCount,
- final int maxListingDepth, String priorLastKey) throws IOException {
- return list(prefix, null, maxListingCount, maxListingDepth, priorLastKey);
- }
-
- /**
- * Searches the given list of {@link FileMetadata} objects for a directory
- * with the given key.
- *
- * @param list
- * The list to search.
- * @param key
- * The key to search for.
- * @return The wanted directory, or null if not found.
- */
- private static FileMetadata getFileMetadataInList(
- final Iterable<FileMetadata> list, String key) {
- for (FileMetadata current : list) {
- if (current.getKey().equals(key)) {
- return current;
- }
- }
- return null;
- }
-
- private PartialListing list(String prefix, String delimiter,
- final int maxListingCount, final int maxListingDepth, String priorLastKey)
+ private FileMetadata[] listInternal(String prefix, final int maxListingCount,
+ final int maxListingDepth)
throws IOException {
try {
checkContainer(ContainerAccessType.PureRead);
@@ -2241,7 +2219,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
objects = listRootBlobs(prefix, true, enableFlatListing);
}
- ArrayList<FileMetadata> fileMetadata = new ArrayList<FileMetadata>();
+ HashMap<String, FileMetadata> fileMetadata = new HashMap<>(256);
+
for (ListBlobItem blobItem : objects) {
// Check that the maximum listing count is not exhausted.
//
@@ -2261,25 +2240,37 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
FileMetadata metadata;
if (retrieveFolderAttribute(blob)) {
- metadata = new FileMetadata(blobKey,
- properties.getLastModified().getTime(),
- getPermissionStatus(blob),
- BlobMaterialization.Explicit);
+ metadata = new FileMetadata(blobKey,
+ properties.getLastModified().getTime(),
+ getPermissionStatus(blob),
+ BlobMaterialization.Explicit,
+ hadoopBlockSize);
} else {
- metadata = new FileMetadata(
- blobKey,
- getDataLength(blob, properties),
- properties.getLastModified().getTime(),
- getPermissionStatus(blob));
+ metadata = new FileMetadata(
+ blobKey,
+ getDataLength(blob, properties),
+ properties.getLastModified().getTime(),
+ getPermissionStatus(blob),
+ hadoopBlockSize);
}
+ // Add the metadata but remove duplicates. Note that the azure
+ // storage java SDK returns two types of entries: CloudBlobWrappter
+ // and CloudDirectoryWrapper. In the case where WASB generated the
+ // data, there will be an empty blob for each "directory", and we will
+ // receive a CloudBlobWrapper. If there are also files within this
+ // "directory", we will also receive a CloudDirectoryWrapper. To
+ // complicate matters, the data may not be generated by WASB, in
+ // which case we may not have an empty blob for each "directory".
+ // So, sometimes we receive both a CloudBlobWrapper and a
+ // CloudDirectoryWrapper for each directory, and sometimes we receive
+ // one or the other but not both. We remove duplicates, but
+ // prefer CloudBlobWrapper over CloudDirectoryWrapper.
+ // Furthermore, it is very unfortunate that the list results are not
+ // ordered, and it is a partial list which uses continuation. So
+ // the HashMap is the best structure to remove the duplicates, despite
+ // its potential large size.
+ fileMetadata.put(blobKey, metadata);
- // Add the metadata to the list, but remove any existing duplicate
- // entries first that we may have added by finding nested files.
- FileMetadata existing = getFileMetadataInList(fileMetadata, blobKey);
- if (existing != null) {
- fileMetadata.remove(existing);
- }
- fileMetadata.add(metadata);
} else if (blobItem instanceof CloudBlobDirectoryWrapper) {
CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
// Determine format of directory name depending on whether an absolute
@@ -2298,12 +2289,15 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// inherit the permissions of the first non-directory blob.
// Also, getting a proper value for last-modified is tricky.
FileMetadata directoryMetadata = new FileMetadata(dirKey, 0,
- defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
+ defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit,
+ hadoopBlockSize);
// Add the directory metadata to the list only if it's not already
- // there.
- if (getFileMetadataInList(fileMetadata, dirKey) == null) {
- fileMetadata.add(directoryMetadata);
+ // there. See earlier note, we prefer CloudBlobWrapper over
+ // CloudDirectoryWrapper because it may have additional metadata (
+ // properties and ACLs).
+ if (!fileMetadata.containsKey(dirKey)) {
+ fileMetadata.put(dirKey, directoryMetadata);
}
if (!enableFlatListing) {
@@ -2314,13 +2308,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
}
}
}
- // Note: Original code indicated that this may be a hack.
- priorLastKey = null;
- PartialListing listing = new PartialListing(priorLastKey,
- fileMetadata.toArray(new FileMetadata[] {}),
- 0 == fileMetadata.size() ? new String[] {}
- : new String[] { prefix });
- return listing;
+ return fileMetadata.values().toArray(new FileMetadata[fileMetadata.size()]);
} catch (Exception e) {
// Re-throw as an Azure storage exception.
//
@@ -2334,13 +2322,13 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
* the sorted order of the blob names.
*
* @param aCloudBlobDirectory Azure blob directory
- * @param aFileMetadataList a list of file metadata objects for each
+ * @param metadataHashMap a map of file metadata objects for each
* non-directory blob.
* @param maxListingCount maximum length of the built up list.
*/
private void buildUpList(CloudBlobDirectoryWrapper aCloudBlobDirectory,
- ArrayList<FileMetadata> aFileMetadataList, final int maxListingCount,
- final int maxListingDepth) throws Exception {
+ HashMap<String, FileMetadata> metadataHashMap, final int maxListingCount,
+ final int maxListingDepth) throws Exception {
// Push the blob directory onto the stack.
//
@@ -2371,12 +2359,12 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// (2) maxListingCount > 0 implies that the number of items in the
// metadata list is less than the max listing count.
while (null != blobItemIterator
- && (maxListingCount <= 0 || aFileMetadataList.size() < maxListingCount)) {
+ && (maxListingCount <= 0 || metadataHashMap.size() < maxListingCount)) {
while (blobItemIterator.hasNext()) {
// Check if the count of items on the list exhausts the maximum
// listing count.
//
- if (0 < maxListingCount && aFileMetadataList.size() >= maxListingCount) {
+ if (0 < maxListingCount && metadataHashMap.size() >= maxListingCount) {
break;
}
@@ -2399,22 +2387,34 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
metadata = new FileMetadata(blobKey,
properties.getLastModified().getTime(),
getPermissionStatus(blob),
- BlobMaterialization.Explicit);
+ BlobMaterialization.Explicit,
+ hadoopBlockSize);
} else {
metadata = new FileMetadata(
blobKey,
getDataLength(blob, properties),
properties.getLastModified().getTime(),
- getPermissionStatus(blob));
+ getPermissionStatus(blob),
+ hadoopBlockSize);
}
- // Add the directory metadata to the list only if it's not already
- // there.
- FileMetadata existing = getFileMetadataInList(aFileMetadataList, blobKey);
- if (existing != null) {
- aFileMetadataList.remove(existing);
- }
- aFileMetadataList.add(metadata);
+ // Add the metadata but remove duplicates. Note that the azure
+ // storage java SDK returns two types of entries: CloudBlobWrappter
+ // and CloudDirectoryWrapper. In the case where WASB generated the
+ // data, there will be an empty blob for each "directory", and we will
+ // receive a CloudBlobWrapper. If there are also files within this
+ // "directory", we will also receive a CloudDirectoryWrapper. To
+ // complicate matters, the data may not be generated by WASB, in
+ // which case we may not have an empty blob for each "directory".
+ // So, sometimes we receive both a CloudBlobWrapper and a
+ // CloudDirectoryWrapper for each directory, and sometimes we receive
+ // one or the other but not both. We remove duplicates, but
+ // prefer CloudBlobWrapper over CloudDirectoryWrapper.
+ // Furthermore, it is very unfortunate that the list results are not
+ // ordered, and it is a partial list which uses continuation. So
+ // the HashMap is the best structure to remove the duplicates, despite
+ // its potential large size.
+ metadataHashMap.put(blobKey, metadata);
} else if (blobItem instanceof CloudBlobDirectoryWrapper) {
CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
@@ -2439,7 +2439,12 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// absolute path is being used or not.
String dirKey = normalizeKey(directory);
- if (getFileMetadataInList(aFileMetadataList, dirKey) == null) {
+ // Add the directory metadata to the list only if it's not already
+ // there. See earlier note, we prefer CloudBlobWrapper over
+ // CloudDirectoryWrapper because it may have additional metadata (
+ // properties and ACLs).
+ if (!metadataHashMap.containsKey(dirKey)) {
+
// Reached the targeted listing depth. Return metadata for the
// directory using default permissions.
//
@@ -2450,10 +2455,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
FileMetadata directoryMetadata = new FileMetadata(dirKey,
0,
defaultPermissionNoBlobMetadata(),
- BlobMaterialization.Implicit);
+ BlobMaterialization.Implicit,
+ hadoopBlockSize);
// Add the directory metadata to the list.
- aFileMetadataList.add(directoryMetadata);
+ metadataHashMap.put(dirKey, directoryMetadata);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749fff57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java
index 5085a0f..cbf3ab9 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.fs.azure;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.PermissionStatus;
/**
@@ -27,12 +29,9 @@ import org.apache.hadoop.fs.permission.PermissionStatus;
* </p>
*/
@InterfaceAudience.Private
-class FileMetadata {
- private final String key;
- private final long length;
- private final long lastModified;
- private final boolean isDir;
- private final PermissionStatus permissionStatus;
+class FileMetadata extends FileStatus {
+ // this is not final so that it can be cleared to save memory when not needed.
+ private String key;
private final BlobMaterialization blobMaterialization;
/**
@@ -46,16 +45,19 @@ class FileMetadata {
* The last modified date (milliseconds since January 1, 1970 UTC.)
* @param permissionStatus
* The permission for the file.
+ * @param blockSize
+ * The Hadoop file block size.
*/
public FileMetadata(String key, long length, long lastModified,
- PermissionStatus permissionStatus) {
+ PermissionStatus permissionStatus, final long blockSize) {
+ super(length, false, 1, blockSize, lastModified, 0,
+ permissionStatus.getPermission(),
+ permissionStatus.getUserName(),
+ permissionStatus.getGroupName(),
+ null);
this.key = key;
- this.length = length;
- this.lastModified = lastModified;
- this.isDir = false;
- this.permissionStatus = permissionStatus;
- this.blobMaterialization = BlobMaterialization.Explicit; // File are never
- // implicit.
+ // Files are never implicit.
+ this.blobMaterialization = BlobMaterialization.Explicit;
}
/**
@@ -70,37 +72,42 @@ class FileMetadata {
* @param blobMaterialization
* Whether this is an implicit (no real blob backing it) or explicit
* directory.
+ * @param blockSize
+ * The Hadoop file block size.
*/
public FileMetadata(String key, long lastModified,
- PermissionStatus permissionStatus, BlobMaterialization blobMaterialization) {
+ PermissionStatus permissionStatus, BlobMaterialization blobMaterialization,
+ final long blockSize) {
+ super(0, true, 1, blockSize, lastModified, 0,
+ permissionStatus.getPermission(),
+ permissionStatus.getUserName(),
+ permissionStatus.getGroupName(),
+ null);
this.key = key;
- this.isDir = true;
- this.length = 0;
- this.lastModified = lastModified;
- this.permissionStatus = permissionStatus;
this.blobMaterialization = blobMaterialization;
}
- public boolean isDir() {
- return isDir;
+ @Override
+ public Path getPath() {
+ Path p = super.getPath();
+ if (p == null) {
+ // Don't store this yet to reduce memory usage, as it will
+ // stay in the Eden Space and later we will update it
+ // with the full canonicalized path.
+ p = NativeAzureFileSystem.keyToPath(key);
+ }
+ return p;
}
+ /**
+ * Returns the Azure storage key for the file. Used internally by the framework.
+ *
+ * @return The key for the file.
+ */
public String getKey() {
return key;
}
- public long getLength() {
- return length;
- }
-
- public long getLastModified() {
- return lastModified;
- }
-
- public PermissionStatus getPermissionStatus() {
- return permissionStatus;
- }
-
/**
* Indicates whether this is an implicit directory (no real blob backing it)
* or an explicit one.
@@ -112,9 +119,7 @@ class FileMetadata {
return blobMaterialization;
}
- @Override
- public String toString() {
- return "FileMetadata[" + key + ", " + length + ", " + lastModified + ", "
- + permissionStatus + "]";
+ void removeKey() {
+ key = null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749fff57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index dfc881a..efccb96 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -31,9 +31,7 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
-import java.util.Set;
import java.util.TimeZone;
-import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
@@ -129,20 +127,12 @@ public class NativeAzureFileSystem extends FileSystem {
this.dstKey = dstKey;
this.folderLease = lease;
this.fs = fs;
- ArrayList<FileMetadata> fileMetadataList = new ArrayList<FileMetadata>();
// List all the files in the folder.
long start = Time.monotonicNow();
- String priorLastKey = null;
- do {
- PartialListing listing = fs.getStoreInterface().listAll(srcKey, AZURE_LIST_ALL,
- AZURE_UNBOUNDED_DEPTH, priorLastKey);
- for(FileMetadata file : listing.getFiles()) {
- fileMetadataList.add(file);
- }
- priorLastKey = listing.getPriorLastKey();
- } while (priorLastKey != null);
- fileMetadata = fileMetadataList.toArray(new FileMetadata[fileMetadataList.size()]);
+ fileMetadata = fs.getStoreInterface().list(srcKey, AZURE_LIST_ALL,
+ AZURE_UNBOUNDED_DEPTH);
+
long end = Time.monotonicNow();
LOG.debug("Time taken to list {} blobs for rename operation is: {} ms", fileMetadata.length, (end - start));
@@ -669,7 +659,6 @@ public class NativeAzureFileSystem extends FileSystem {
public static final Logger LOG = LoggerFactory.getLogger(NativeAzureFileSystem.class);
- static final String AZURE_BLOCK_SIZE_PROPERTY_NAME = "fs.azure.block.size";
/**
* The time span in seconds before which we consider a temp blob to be
* dangling (not being actively uploaded to) and up for reclamation.
@@ -685,8 +674,6 @@ public class NativeAzureFileSystem extends FileSystem {
private static final int AZURE_LIST_ALL = -1;
private static final int AZURE_UNBOUNDED_DEPTH = -1;
- private static final long MAX_AZURE_BLOCK_SIZE = 512 * 1024 * 1024L;
-
/**
* The configuration property that determines which group owns files created
* in WASB.
@@ -1196,7 +1183,6 @@ public class NativeAzureFileSystem extends FileSystem {
private NativeFileSystemStore store;
private AzureNativeFileSystemStore actualStore;
private Path workingDir;
- private long blockSize = MAX_AZURE_BLOCK_SIZE;
private AzureFileSystemInstrumentation instrumentation;
private String metricsSourceName;
private boolean isClosed = false;
@@ -1361,13 +1347,10 @@ public class NativeAzureFileSystem extends FileSystem {
this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
this.workingDir = new Path("/user", UserGroupInformation.getCurrentUser()
.getShortUserName()).makeQualified(getUri(), getWorkingDirectory());
- this.blockSize = conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME,
- MAX_AZURE_BLOCK_SIZE);
this.appendSupportEnabled = conf.getBoolean(APPEND_SUPPORT_ENABLE_PROPERTY_NAME, false);
LOG.debug("NativeAzureFileSystem. Initializing.");
- LOG.debug(" blockSize = {}",
- conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME, MAX_AZURE_BLOCK_SIZE));
+ LOG.debug(" blockSize = {}", store.getHadoopBlockSize());
// Initialize thread counts from user configuration
deleteThreadCount = conf.getInt(AZURE_DELETE_THREADS, DEFAULT_AZURE_DELETE_THREADS);
@@ -1491,7 +1474,7 @@ public class NativeAzureFileSystem extends FileSystem {
}
}
- private static Path keyToPath(String key) {
+ static Path keyToPath(String key) {
if (key.equals("/")) {
return new Path("/"); // container
}
@@ -1599,7 +1582,7 @@ public class NativeAzureFileSystem extends FileSystem {
throw new FileNotFoundException(f.toString());
}
- if (meta.isDir()) {
+ if (meta.isDirectory()) {
throw new FileNotFoundException(f.toString()
+ " is a directory not a file.");
}
@@ -1815,7 +1798,7 @@ public class NativeAzureFileSystem extends FileSystem {
FileMetadata existingMetadata = store.retrieveMetadata(key);
if (existingMetadata != null) {
- if (existingMetadata.isDir()) {
+ if (existingMetadata.isDirectory()) {
throw new FileAlreadyExistsException("Cannot create file " + f
+ "; already exists as a directory.");
}
@@ -1833,7 +1816,7 @@ public class NativeAzureFileSystem extends FileSystem {
// already exists.
String parentKey = pathToKey(parentFolder);
FileMetadata parentMetadata = store.retrieveMetadata(parentKey);
- if (parentMetadata != null && parentMetadata.isDir() &&
+ if (parentMetadata != null && parentMetadata.isDirectory() &&
parentMetadata.getBlobMaterialization() == BlobMaterialization.Explicit) {
if (parentFolderLease != null) {
store.updateFolderLastModifiedTime(parentKey, parentFolderLease);
@@ -1850,7 +1833,7 @@ public class NativeAzureFileSystem extends FileSystem {
firstExisting = firstExisting.getParent();
metadata = store.retrieveMetadata(pathToKey(firstExisting));
}
- mkdirs(parentFolder, metadata.getPermissionStatus().getPermission(), true);
+ mkdirs(parentFolder, metadata.getPermission(), true);
}
}
@@ -1988,7 +1971,7 @@ public class NativeAzureFileSystem extends FileSystem {
+ parentPath + " whose metadata cannot be retrieved. Can't resolve");
}
- if (!parentMetadata.isDir()) {
+ if (!parentMetadata.isDirectory()) {
// Invalid state: the parent path is actually a file. Throw.
throw new AzureException("File " + f + " has a parent directory "
+ parentPath + " which is also a file. Can't resolve.");
@@ -1997,7 +1980,7 @@ public class NativeAzureFileSystem extends FileSystem {
// The path exists, determine if it is a folder containing objects,
// an empty folder, or a simple file and take the appropriate actions.
- if (!metaFile.isDir()) {
+ if (!metaFile.isDirectory()) {
// The path specifies a file. We need to check the parent path
// to make sure it's a proper materialized directory before we
// delete the file. Otherwise we may get into a situation where
@@ -2114,9 +2097,9 @@ public class NativeAzureFileSystem extends FileSystem {
AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
@Override
public boolean execute(FileMetadata file) throws IOException{
- if (!deleteFile(file.getKey(), file.isDir())) {
+ if (!deleteFile(file.getKey(), file.isDirectory())) {
LOG.warn("Attempt to delete non-existent {} {}",
- file.isDir() ? "directory" : "file",
+ file.isDirectory() ? "directory" : "file",
file.getKey());
}
return true;
@@ -2138,7 +2121,7 @@ public class NativeAzureFileSystem extends FileSystem {
// Delete the current directory if all underlying contents are deleted
if (isPartialDelete || (store.retrieveMetadata(metaFile.getKey()) != null
- && !deleteFile(metaFile.getKey(), metaFile.isDir()))) {
+ && !deleteFile(metaFile.getKey(), metaFile.isDirectory()))) {
LOG.error("Failed delete directory : {}", f);
return false;
}
@@ -2191,7 +2174,7 @@ public class NativeAzureFileSystem extends FileSystem {
// The path exists, determine if it is a folder containing objects,
// an empty folder, or a simple file and take the appropriate actions.
- if (!metaFile.isDir()) {
+ if (!metaFile.isDirectory()) {
// The path specifies a file. We need to check the parent path
// to make sure it's a proper materialized directory before we
// delete the file. Otherwise we may get into a situation where
@@ -2234,7 +2217,7 @@ public class NativeAzureFileSystem extends FileSystem {
+ parentPath + " whose metadata cannot be retrieved. Can't resolve");
}
- if (!parentMetadata.isDir()) {
+ if (!parentMetadata.isDirectory()) {
// Invalid state: the parent path is actually a file. Throw.
throw new AzureException("File " + f + " has a parent directory "
+ parentPath + " which is also a file. Can't resolve.");
@@ -2319,38 +2302,27 @@ public class NativeAzureFileSystem extends FileSystem {
}
}
- // List all the blobs in the current folder.
- String priorLastKey = null;
-
// Start time for list operation
long start = Time.monotonicNow();
- ArrayList<FileMetadata> fileMetadataList = new ArrayList<FileMetadata>();
+ final FileMetadata[] contents;
// List all the files in the folder with AZURE_UNBOUNDED_DEPTH depth.
- do {
- try {
- PartialListing listing = store.listAll(key, AZURE_LIST_ALL,
- AZURE_UNBOUNDED_DEPTH, priorLastKey);
- for(FileMetadata file : listing.getFiles()) {
- fileMetadataList.add(file);
- }
- priorLastKey = listing.getPriorLastKey();
- } catch (IOException e) {
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
- return false;
- }
+ try {
+ contents = store.list(key, AZURE_LIST_ALL,
+ AZURE_UNBOUNDED_DEPTH);
+ } catch (IOException e) {
+ Throwable innerException = checkForAzureStorageException(e);
- throw e;
+ if (innerException instanceof StorageException
+ && isFileNotFoundException((StorageException) innerException)) {
+ return false;
}
- } while (priorLastKey != null);
- long end = Time.monotonicNow();
- LOG.debug("Time taken to list {} blobs for delete operation: {} ms", fileMetadataList.size(), (end - start));
+ throw e;
+ }
- final FileMetadata[] contents = fileMetadataList.toArray(new FileMetadata[fileMetadataList.size()]);
+ long end = Time.monotonicNow();
+ LOG.debug("Time taken to list {} blobs for delete operation: {} ms", contents.length, (end - start));
if (contents.length > 0) {
if (!recursive) {
@@ -2365,9 +2337,9 @@ public class NativeAzureFileSystem extends FileSystem {
AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
@Override
public boolean execute(FileMetadata file) throws IOException{
- if (!deleteFile(file.getKey(), file.isDir())) {
+ if (!deleteFile(file.getKey(), file.isDirectory())) {
LOG.warn("Attempt to delete non-existent {} {}",
- file.isDir() ? "directory" : "file",
+ file.isDirectory() ? "directory" : "file",
file.getKey());
}
return true;
@@ -2384,7 +2356,7 @@ public class NativeAzureFileSystem extends FileSystem {
// Delete the current directory
if (store.retrieveMetadata(metaFile.getKey()) != null
- && !deleteFile(metaFile.getKey(), metaFile.isDir())) {
+ && !deleteFile(metaFile.getKey(), metaFile.isDirectory())) {
LOG.error("Failed delete directory : {}", f);
return false;
}
@@ -2456,13 +2428,13 @@ public class NativeAzureFileSystem extends FileSystem {
boolean isPartialDelete = false;
- Path pathToDelete = makeAbsolute(keyToPath(folderToDelete.getKey()));
+ Path pathToDelete = makeAbsolute(folderToDelete.getPath());
foldersToProcess.push(folderToDelete);
while (!foldersToProcess.empty()) {
FileMetadata currentFolder = foldersToProcess.pop();
- Path currentPath = makeAbsolute(keyToPath(currentFolder.getKey()));
+ Path currentPath = makeAbsolute(currentFolder.getPath());
boolean canDeleteChildren = true;
// If authorization is enabled, check for 'write' permission on current folder
@@ -2478,8 +2450,8 @@ public class NativeAzureFileSystem extends FileSystem {
if (canDeleteChildren) {
// get immediate children list
- ArrayList<FileMetadata> fileMetadataList = getChildrenMetadata(currentFolder.getKey(),
- maxListingDepth);
+ FileMetadata[] fileMetadataList = store.list(currentFolder.getKey(),
+ AZURE_LIST_ALL, maxListingDepth);
// Process children of currentFolder and add them to list of contents
// that can be deleted. We Perform stickybit check on every file and
@@ -2490,12 +2462,12 @@ public class NativeAzureFileSystem extends FileSystem {
// This file/folder cannot be deleted and neither can the parent paths be deleted.
// Remove parent paths from list of contents that can be deleted.
canDeleteChildren = false;
- Path filePath = makeAbsolute(keyToPath(childItem.getKey()));
+ Path filePath = makeAbsolute(childItem.getPath());
LOG.error("User does not have permissions to delete {}. "
+ "Parent directory has sticky bit set.", filePath);
} else {
// push the child directories to the stack to process their contents
- if (childItem.isDir()) {
+ if (childItem.isDirectory()) {
foldersToProcess.push(childItem);
}
// Add items to list of contents that can be deleted.
@@ -2540,23 +2512,6 @@ public class NativeAzureFileSystem extends FileSystem {
return isPartialDelete;
}
- private ArrayList<FileMetadata> getChildrenMetadata(String key, int maxListingDepth)
- throws IOException {
-
- String priorLastKey = null;
- ArrayList<FileMetadata> fileMetadataList = new ArrayList<FileMetadata>();
- do {
- PartialListing listing = store.listAll(key, AZURE_LIST_ALL,
- maxListingDepth, priorLastKey);
- for (FileMetadata file : listing.getFiles()) {
- fileMetadataList.add(file);
- }
- priorLastKey = listing.getPriorLastKey();
- } while (priorLastKey != null);
-
- return fileMetadataList;
- }
-
private boolean isStickyBitCheckViolated(FileMetadata metaData,
FileMetadata parentMetadata, boolean throwOnException) throws IOException {
try {
@@ -2602,13 +2557,13 @@ public class NativeAzureFileSystem extends FileSystem {
}
// stickybit is not set on parent and hence cannot be violated
- if (!parentMetadata.getPermissionStatus().getPermission().getStickyBit()) {
+ if (!parentMetadata.getPermission().getStickyBit()) {
return false;
}
String currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
- String parentDirectoryOwner = parentMetadata.getPermissionStatus().getUserName();
- String currentFileOwner = metaData.getPermissionStatus().getUserName();
+ String parentDirectoryOwner = parentMetadata.getOwner();
+ String currentFileOwner = metaData.getOwner();
// Files/Folders with no owner set will not pass stickybit check
if ((parentDirectoryOwner.equalsIgnoreCase(currentUser))
@@ -2687,7 +2642,15 @@ public class NativeAzureFileSystem extends FileSystem {
Path absolutePath = makeAbsolute(f);
String key = pathToKey(absolutePath);
if (key.length() == 0) { // root always exists
- return newDirectory(null, absolutePath);
+ return new FileStatus(
+ 0,
+ true,
+ 1,
+ store.getHadoopBlockSize(),
+ 0,
+ 0,
+ FsPermission.getDefault(), "", "",
+ absolutePath.makeQualified(getUri(), getWorkingDirectory()));
}
// The path is either a folder or a file. Retrieve metadata to
@@ -2709,7 +2672,7 @@ public class NativeAzureFileSystem extends FileSystem {
}
if (meta != null) {
- if (meta.isDir()) {
+ if (meta.isDirectory()) {
// The path is a folder with files in it.
//
@@ -2723,14 +2686,14 @@ public class NativeAzureFileSystem extends FileSystem {
}
// Return reference to the directory object.
- return newDirectory(meta, absolutePath);
+ return updateFileStatusPath(meta, absolutePath);
}
// The path is a file.
LOG.debug("Found the path: {} as a file.", f.toString());
// Return with reference to a file object.
- return newFile(meta, absolutePath);
+ return updateFileStatusPath(meta, absolutePath);
}
// File not found. Throw exception no such file or directory.
@@ -2787,7 +2750,7 @@ public class NativeAzureFileSystem extends FileSystem {
performAuthCheck(absolutePath, WasbAuthorizationOperations.READ, "liststatus", absolutePath);
String key = pathToKey(absolutePath);
- Set<FileStatus> status = new TreeSet<FileStatus>();
+
FileMetadata meta = null;
try {
meta = store.retrieveMetadata(key);
@@ -2804,101 +2767,93 @@ public class NativeAzureFileSystem extends FileSystem {
throw ex;
}
- if (meta != null) {
- if (!meta.isDir()) {
-
- LOG.debug("Found path as a file");
-
- return new FileStatus[] { newFile(meta, absolutePath) };
- }
-
- String partialKey = null;
- PartialListing listing = null;
-
- try {
- listing = store.list(key, AZURE_LIST_ALL, 1, partialKey);
- } catch (IOException ex) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex);
-
- if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ if (meta == null) {
+ // There is no metadata found for the path.
+ LOG.debug("Did not find any metadata for path: {}", key);
+ throw new FileNotFoundException(f + " is not found");
+ }
- throw new FileNotFoundException(String.format("%s is not found", key));
- }
+ if (!meta.isDirectory()) {
+ LOG.debug("Found path as a file");
+ return new FileStatus[] { updateFileStatusPath(meta, absolutePath) };
+ }
- throw ex;
- }
- // NOTE: We don't check for Null condition as the Store API should return
- // an empty list if there are not listing.
+ FileMetadata[] listing;
- // For any -RenamePending.json files in the listing,
- // push the rename forward.
- boolean renamed = conditionalRedoFolderRenames(listing);
+ listing = listWithErrorHandling(key, AZURE_LIST_ALL, 1);
- // If any renames were redone, get another listing,
- // since the current one may have changed due to the redo.
- if (renamed) {
- listing = null;
- try {
- listing = store.list(key, AZURE_LIST_ALL, 1, partialKey);
- } catch (IOException ex) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex);
+ // NOTE: We don't check for Null condition as the Store API should return
+ // an empty list if there are not listing.
- if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ // For any -RenamePending.json files in the listing,
+ // push the rename forward.
+ boolean renamed = conditionalRedoFolderRenames(listing);
- throw new FileNotFoundException(String.format("%s is not found", key));
- }
+ // If any renames were redone, get another listing,
+ // since the current one may have changed due to the redo.
+ if (renamed) {
+ listing = listWithErrorHandling(key, AZURE_LIST_ALL, 1);
+ }
- throw ex;
- }
- }
+ // We only need to check for AZURE_TEMP_FOLDER if the key is the root,
+ // and if it is not the root we also know the exact size of the array
+ // of FileStatus.
- // NOTE: We don't check for Null condition as the Store API should return
- // and empty list if there are not listing.
+ FileMetadata[] result = null;
- for (FileMetadata fileMetadata : listing.getFiles()) {
- Path subpath = keyToPath(fileMetadata.getKey());
+ if (key.equals("/")) {
+ ArrayList<FileMetadata> status = new ArrayList<>(listing.length);
- // Test whether the metadata represents a file or directory and
- // add the appropriate metadata object.
- //
- // Note: There was a very old bug here where directories were added
- // to the status set as files flattening out recursive listings
- // using "-lsr" down the file system hierarchy.
- if (fileMetadata.isDir()) {
+ for (FileMetadata fileMetadata : listing) {
+ if (fileMetadata.isDirectory()) {
// Make sure we hide the temp upload folder
if (fileMetadata.getKey().equals(AZURE_TEMP_FOLDER)) {
// Don't expose that.
continue;
}
- status.add(newDirectory(fileMetadata, subpath));
+ status.add(updateFileStatusPath(fileMetadata, fileMetadata.getPath()));
} else {
- status.add(newFile(fileMetadata, subpath));
+ status.add(updateFileStatusPath(fileMetadata, fileMetadata.getPath()));
}
}
+ result = status.toArray(new FileMetadata[0]);
+ } else {
+ for (int i = 0; i < listing.length; i++) {
+ FileMetadata fileMetadata = listing[i];
+ listing[i] = updateFileStatusPath(fileMetadata, fileMetadata.getPath());
+ }
+ result = listing;
+ }
- LOG.debug("Found path as a directory with {}"
- + " files in it.", status.size());
+ LOG.debug("Found path as a directory with {}"
+ + " files in it.", result.length);
- } else {
- // There is no metadata found for the path.
- LOG.debug("Did not find any metadata for path: {}", key);
+ return result;
+ }
- throw new FileNotFoundException(f + " is not found");
+ private FileMetadata[] listWithErrorHandling(String prefix, final int maxListingCount,
+ final int maxListingDepth) throws IOException {
+ try {
+ return store.list(prefix, maxListingCount, maxListingDepth);
+ } catch (IOException ex) {
+ Throwable innerException
+ = NativeAzureFileSystemHelper.checkForAzureStorageException(ex);
+ if (innerException instanceof StorageException
+ && NativeAzureFileSystemHelper.isFileNotFoundException(
+ (StorageException) innerException)) {
+ throw new FileNotFoundException(String.format("%s is not found", prefix));
+ }
+ throw ex;
}
-
- return status.toArray(new FileStatus[0]);
}
// Redo any folder renames needed if there are rename pending files in the
// directory listing. Return true if one or more redo operations were done.
- private boolean conditionalRedoFolderRenames(PartialListing listing)
+ private boolean conditionalRedoFolderRenames(FileMetadata[] listing)
throws IllegalArgumentException, IOException {
boolean renamed = false;
- for (FileMetadata fileMetadata : listing.getFiles()) {
- Path subpath = keyToPath(fileMetadata.getKey());
+ for (FileMetadata fileMetadata : listing) {
+ Path subpath = fileMetadata.getPath();
if (isRenamePendingFile(subpath)) {
FolderRenamePending pending =
new FolderRenamePending(subpath, this);
@@ -2914,32 +2869,11 @@ public class NativeAzureFileSystem extends FileSystem {
return path.toString().endsWith(FolderRenamePending.SUFFIX);
}
- private FileStatus newFile(FileMetadata meta, Path path) {
- return new FileStatus (
- meta.getLength(),
- false,
- 1,
- blockSize,
- meta.getLastModified(),
- 0,
- meta.getPermissionStatus().getPermission(),
- meta.getPermissionStatus().getUserName(),
- meta.getPermissionStatus().getGroupName(),
- path.makeQualified(getUri(), getWorkingDirectory()));
- }
-
- private FileStatus newDirectory(FileMetadata meta, Path path) {
- return new FileStatus (
- 0,
- true,
- 1,
- blockSize,
- meta == null ? 0 : meta.getLastModified(),
- 0,
- meta == null ? FsPermission.getDefault() : meta.getPermissionStatus().getPermission(),
- meta == null ? "" : meta.getPermissionStatus().getUserName(),
- meta == null ? "" : meta.getPermissionStatus().getGroupName(),
- path.makeQualified(getUri(), getWorkingDirectory()));
+ private FileMetadata updateFileStatusPath(FileMetadata meta, Path path) {
+ meta.setPath(path.makeQualified(getUri(), getWorkingDirectory()));
+ // reduce memory use by setting the internal-only key to null
+ meta.removeKey();
+ return meta;
}
private static enum UMaskApplyMode {
@@ -3000,8 +2934,8 @@ public class NativeAzureFileSystem extends FileSystem {
String currentKey = pathToKey(current);
FileMetadata currentMetadata = store.retrieveMetadata(currentKey);
- if (currentMetadata != null && currentMetadata.isDir()) {
- Path ancestor = keyToPath(currentMetadata.getKey());
+ if (currentMetadata != null && currentMetadata.isDirectory()) {
+ Path ancestor = currentMetadata.getPath();
LOG.debug("Found ancestor {}, for path: {}", ancestor.toString(), f.toString());
return ancestor;
}
@@ -3052,7 +2986,7 @@ public class NativeAzureFileSystem extends FileSystem {
current = parent, parent = current.getParent()) {
String currentKey = pathToKey(current);
FileMetadata currentMetadata = store.retrieveMetadata(currentKey);
- if (currentMetadata != null && !currentMetadata.isDir()) {
+ if (currentMetadata != null && !currentMetadata.isDirectory()) {
throw new FileAlreadyExistsException("Cannot create directory " + f + " because "
+ current + " is an existing file.");
} else if (currentMetadata == null) {
@@ -3099,7 +3033,7 @@ public class NativeAzureFileSystem extends FileSystem {
if (meta == null) {
throw new FileNotFoundException(f.toString());
}
- if (meta.isDir()) {
+ if (meta.isDirectory()) {
throw new FileNotFoundException(f.toString()
+ " is a directory not a file.");
}
@@ -3120,7 +3054,7 @@ public class NativeAzureFileSystem extends FileSystem {
}
return new FSDataInputStream(new BufferedFSInputStream(
- new NativeAzureFsInputStream(inputStream, key, meta.getLength()), bufferSize));
+ new NativeAzureFsInputStream(inputStream, key, meta.getLen()), bufferSize));
}
@Override
@@ -3196,7 +3130,7 @@ public class NativeAzureFileSystem extends FileSystem {
}
}
- if (dstMetadata != null && dstMetadata.isDir()) {
+ if (dstMetadata != null && dstMetadata.isDirectory()) {
// It's an existing directory.
performAuthCheck(absoluteDstPath, WasbAuthorizationOperations.WRITE, "rename",
absoluteDstPath);
@@ -3232,7 +3166,7 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Parent of the destination {}"
+ " doesn't exist, failing the rename.", dst);
return false;
- } else if (!parentOfDestMetadata.isDir()) {
+ } else if (!parentOfDestMetadata.isDirectory()) {
LOG.debug("Parent of the destination {}"
+ " is a file, failing the rename.", dst);
return false;
@@ -3261,7 +3195,7 @@ public class NativeAzureFileSystem extends FileSystem {
// Source doesn't exist
LOG.debug("Source {} doesn't exist, failing the rename.", src);
return false;
- } else if (!srcMetadata.isDir()) {
+ } else if (!srcMetadata.isDirectory()) {
LOG.debug("Source {} found as a file, renaming.", src);
try {
// HADOOP-15086 - file rename must ensure that the destination does
@@ -3335,7 +3269,7 @@ public class NativeAzureFileSystem extends FileSystem {
// single file. In this case, the parent folder no longer exists if the
// file is renamed; so we can safely ignore the null pointer case.
if (parentMetadata != null) {
- if (parentMetadata.isDir()
+ if (parentMetadata.isDirectory()
&& parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
store.storeEmptyFolder(parentKey,
createPermissionStatus(FsPermission.getDefault()));
@@ -3511,7 +3445,7 @@ public class NativeAzureFileSystem extends FileSystem {
&& !isAllowedUser(currentUgi.getShortUserName(), daemonUsers)) {
//Check if the user is the owner of the file.
- String owner = metadata.getPermissionStatus().getUserName();
+ String owner = metadata.getOwner();
if (!currentUgi.getShortUserName().equals(owner)) {
throw new WasbAuthorizationException(
String.format("user '%s' does not have the privilege to "
@@ -3522,16 +3456,16 @@ public class NativeAzureFileSystem extends FileSystem {
}
permission = applyUMask(permission,
- metadata.isDir() ? UMaskApplyMode.ChangeExistingDirectory
+ metadata.isDirectory() ? UMaskApplyMode.ChangeExistingDirectory
: UMaskApplyMode.ChangeExistingFile);
if (metadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
// It's an implicit folder, need to materialize it.
store.storeEmptyFolder(key, createPermissionStatus(permission));
- } else if (!metadata.getPermissionStatus().getPermission().
+ } else if (!metadata.getPermission().
equals(permission)) {
store.changePermissionStatus(key, new PermissionStatus(
- metadata.getPermissionStatus().getUserName(),
- metadata.getPermissionStatus().getGroupName(),
+ metadata.getOwner(),
+ metadata.getGroup(),
permission));
}
}
@@ -3579,10 +3513,10 @@ public class NativeAzureFileSystem extends FileSystem {
PermissionStatus newPermissionStatus = new PermissionStatus(
username == null ?
- metadata.getPermissionStatus().getUserName() : username,
+ metadata.getOwner() : username,
groupname == null ?
- metadata.getPermissionStatus().getGroupName() : groupname,
- metadata.getPermissionStatus().getPermission());
+ metadata.getGroup() : groupname,
+ metadata.getPermission());
if (metadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
// It's an implicit folder, need to materialize it.
store.storeEmptyFolder(key, newPermissionStatus);
@@ -3778,30 +3712,26 @@ public class NativeAzureFileSystem extends FileSystem {
AZURE_TEMP_EXPIRY_DEFAULT) * 1000;
// Go over all the blobs under the given root and look for blobs to
// recover.
- String priorLastKey = null;
- do {
- PartialListing listing = store.listAll(pathToKey(root), AZURE_LIST_ALL,
- AZURE_UNBOUNDED_DEPTH, priorLastKey);
-
- for (FileMetadata file : listing.getFiles()) {
- if (!file.isDir()) { // We don't recover directory blobs
- // See if this blob has a link in it (meaning it's a place-holder
- // blob for when the upload to the temp blob is complete).
- String link = store.getLinkInFileMetadata(file.getKey());
- if (link != null) {
- // It has a link, see if the temp blob it is pointing to is
- // existent and old enough to be considered dangling.
- FileMetadata linkMetadata = store.retrieveMetadata(link);
- if (linkMetadata != null
- && linkMetadata.getLastModified() >= cutoffForDangling) {
- // Found one!
- handler.handleFile(file, linkMetadata);
- }
+ FileMetadata[] listing = store.list(pathToKey(root), AZURE_LIST_ALL,
+ AZURE_UNBOUNDED_DEPTH);
+
+ for (FileMetadata file : listing) {
+ if (!file.isDirectory()) { // We don't recover directory blobs
+ // See if this blob has a link in it (meaning it's a place-holder
+ // blob for when the upload to the temp blob is complete).
+ String link = store.getLinkInFileMetadata(file.getKey());
+ if (link != null) {
+ // It has a link, see if the temp blob it is pointing to is
+ // existent and old enough to be considered dangling.
+ FileMetadata linkMetadata = store.retrieveMetadata(link);
+ if (linkMetadata != null
+ && linkMetadata.getModificationTime() >= cutoffForDangling) {
+ // Found one!
+ handler.handleFile(file, linkMetadata);
}
}
}
- priorLastKey = listing.getPriorLastKey();
- } while (priorLastKey != null);
+ }
}
/**
@@ -3888,7 +3818,7 @@ public class NativeAzureFileSystem extends FileSystem {
meta = store.retrieveMetadata(key);
if (meta != null) {
- owner = meta.getPermissionStatus().getUserName();
+ owner = meta.getOwner();
LOG.debug("Retrieved '{}' as owner for path - {}", owner, absolutePath);
} else {
// meta will be null if file/folder doen not exist
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749fff57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java
index b67ab1b..36e3819 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java
@@ -58,20 +58,21 @@ interface NativeFileSystemStore {
boolean isAtomicRenameKey(String key);
+ /**
+ * Returns the file block size. This is a fake value used for integration
+ * of the Azure store with Hadoop.
+ * @return The file block size.
+ */
+ long getHadoopBlockSize();
+
void storeEmptyLinkFile(String key, String tempBlobKey,
PermissionStatus permissionStatus) throws AzureException;
String getLinkInFileMetadata(String key) throws AzureException;
- PartialListing list(String prefix, final int maxListingCount,
+ FileMetadata[] list(String prefix, final int maxListingCount,
final int maxListingDepth) throws IOException;
- PartialListing list(String prefix, final int maxListingCount,
- final int maxListingDepth, String priorLastKey) throws IOException;
-
- PartialListing listAll(String prefix, final int maxListingCount,
- final int maxListingDepth, String priorLastKey) throws IOException;
-
void changePermissionStatus(String key, PermissionStatus newPermission)
throws AzureException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749fff57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PartialListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PartialListing.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PartialListing.java
deleted file mode 100644
index 4a80d2e..0000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PartialListing.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * <p>
- * Holds information on a directory listing for a {@link NativeFileSystemStore}.
- * This includes the {@link FileMetadata files} and directories (their names)
- * contained in a directory.
- * </p>
- * <p>
- * This listing may be returned in chunks, so a <code>priorLastKey</code> is
- * provided so that the next chunk may be requested.
- * </p>
- *
- * @see NativeFileSystemStore#list(String, int, String)
- */
-@InterfaceAudience.Private
-class PartialListing {
-
- private final String priorLastKey;
- private final FileMetadata[] files;
- private final String[] commonPrefixes;
-
- public PartialListing(String priorLastKey, FileMetadata[] files,
- String[] commonPrefixes) {
- this.priorLastKey = priorLastKey;
- this.files = files;
- this.commonPrefixes = commonPrefixes;
- }
-
- public FileMetadata[] getFiles() {
- return files;
- }
-
- public String[] getCommonPrefixes() {
- return commonPrefixes;
- }
-
- public String getPriorLastKey() {
- return priorLastKey;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749fff57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestListPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestListPerformance.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestListPerformance.java
new file mode 100644
index 0000000..e7a3fa8
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestListPerformance.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.blob.CloudBlockBlob;
+import org.junit.Assume;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.azure.integration.AbstractAzureScaleTest;
+import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+/**
+ * Test list performance.
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+
+public class ITestListPerformance extends AbstractAzureScaleTest {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ITestListPerformance.class);
+
+ private static final Path TEST_DIR_PATH = new Path(
+ "DirectoryWithManyFiles");
+
+ private static final int NUMBER_OF_THREADS = 10;
+ private static final int NUMBER_OF_FILES_PER_THREAD = 1000;
+
+ private int threads;
+
+ private int filesPerThread;
+
+ private int expectedFileCount;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ Configuration conf = getConfiguration();
+ // fail fast
+ threads = AzureTestUtils.getTestPropertyInt(conf,
+ "fs.azure.scale.test.list.performance.threads", NUMBER_OF_THREADS);
+ filesPerThread = AzureTestUtils.getTestPropertyInt(conf,
+ "fs.azure.scale.test.list.performance.files", NUMBER_OF_FILES_PER_THREAD);
+ expectedFileCount = threads * filesPerThread;
+ LOG.info("Thread = {}, Files per Thread = {}, expected files = {}",
+ threads, filesPerThread, expectedFileCount);
+ conf.set("fs.azure.io.retry.max.retries", "1");
+ conf.set("fs.azure.delete.threads", "16");
+ createTestAccount();
+ }
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ return AzureBlobStorageTestAccount.create(
+ "itestlistperformance",
+ EnumSet.of(AzureBlobStorageTestAccount.CreateOptions.CreateContainer),
+ null,
+ true);
+ }
+
+ @Test
+ public void test_0101_CreateDirectoryWithFiles() throws Exception {
+ Assume.assumeFalse("Test path exists; skipping", fs.exists(TEST_DIR_PATH));
+
+ ExecutorService executorService = Executors.newFixedThreadPool(threads);
+ CloudBlobContainer container = testAccount.getRealContainer();
+
+ final String basePath = (fs.getWorkingDirectory().toUri().getPath() + "/" + TEST_DIR_PATH + "/").substring(1);
+
+ ArrayList<Callable<Integer>> tasks = new ArrayList<>(threads);
+ fs.mkdirs(TEST_DIR_PATH);
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ for (int i = 0; i < threads; i++) {
+ tasks.add(
+ new Callable<Integer>() {
+ public Integer call() {
+ int written = 0;
+ for (int j = 0; j < filesPerThread; j++) {
+ String blobName = basePath + UUID.randomUUID().toString();
+ try {
+ CloudBlockBlob blob = container.getBlockBlobReference(
+ blobName);
+ blob.uploadText("");
+ written ++;
+ } catch (Exception e) {
+ LOG.error("Filed to write {}", blobName, e);
+ break;
+ }
+ }
+ LOG.info("Thread completed with {} files written", written);
+ return written;
+ }
+ }
+ );
+ }
+
+ List<Future<Integer>> futures = executorService.invokeAll(tasks,
+ getTestTimeoutMillis(), TimeUnit.MILLISECONDS);
+ long elapsedMs = timer.elapsedTimeMs();
+ LOG.info("time to create files: {} millis", elapsedMs);
+
+ for (Future<Integer> future : futures) {
+ assertTrue("Future timed out", future.isDone());
+ assertEquals("Future did not write all files timed out",
+ filesPerThread, future.get().intValue());
+ }
+ }
+
+ @Test
+ public void test_0200_ListStatusPerformance() throws Exception {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ FileStatus[] fileList = fs.listStatus(TEST_DIR_PATH);
+ long elapsedMs = timer.elapsedTimeMs();
+ LOG.info(String.format(
+ "files=%1$d, elapsedMs=%2$d",
+ fileList.length,
+ elapsedMs));
+ Map<Path, FileStatus> foundInList =new HashMap<>(expectedFileCount);
+
+ for (FileStatus fileStatus : fileList) {
+ foundInList.put(fileStatus.getPath(), fileStatus);
+ LOG.info("{}: {}", fileStatus.getPath(),
+ fileStatus.isDirectory() ? "dir" : "file");
+ }
+ assertEquals("Mismatch between expected files and actual",
+ expectedFileCount, fileList.length);
+
+
+ // now do a listFiles() recursive
+ ContractTestUtils.NanoTimer initialStatusCallTimer
+ = new ContractTestUtils.NanoTimer();
+ RemoteIterator<LocatedFileStatus> listing
+ = fs.listFiles(TEST_DIR_PATH, true);
+ long initialListTime = initialStatusCallTimer.elapsedTimeMs();
+ timer = new ContractTestUtils.NanoTimer();
+ while (listing.hasNext()) {
+ FileStatus fileStatus = listing.next();
+ Path path = fileStatus.getPath();
+ FileStatus removed = foundInList.remove(path);
+ assertNotNull("Did not find " + path + "{} in the previous listing",
+ removed);
+ }
+ elapsedMs = timer.elapsedTimeMs();
+ LOG.info("time for listFiles() initial call: {} millis;"
+ + " time to iterate: {} millis", initialListTime, elapsedMs);
+ assertEquals("Not all files from listStatus() were found in listFiles()",
+ 0, foundInList.size());
+
+ }
+
+ @Test
+ public void test_0300_BulkDeletePerformance() throws Exception {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ fs.delete(TEST_DIR_PATH,true);
+ long elapsedMs = timer.elapsedTimeMs();
+ LOG.info("time for delete(): {} millis; {} nanoS per file",
+ elapsedMs, timer.nanosPerOperation(expectedFileCount));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] hadoop git commit: HDFS-13583. RBF: Router admin clrQuota is
not synchronized with nameservice. Contributed by Dibyendu Karmakar.
Posted by ey...@apache.org.
HDFS-13583. RBF: Router admin clrQuota is not synchronized with nameservice. Contributed by Dibyendu Karmakar.
(cherry picked from commit 17a87977f29ced49724f561a68565217c8cb4e94)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5aca0588
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5aca0588
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5aca0588
Branch: refs/remotes/origin/branch-3.1
Commit: 5aca0588ea4ebef8dd4aca40e5dd414d0db23b61
Parents: 23b8546
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Jul 24 11:15:47 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Tue Jul 24 11:19:21 2018 +0800
----------------------------------------------------------------------
.../hdfs/server/federation/router/Quota.java | 9 ++++++-
.../federation/router/RouterAdminServer.java | 8 ++++--
.../federation/router/RouterQuotaManager.java | 4 +--
.../router/RouterQuotaUpdateService.java | 2 +-
.../federation/router/RouterQuotaUsage.java | 4 +--
.../federation/store/records/MountTable.java | 4 +--
.../store/records/impl/pb/MountTablePBImpl.java | 4 +--
.../hdfs/tools/federation/RouterAdmin.java | 8 +++---
.../federation/router/TestRouterAdmin.java | 8 ++++++
.../federation/router/TestRouterAdminCLI.java | 16 +++++++++---
.../federation/router/TestRouterQuota.java | 26 +++++++++++++++++---
.../router/TestRouterQuotaManager.java | 20 +++++++--------
.../store/records/TestMountTable.java | 4 +--
13 files changed, 82 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
index 75d3e04..846ccd1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
@@ -162,6 +162,8 @@ public class Quota {
private QuotaUsage aggregateQuota(Map<RemoteLocation, QuotaUsage> results) {
long nsCount = 0;
long ssCount = 0;
+ long nsQuota = HdfsConstants.QUOTA_RESET;
+ long ssQuota = HdfsConstants.QUOTA_RESET;
boolean hasQuotaUnSet = false;
for (Map.Entry<RemoteLocation, QuotaUsage> entry : results.entrySet()) {
@@ -173,6 +175,8 @@ public class Quota {
if (usage.getQuota() == -1 && usage.getSpaceQuota() == -1) {
hasQuotaUnSet = true;
}
+ nsQuota = usage.getQuota();
+ ssQuota = usage.getSpaceQuota();
nsCount += usage.getFileAndDirectoryCount();
ssCount += usage.getSpaceConsumed();
@@ -187,7 +191,10 @@ public class Quota {
QuotaUsage.Builder builder = new QuotaUsage.Builder()
.fileAndDirectoryCount(nsCount).spaceConsumed(ssCount);
if (hasQuotaUnSet) {
- builder.quota(HdfsConstants.QUOTA_DONT_SET);
+ builder.quota(HdfsConstants.QUOTA_RESET)
+ .spaceQuota(HdfsConstants.QUOTA_RESET);
+ } else {
+ builder.quota(nsQuota).spaceQuota(ssQuota);
}
return builder.build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
index 8e23eca..114f008 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
@@ -28,6 +28,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService;
import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolServerSideTranslatorPB;
@@ -253,8 +254,11 @@ public class RouterAdminServer extends AbstractService
if (nsQuota != HdfsConstants.QUOTA_DONT_SET
|| ssQuota != HdfsConstants.QUOTA_DONT_SET) {
- this.router.getRpcServer().getQuotaModule().setQuota(path, nsQuota,
- ssQuota, null);
+ HdfsFileStatus ret = this.router.getRpcServer().getFileInfo(path);
+ if (ret != null) {
+ this.router.getRpcServer().getQuotaModule().setQuota(path, nsQuota,
+ ssQuota, null);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
index 0df34fc..87a8724 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
@@ -161,8 +161,8 @@ public class RouterQuotaManager {
long ssQuota = quota.getSpaceQuota();
// once nsQuota or ssQuota was set, this mount table is quota set
- if (nsQuota != HdfsConstants.QUOTA_DONT_SET
- || ssQuota != HdfsConstants.QUOTA_DONT_SET) {
+ if (nsQuota != HdfsConstants.QUOTA_RESET
+ || ssQuota != HdfsConstants.QUOTA_RESET) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
index 506e2ee..4813b53 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
@@ -111,7 +111,7 @@ public class RouterQuotaUpdateService extends PeriodicService {
// If quota is not set in some subclusters under federation path,
// set quota for this path.
- if (currentQuotaUsage.getQuota() == HdfsConstants.QUOTA_DONT_SET) {
+ if (currentQuotaUsage.getQuota() == HdfsConstants.QUOTA_RESET) {
try {
this.rpcServer.setQuota(src, nsQuota, ssQuota, null);
} catch (IOException ioe) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
index eedd80f..18268aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
@@ -96,14 +96,14 @@ public final class RouterQuotaUsage extends QuotaUsage {
public String toString() {
String nsQuota = String.valueOf(getQuota());
String nsCount = String.valueOf(getFileAndDirectoryCount());
- if (getQuota() == HdfsConstants.QUOTA_DONT_SET) {
+ if (getQuota() == HdfsConstants.QUOTA_RESET) {
nsQuota = "-";
nsCount = "-";
}
String ssQuota = StringUtils.byteDesc(getSpaceQuota());
String ssCount = StringUtils.byteDesc(getSpaceConsumed());
- if (getSpaceQuota() == HdfsConstants.QUOTA_DONT_SET) {
+ if (getSpaceQuota() == HdfsConstants.QUOTA_RESET) {
ssQuota = "-";
ssCount = "-";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
index 005882e..7d1ab70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
@@ -153,9 +153,9 @@ public abstract class MountTable extends BaseRecord {
// Set quota for mount table
RouterQuotaUsage quota = new RouterQuotaUsage.Builder()
.fileAndDirectoryCount(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
- .quota(HdfsConstants.QUOTA_DONT_SET)
+ .quota(HdfsConstants.QUOTA_RESET)
.spaceConsumed(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
- .spaceQuota(HdfsConstants.QUOTA_DONT_SET).build();
+ .spaceQuota(HdfsConstants.QUOTA_RESET).build();
record.setQuota(quota);
// Validate
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
index e62d0a8..4c7622c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
@@ -257,9 +257,9 @@ public class MountTablePBImpl extends MountTable implements PBRecord {
public RouterQuotaUsage getQuota() {
MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- long nsQuota = HdfsConstants.QUOTA_DONT_SET;
+ long nsQuota = HdfsConstants.QUOTA_RESET;
long nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
- long ssQuota = HdfsConstants.QUOTA_DONT_SET;
+ long ssQuota = HdfsConstants.QUOTA_RESET;
long ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
if (proto.hasQuota()) {
QuotaUsageProto quotaProto = proto.getQuota();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
index b0a2062..91e1669 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -632,8 +632,8 @@ public class RouterAdmin extends Configured implements Tool {
* @throws IOException Error clearing the mount point.
*/
private boolean clrQuota(String mount) throws IOException {
- return updateQuota(mount, HdfsConstants.QUOTA_DONT_SET,
- HdfsConstants.QUOTA_DONT_SET);
+ return updateQuota(mount, HdfsConstants.QUOTA_RESET,
+ HdfsConstants.QUOTA_RESET);
}
/**
@@ -668,8 +668,8 @@ public class RouterAdmin extends Configured implements Tool {
long nsCount = existingEntry.getQuota().getFileAndDirectoryCount();
long ssCount = existingEntry.getQuota().getSpaceConsumed();
// If nsQuota and ssQuota were unset, clear nsQuota and ssQuota.
- if (nsQuota == HdfsConstants.QUOTA_DONT_SET &&
- ssQuota == HdfsConstants.QUOTA_DONT_SET) {
+ if (nsQuota == HdfsConstants.QUOTA_RESET &&
+ ssQuota == HdfsConstants.QUOTA_RESET) {
nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
index 769bfe7..c834dcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
@@ -64,6 +64,8 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
/**
* The administrator interface of the {@link Router} implemented by
@@ -101,6 +103,12 @@ public class TestRouterAdmin {
membership.registerNamenode(
createNamenodeReport("ns1", "nn1", HAServiceState.ACTIVE));
stateStore.refreshCaches(true);
+
+ RouterRpcServer spyRpcServer =
+ Mockito.spy(routerContext.getRouter().createRpcServer());
+ Whitebox
+ .setInternalState(routerContext.getRouter(), "rpcServer", spyRpcServer);
+ Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString());
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
index 5207f00..2da5fb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
@@ -115,6 +115,14 @@ public class TestRouterAdminCLI {
Mockito.anyLong(), Mockito.anyLong(), Mockito.any());
Whitebox.setInternalState(
routerContext.getRouter().getRpcServer(), "quotaCall", quota);
+
+ RouterRpcServer spyRpcServer =
+ Mockito.spy(routerContext.getRouter().createRpcServer());
+ Whitebox
+ .setInternalState(routerContext.getRouter(), "rpcServer", spyRpcServer);
+
+ Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString());
+
}
@AfterClass
@@ -447,10 +455,10 @@ public class TestRouterAdminCLI {
// verify the default quota set
assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
quotaUsage.getFileAndDirectoryCount());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getQuota());
assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
quotaUsage.getSpaceConsumed());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getSpaceQuota());
long nsQuota = 50;
long ssQuota = 100;
@@ -494,8 +502,8 @@ public class TestRouterAdminCLI {
quotaUsage = mountTable.getQuota();
// verify if quota unset successfully
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getSpaceQuota());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
index 431b394..6a29446 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
@@ -460,8 +461,10 @@ public class TestRouterQuota {
public void testQuotaSynchronization() throws IOException {
long updateNsQuota = 3;
long updateSsQuota = 4;
+ FileSystem nnFs = nnContext1.getFileSystem();
+ nnFs.mkdirs(new Path("/testsync"));
MountTable mountTable = MountTable.newInstance("/quotaSync",
- Collections.singletonMap("ns0", "/"), Time.now(), Time.now());
+ Collections.singletonMap("ns0", "/testsync"), Time.now(), Time.now());
mountTable.setQuota(new RouterQuotaUsage.Builder().quota(1)
.spaceQuota(2).build());
// Add new mount table
@@ -469,7 +472,7 @@ public class TestRouterQuota {
// ensure the quota is not set as updated value
QuotaUsage realQuota = nnContext1.getFileSystem()
- .getQuotaUsage(new Path("/"));
+ .getQuotaUsage(new Path("/testsync"));
assertNotEquals(updateNsQuota, realQuota.getQuota());
assertNotEquals(updateSsQuota, realQuota.getSpaceQuota());
@@ -489,9 +492,26 @@ public class TestRouterQuota {
// verify if the quota is updated in real path
realQuota = nnContext1.getFileSystem().getQuotaUsage(
- new Path("/"));
+ new Path("/testsync"));
assertEquals(updateNsQuota, realQuota.getQuota());
assertEquals(updateSsQuota, realQuota.getSpaceQuota());
+
+ // Clear the quota
+ mountTable.setQuota(new RouterQuotaUsage.Builder()
+ .quota(HdfsConstants.QUOTA_RESET)
+ .spaceQuota(HdfsConstants.QUOTA_RESET).build());
+
+ updateRequest = UpdateMountTableEntryRequest
+ .newInstance(mountTable);
+ client = routerContext.getAdminClient();
+ mountTableManager = client.getMountTableManager();
+ mountTableManager.updateMountTableEntry(updateRequest);
+
+ // verify if the quota is updated in real path
+ realQuota = nnContext1.getFileSystem().getQuotaUsage(
+ new Path("/testsync"));
+ assertEquals(HdfsConstants.QUOTA_RESET, realQuota.getQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, realQuota.getSpaceQuota());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
index ce3ee17..4a1dd2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
@@ -81,8 +81,8 @@ public class TestRouterQuotaManager {
// test case2: get quota from an no-quota set path
RouterQuotaUsage.Builder quota = new RouterQuotaUsage.Builder()
- .quota(HdfsConstants.QUOTA_DONT_SET)
- .spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+ .quota(HdfsConstants.QUOTA_RESET)
+ .spaceQuota(HdfsConstants.QUOTA_RESET);
manager.put("/noQuotaSet", quota.build());
quotaGet = manager.getQuotaUsage("/noQuotaSet");
// it should return null
@@ -90,36 +90,36 @@ public class TestRouterQuotaManager {
// test case3: get quota from an quota-set path
quota.quota(1);
- quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+ quota.spaceQuota(HdfsConstants.QUOTA_RESET);
manager.put("/hasQuotaSet", quota.build());
quotaGet = manager.getQuotaUsage("/hasQuotaSet");
assertEquals(1, quotaGet.getQuota());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
// test case4: get quota with an non-exist child path
quotaGet = manager.getQuotaUsage("/hasQuotaSet/file");
// it will return the nearest ancestor which quota was set
assertEquals(1, quotaGet.getQuota());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
// test case5: get quota with an child path which its parent
// wasn't quota set
- quota.quota(HdfsConstants.QUOTA_DONT_SET);
- quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+ quota.quota(HdfsConstants.QUOTA_RESET);
+ quota.spaceQuota(HdfsConstants.QUOTA_RESET);
manager.put("/hasQuotaSet/noQuotaSet", quota.build());
// here should returns the quota of path /hasQuotaSet
// (the nearest ancestor which quota was set)
quotaGet = manager.getQuotaUsage("/hasQuotaSet/noQuotaSet/file");
assertEquals(1, quotaGet.getQuota());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
// test case6: get quota with an child path which its parent was quota set
quota.quota(2);
- quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+ quota.spaceQuota(HdfsConstants.QUOTA_RESET);
manager.put("/hasQuotaSet/hasQuotaSet", quota.build());
// here should return the quota of path /hasQuotaSet/hasQuotaSet
quotaGet = manager.getQuotaUsage("/hasQuotaSet/hasQuotaSet/file");
assertEquals(2, quotaGet.getQuota());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5aca0588/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
index 43cf176..05552738 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
@@ -84,9 +84,9 @@ public class TestMountTable {
RouterQuotaUsage quota = record.getQuota();
assertEquals(0, quota.getFileAndDirectoryCount());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quota.getQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quota.getQuota());
assertEquals(0, quota.getSpaceConsumed());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quota.getSpaceQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quota.getSpaceQuota());
MountTable record2 =
MountTable.newInstance(SRC, DST_MAP, DATE_CREATED, DATE_MOD);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] hadoop git commit: HADOOP-15541. [s3a] Shouldn't try to drain
stream before aborting connection in case of timeout. Contributed by Sean
Mackrory.
Posted by ey...@apache.org.
HADOOP-15541. [s3a] Shouldn't try to drain stream before aborting
connection in case of timeout. Contributed by Sean Mackrory.
(cherry picked from commit d503f65b6689b19278ec2a0cf9da5a8762539de8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/caf38532
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/caf38532
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/caf38532
Branch: refs/remotes/origin/branch-3.1
Commit: caf38532f3f3eafb4c874a6debddaad2fb2aa201
Parents: 2aaad40
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jul 11 14:55:11 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jul 11 14:55:11 2018 +0100
----------------------------------------------------------------------
.../apache/hadoop/fs/s3a/S3AInputStream.java | 24 +++++++++++++-------
1 file changed, 16 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf38532/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index c54d3e26..91a2d9d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory;
import java.io.EOFException;
import java.io.IOException;
+import java.net.SocketTimeoutException;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
@@ -155,11 +156,11 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
* @throws IOException on any failure to open the object
*/
@Retries.OnceTranslated
- private synchronized void reopen(String reason, long targetPos, long length)
- throws IOException {
+ private synchronized void reopen(String reason, long targetPos, long length,
+ boolean forceAbort) throws IOException {
if (wrappedStream != null) {
- closeStream("reopen(" + reason + ")", contentRangeFinish, false);
+ closeStream("reopen(" + reason + ")", contentRangeFinish, forceAbort);
}
contentRangeFinish = calculateRequestLimit(inputPolicy, targetPos,
@@ -324,7 +325,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
//re-open at specific location if needed
if (wrappedStream == null) {
- reopen("read from new offset", targetPos, len);
+ reopen("read from new offset", targetPos, len, false);
}
});
}
@@ -367,8 +368,11 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
b = wrappedStream.read();
} catch (EOFException e) {
return -1;
+ } catch (SocketTimeoutException e) {
+ onReadFailure(e, 1, true);
+ b = wrappedStream.read();
} catch (IOException e) {
- onReadFailure(e, 1);
+ onReadFailure(e, 1, false);
b = wrappedStream.read();
}
return b;
@@ -393,12 +397,13 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
* @throws IOException any exception thrown on the re-open attempt.
*/
@Retries.OnceTranslated
- private void onReadFailure(IOException ioe, int length) throws IOException {
+ private void onReadFailure(IOException ioe, int length, boolean forceAbort)
+ throws IOException {
LOG.info("Got exception while trying to read from stream {}" +
" trying to recover: " + ioe, uri);
streamStatistics.readException();
- reopen("failure recovery", pos, length);
+ reopen("failure recovery", pos, length, forceAbort);
}
/**
@@ -446,8 +451,11 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
} catch (EOFException e) {
// the base implementation swallows EOFs.
return -1;
+ } catch (SocketTimeoutException e) {
+ onReadFailure(e, len, true);
+ bytes = wrappedStream.read(buf, off, len);
} catch (IOException e) {
- onReadFailure(e, len);
+ onReadFailure(e, len, false);
bytes= wrappedStream.read(buf, off, len);
}
return bytes;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] hadoop git commit: YARN-8501. Reduce complexity of
RMWebServices getApps method. Contributed by Szilard Nemeth
Posted by ey...@apache.org.
YARN-8501. Reduce complexity of RMWebServices getApps method.
Contributed by Szilard Nemeth
(cherry picked from commit 5836e0a46bf9793e0a61bb8ec46536f4a67d38d7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76b8beb2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76b8beb2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76b8beb2
Branch: refs/remotes/origin/branch-3.1
Commit: 76b8beb289c0d0b77653eae69de3b1469a417883
Parents: a147098
Author: Eric Yang <ey...@apache.org>
Authored: Thu Jul 19 12:30:38 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Jul 19 12:32:55 2018 -0400
----------------------------------------------------------------------
.../hadoop/yarn/server/webapp/WebServices.java | 2 +-
.../webapp/ApplicationsRequestBuilder.java | 231 ++++++++
.../resourcemanager/webapp/RMWebServices.java | 145 +----
.../webapp/TestApplicationsRequestBuilder.java | 529 +++++++++++++++++++
4 files changed, 777 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76b8beb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index df4656f..8b00b9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -392,7 +392,7 @@ public class WebServices {
response.setContentType(null);
}
- protected static Set<String>
+ public static Set<String>
parseQueries(Set<String> queries, boolean isState) {
Set<String> params = new HashSet<String>();
if (!queries.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76b8beb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ApplicationsRequestBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ApplicationsRequestBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ApplicationsRequestBuilder.java
new file mode 100644
index 0000000..876d044
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ApplicationsRequestBuilder.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+ .CapacityScheduler;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+
+import java.io.IOException;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.server.webapp.WebServices.parseQueries;
+
+public class ApplicationsRequestBuilder {
+
+ private Set<String> statesQuery = Sets.newHashSet();
+ private Set<String> users = Sets.newHashSetWithExpectedSize(1);
+ private Set<String> queues = Sets.newHashSetWithExpectedSize(1);
+ private String limit = null;
+ private Long limitNumber;
+
+ // set values suitable in case both of begin/end not specified
+ private long startedTimeBegin = 0;
+ private long startedTimeEnd = Long.MAX_VALUE;
+ private long finishTimeBegin = 0;
+ private long finishTimeEnd = Long.MAX_VALUE;
+ private Set<String> appTypes = Sets.newHashSet();
+ private Set<String> appTags = Sets.newHashSet();
+ private ResourceManager rm;
+
+ private ApplicationsRequestBuilder() {
+ }
+
+ public static ApplicationsRequestBuilder create() {
+ return new ApplicationsRequestBuilder();
+ }
+
+ public ApplicationsRequestBuilder withStateQuery(String stateQuery) {
+ // stateQuery is deprecated.
+ if (stateQuery != null && !stateQuery.isEmpty()) {
+ statesQuery.add(stateQuery);
+ }
+ return this;
+ }
+
+ public ApplicationsRequestBuilder withStatesQuery(
+ Set<String> statesQuery) {
+ if (statesQuery != null) {
+ this.statesQuery.addAll(statesQuery);
+ }
+ return this;
+ }
+
+ public ApplicationsRequestBuilder withUserQuery(String userQuery) {
+ if (userQuery != null && !userQuery.isEmpty()) {
+ users.add(userQuery);
+ }
+ return this;
+ }
+
+ public ApplicationsRequestBuilder withQueueQuery(ResourceManager rm,
+ String queueQuery) {
+ this.rm = rm;
+ if (queueQuery != null && !queueQuery.isEmpty()) {
+ queues.add(queueQuery);
+ }
+ return this;
+ }
+
+ public ApplicationsRequestBuilder withLimit(String limit) {
+ if (limit != null && !limit.isEmpty()) {
+ this.limit = limit;
+ }
+ return this;
+ }
+
+ public ApplicationsRequestBuilder withStartedTimeBegin(
+ String startedBegin) {
+ if (startedBegin != null && !startedBegin.isEmpty()) {
+ startedTimeBegin = parseLongValue(startedBegin, "startedTimeBegin");
+ }
+ return this;
+ }
+
+ public ApplicationsRequestBuilder withStartedTimeEnd(String startedEnd) {
+ if (startedEnd != null && !startedEnd.isEmpty()) {
+ startedTimeEnd = parseLongValue(startedEnd, "startedTimeEnd");
+ }
+ return this;
+ }
+
+ public ApplicationsRequestBuilder withFinishTimeBegin(String finishBegin) {
+ if (finishBegin != null && !finishBegin.isEmpty()) {
+ finishTimeBegin = parseLongValue(finishBegin, "finishedTimeBegin");
+ }
+ return this;
+ }
+
+ public ApplicationsRequestBuilder withFinishTimeEnd(String finishEnd) {
+ if (finishEnd != null && !finishEnd.isEmpty()) {
+ finishTimeEnd = parseLongValue(finishEnd, "finishedTimeEnd");
+ }
+ return this;
+ }
+
+ public ApplicationsRequestBuilder withApplicationTypes(
+ Set<String> applicationTypes) {
+ if (applicationTypes != null) {
+ appTypes = parseQueries(applicationTypes, false);
+ }
+ return this;
+ }
+
+ public ApplicationsRequestBuilder withApplicationTags(
+ Set<String> applicationTags) {
+ if (applicationTags != null) {
+ appTags = parseQueries(applicationTags, false);
+ }
+ return this;
+ }
+
+ private void validate() {
+ queues.forEach(q -> validateQueueExists(rm, q));
+ validateLimit();
+ validateStartTime();
+ validateFinishTime();
+ }
+
+ private void validateQueueExists(ResourceManager rm, String queueQuery) {
+ ResourceScheduler rs = rm.getResourceScheduler();
+ if (rs instanceof CapacityScheduler) {
+ CapacityScheduler cs = (CapacityScheduler) rs;
+ try {
+ cs.getQueueInfo(queueQuery, false, false);
+ } catch (IOException e) {
+ throw new BadRequestException(e.getMessage());
+ }
+ }
+ }
+
+ private void validateLimit() {
+ if (limit != null) {
+ limitNumber = parseLongValue(limit, "limit");
+ if (limitNumber <= 0) {
+ throw new BadRequestException("limit value must be greater then 0");
+ }
+ }
+ }
+
+ private long parseLongValue(String strValue, String queryName) {
+ try {
+ return Long.parseLong(strValue);
+ } catch (NumberFormatException e) {
+ throw new BadRequestException(queryName + " value must be a number!");
+ }
+ }
+
+ private void validateStartTime() {
+ if (startedTimeBegin < 0) {
+ throw new BadRequestException("startedTimeBegin must be greater than 0");
+ }
+ if (startedTimeEnd < 0) {
+ throw new BadRequestException("startedTimeEnd must be greater than 0");
+ }
+ if (startedTimeBegin > startedTimeEnd) {
+ throw new BadRequestException(
+ "startedTimeEnd must be greater than startTimeBegin");
+ }
+ }
+
+ private void validateFinishTime() {
+ if (finishTimeBegin < 0) {
+ throw new BadRequestException("finishTimeBegin must be greater than 0");
+ }
+ if (finishTimeEnd < 0) {
+ throw new BadRequestException("finishTimeEnd must be greater than 0");
+ }
+ if (finishTimeBegin > finishTimeEnd) {
+ throw new BadRequestException(
+ "finishTimeEnd must be greater than finishTimeBegin");
+ }
+ }
+
+ public GetApplicationsRequest build() {
+ validate();
+ GetApplicationsRequest request = GetApplicationsRequest.newInstance();
+
+ Set<String> appStates = parseQueries(statesQuery, true);
+ if (!appStates.isEmpty()) {
+ request.setApplicationStates(appStates);
+ }
+ if (!users.isEmpty()) {
+ request.setUsers(users);
+ }
+ if (!queues.isEmpty()) {
+ request.setQueues(queues);
+ }
+ if (limitNumber != null) {
+ request.setLimit(limitNumber);
+ }
+ request.setStartRange(startedTimeBegin, startedTimeEnd);
+ request.setFinishRange(finishTimeBegin, finishTimeEnd);
+
+ if (!appTypes.isEmpty()) {
+ request.setApplicationTypes(appTypes);
+ }
+ if (!appTags.isEmpty()) {
+ request.setApplicationTags(appTags);
+ }
+
+ return request;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76b8beb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 864653c..ffd9646 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -482,7 +482,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
@QueryParam(RMWSConsts.FINAL_STATUS) String finalStatusQuery,
@QueryParam(RMWSConsts.USER) String userQuery,
@QueryParam(RMWSConsts.QUEUE) String queueQuery,
- @QueryParam(RMWSConsts.LIMIT) String count,
+ @QueryParam(RMWSConsts.LIMIT) String limit,
@QueryParam(RMWSConsts.STARTED_TIME_BEGIN) String startedBegin,
@QueryParam(RMWSConsts.STARTED_TIME_END) String startedEnd,
@QueryParam(RMWSConsts.FINISHED_TIME_BEGIN) String finishBegin,
@@ -493,135 +493,22 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
initForReadableEndpoints();
- boolean checkCount = false;
- boolean checkStart = false;
- boolean checkEnd = false;
- boolean checkAppTypes = false;
- boolean checkAppStates = false;
- boolean checkAppTags = false;
- long countNum = 0;
-
- // set values suitable in case both of begin/end not specified
- long sBegin = 0;
- long sEnd = Long.MAX_VALUE;
- long fBegin = 0;
- long fEnd = Long.MAX_VALUE;
-
- if (count != null && !count.isEmpty()) {
- checkCount = true;
- countNum = Long.parseLong(count);
- if (countNum <= 0) {
- throw new BadRequestException("limit value must be greater then 0");
- }
- }
-
- if (startedBegin != null && !startedBegin.isEmpty()) {
- checkStart = true;
- sBegin = Long.parseLong(startedBegin);
- if (sBegin < 0) {
- throw new BadRequestException(
- "startedTimeBegin must be greater than 0");
- }
- }
- if (startedEnd != null && !startedEnd.isEmpty()) {
- checkStart = true;
- sEnd = Long.parseLong(startedEnd);
- if (sEnd < 0) {
- throw new BadRequestException("startedTimeEnd must be greater than 0");
- }
- }
- if (sBegin > sEnd) {
- throw new BadRequestException(
- "startedTimeEnd must be greater than startTimeBegin");
- }
-
- if (finishBegin != null && !finishBegin.isEmpty()) {
- checkEnd = true;
- fBegin = Long.parseLong(finishBegin);
- if (fBegin < 0) {
- throw new BadRequestException("finishTimeBegin must be greater than 0");
- }
- }
- if (finishEnd != null && !finishEnd.isEmpty()) {
- checkEnd = true;
- fEnd = Long.parseLong(finishEnd);
- if (fEnd < 0) {
- throw new BadRequestException("finishTimeEnd must be greater than 0");
- }
- }
- if (fBegin > fEnd) {
- throw new BadRequestException(
- "finishTimeEnd must be greater than finishTimeBegin");
- }
-
- Set<String> appTypes = parseQueries(applicationTypes, false);
- if (!appTypes.isEmpty()) {
- checkAppTypes = true;
- }
-
- Set<String> appTags = parseQueries(applicationTags, false);
- if (!appTags.isEmpty()) {
- checkAppTags = true;
- }
-
- // stateQuery is deprecated.
- if (stateQuery != null && !stateQuery.isEmpty()) {
- statesQuery.add(stateQuery);
- }
- Set<String> appStates = parseQueries(statesQuery, true);
- if (!appStates.isEmpty()) {
- checkAppStates = true;
- }
-
- GetApplicationsRequest request = GetApplicationsRequest.newInstance();
-
- if (checkStart) {
- request.setStartRange(sBegin, sEnd);
- }
-
- if (checkEnd) {
- request.setFinishRange(fBegin, fEnd);
- }
-
- if (checkCount) {
- request.setLimit(countNum);
- }
-
- if (checkAppTypes) {
- request.setApplicationTypes(appTypes);
- }
-
- if (checkAppTags) {
- request.setApplicationTags(appTags);
- }
-
- if (checkAppStates) {
- request.setApplicationStates(appStates);
- }
-
- if (queueQuery != null && !queueQuery.isEmpty()) {
- ResourceScheduler rs = rm.getResourceScheduler();
- if (rs instanceof CapacityScheduler) {
- CapacityScheduler cs = (CapacityScheduler) rs;
- // validate queue exists
- try {
- cs.getQueueInfo(queueQuery, false, false);
- } catch (IOException e) {
- throw new BadRequestException(e.getMessage());
- }
- }
- Set<String> queues = new HashSet<String>(1);
- queues.add(queueQuery);
- request.setQueues(queues);
- }
-
- if (userQuery != null && !userQuery.isEmpty()) {
- Set<String> users = new HashSet<String>(1);
- users.add(userQuery);
- request.setUsers(users);
- }
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create()
+ .withStateQuery(stateQuery)
+ .withStatesQuery(statesQuery)
+ .withUserQuery(userQuery)
+ .withQueueQuery(rm, queueQuery)
+ .withLimit(limit)
+ .withStartedTimeBegin(startedBegin)
+ .withStartedTimeEnd(startedEnd)
+ .withFinishTimeBegin(finishBegin)
+ .withFinishTimeEnd(finishEnd)
+ .withApplicationTypes(applicationTypes)
+ .withApplicationTags(applicationTags)
+ .build();
- List<ApplicationReport> appReports = null;
+ List<ApplicationReport> appReports;
try {
appReports = rm.getClientRMService().getApplications(request)
.getApplicationList();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76b8beb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestApplicationsRequestBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestApplicationsRequestBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestApplicationsRequestBuilder.java
new file mode 100644
index 0000000..7c9b711
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestApplicationsRequestBuilder.java
@@ -0,0 +1,529 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.server.webapp.WebServices.parseQueries;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestApplicationsRequestBuilder {
+
+ private GetApplicationsRequest getDefaultRequest() {
+ GetApplicationsRequest req = GetApplicationsRequest.newInstance();
+ req.setStartRange(0, Long.MAX_VALUE);
+ req.setFinishRange(0, Long.MAX_VALUE);
+ return req;
+ }
+
+ @Test
+ public void testDefaultRequest() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithNullStateQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withStateQuery(null).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithEmptyStateQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withStateQuery("").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidStateQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStateQuery("invalidState").build();
+ }
+
+ @Test
+ public void testRequestWithValidStateQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStateQuery(YarnApplicationState.NEW_SAVING.toString()).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ Set<String> appStates =
+ Sets.newHashSet(YarnApplicationState.NEW_SAVING.toString());
+ Set<String> appStatesLowerCase = parseQueries(appStates, true);
+ expectedRequest.setApplicationStates(appStatesLowerCase);
+
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithEmptyStateQueries() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStatesQuery(Sets.newHashSet()).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidStateQueries() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStatesQuery(Sets.newHashSet("a1", "a2", "")).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithNullStateQueries() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withStatesQuery(null).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithValidStateQueries() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStatesQuery(
+ Sets.newHashSet(YarnApplicationState.NEW_SAVING.toString(),
+ YarnApplicationState.NEW.toString()))
+ .build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ Set<String> appStates =
+ Sets.newHashSet(YarnApplicationState.NEW_SAVING.toString(),
+ YarnApplicationState.NEW.toString());
+ Set<String> appStatesLowerCase = parseQueries(appStates, true);
+ expectedRequest.setApplicationStates(appStatesLowerCase);
+
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithNullUserQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withUserQuery(null).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithEmptyUserQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withUserQuery("").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithUserQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withUserQuery("user1").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setUsers(Sets.newHashSet("user1"));
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithNullQueueQuery() {
+ ResourceManager rm = mock(ResourceManager.class);
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withQueueQuery(rm, null).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithEmptyQueueQuery() {
+ ResourceManager rm = mock(ResourceManager.class);
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withQueueQuery(rm, "").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithQueueQueryExistingQueue() {
+ ResourceManager rm = mock(ResourceManager.class);
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withQueueQuery(rm, "queue1").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setQueues(Sets.newHashSet("queue1"));
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithQueueQueryNotExistingQueue() throws IOException {
+ CapacityScheduler cs = mock(CapacityScheduler.class);
+ when(cs.getQueueInfo(eq("queue1"), anyBoolean(), anyBoolean()))
+ .thenThrow(new IOException());
+ ResourceManager rm = mock(ResourceManager.class);
+ when(rm.getResourceScheduler()).thenReturn(cs);
+
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withQueueQuery(rm, "queue1").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setQueues(Sets.newHashSet("queue1"));
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithNullLimitQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withLimit(null).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithEmptyLimitQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withLimit("").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidLimitQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withLimit("bla").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidNegativeLimitQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withLimit("-10").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithValidLimitQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withLimit("999").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setLimit(999L);
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithNullStartedTimeBeginQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStartedTimeBegin(null).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithEmptyStartedTimeBeginQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withStartedTimeBegin("").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidStartedTimeBeginQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStartedTimeBegin("bla").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidNegativeStartedTimeBeginQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStartedTimeBegin("-1").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithValidStartedTimeBeginQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStartedTimeBegin("999").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setStartRange(999L, Long.MAX_VALUE);
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithNullStartedTimeEndQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withStartedTimeEnd(null).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithEmptywithStartedTimeEndQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withStartedTimeEnd("").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidStartedTimeEndQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStartedTimeEnd("bla").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidNegativeStartedTimeEndQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withStartedTimeEnd("-1").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithValidStartedTimeEndQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStartedTimeEnd("999").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setStartRange(0L, 999L);
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithNullFinishedTimeBeginQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withFinishTimeBegin(null).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithEmptyFinishedTimeBeginQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withFinishTimeBegin("").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidFinishedTimeBeginQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withFinishTimeBegin("bla").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidNegativeFinishedTimeBeginQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withFinishTimeBegin("-1").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithValidFinishedTimeBeginQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withFinishTimeBegin("999").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setFinishRange(999L, Long.MAX_VALUE);
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithNullFinishedTimeEndQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withFinishTimeEnd(null).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithEmptyFinishTimeEndQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withFinishTimeEnd("").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidFinishTimeEndQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withFinishTimeEnd("bla").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidNegativeFinishedTimeEndQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withFinishTimeEnd("-1").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithValidFinishTimeEndQuery() {
+ GetApplicationsRequest request =
+ ApplicationsRequestBuilder.create().withFinishTimeEnd("999").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setFinishRange(0L, 999L);
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithValidStartTimeRangeQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStartedTimeBegin("1000").withStartedTimeEnd("2000").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setStartRange(1000L, 2000L);
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidStartTimeRangeQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withStartedTimeBegin("2000").withStartedTimeEnd("1000").build();
+ }
+
+ @Test
+ public void testRequestWithValidFinishTimeRangeQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withFinishTimeBegin("1000").withFinishTimeEnd("2000").build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setFinishRange(1000L, 2000L);
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test(expected = BadRequestException.class)
+ public void testRequestWithInvalidFinishTimeRangeQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withFinishTimeBegin("2000").withFinishTimeEnd("1000").build();
+ }
+
+ @Test
+ public void testRequestWithNullApplicationTypesQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withApplicationTypes(null).build();
+ }
+
+ @Test
+ public void testRequestWithEmptyApplicationTypesQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withApplicationTypes(Sets.newHashSet()).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setApplicationTypes(Sets.newHashSet());
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithValidApplicationTypesQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withApplicationTypes(Sets.newHashSet("type1")).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setApplicationTypes(Sets.newHashSet("type1"));
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithNullApplicationTagsQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withApplicationTags(null).build();
+ }
+
+ @Test
+ public void testRequestWithEmptyApplicationTagsQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withApplicationTags(Sets.newHashSet()).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setApplicationTags(Sets.newHashSet());
+ assertEquals(expectedRequest, request);
+ }
+
+ @Test
+ public void testRequestWithValidApplicationTagsQuery() {
+ GetApplicationsRequest request = ApplicationsRequestBuilder.create()
+ .withApplicationTags(Sets.newHashSet("tag1")).build();
+
+ GetApplicationsRequest expectedRequest = getDefaultRequest();
+ expectedRequest.setApplicationTags(Sets.newHashSet("tag1"));
+ assertEquals(expectedRequest, request);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] hadoop git commit: YARN-8541. RM startup failure on recovery
after user deletion. Contributed by Bibin A Chundatt.
Posted by ey...@apache.org.
YARN-8541. RM startup failure on recovery after user deletion. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e65057e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e65057e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e65057e
Branch: refs/remotes/origin/branch-3.1
Commit: 8e65057eb10d03db08781da7a5ad8855155883ed
Parents: b89624a
Author: bibinchundatt <bi...@apache.org>
Authored: Wed Jul 25 15:54:32 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Wed Jul 25 15:54:32 2018 +0530
----------------------------------------------------------------------
.../server/resourcemanager/RMAppManager.java | 48 ++++++++++----------
.../placement/PlacementManager.java | 9 ----
.../TestWorkPreservingRMRestart.java | 48 ++++++++++++++++++++
3 files changed, 72 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e65057e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 3e64cfc..7011aaa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -364,17 +364,9 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
ApplicationSubmissionContext submissionContext, long submitTime,
String user, boolean isRecovery, long startTime) throws YarnException {
- ApplicationPlacementContext placementContext = null;
- try {
- placementContext = placeApplication(rmContext, submissionContext, user);
- } catch (YarnException e) {
- String msg =
- "Failed to place application " + submissionContext.getApplicationId()
- + " to queue and specified " + "queue is invalid : "
- + submissionContext.getQueue();
- LOG.error(msg, e);
- throw e;
- }
+ ApplicationPlacementContext placementContext =
+ placeApplication(rmContext.getQueuePlacementManager(),
+ submissionContext, user, isRecovery);
// We only replace the queue when it's a new application
if (!isRecovery) {
@@ -789,23 +781,31 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
}
@VisibleForTesting
- ApplicationPlacementContext placeApplication(RMContext rmContext,
- ApplicationSubmissionContext context, String user) throws YarnException {
+ ApplicationPlacementContext placeApplication(
+ PlacementManager placementManager, ApplicationSubmissionContext context,
+ String user, boolean isRecovery) throws YarnException {
ApplicationPlacementContext placementContext = null;
- PlacementManager placementManager = rmContext.getQueuePlacementManager();
-
if (placementManager != null) {
- placementContext = placementManager.placeApplication(context, user);
- } else{
- if ( context.getQueue() == null || context.getQueue().isEmpty()) {
- final String msg = "Queue Placement Manager is not set. Cannot place "
- + "application : " + context.getApplicationId() + " to queue and "
- + "specified queue is invalid " + context.getQueue();
- LOG.error(msg);
- throw new YarnException(msg);
+ try {
+ placementContext = placementManager.placeApplication(context, user);
+ } catch (YarnException e) {
+ // Placement could also fail if the user doesn't exist in system
+ // skip if the user is not found during recovery.
+ if (isRecovery) {
+ LOG.warn("PlaceApplication failed,skipping on recovery of rm");
+ return placementContext;
+ }
+ throw e;
}
}
-
+ if (placementContext == null && (context.getQueue() == null) || context
+ .getQueue().isEmpty()) {
+ String msg = "Failed to place application " + context.getApplicationId()
+ + " to queue and specified " + "queue is invalid : " + context
+ .getQueue();
+ LOG.error(msg);
+ throw new YarnException(msg);
+ }
return placementContext;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e65057e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementManager.java
index 5fa7723..74cf7ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementManager.java
@@ -70,15 +70,6 @@ public class PlacementManager {
}
}
- // Failed to get where to place application
- if (null == placement && null == asc.getQueue()) {
- String msg = "Failed to place application " +
- asc.getApplicationId() + " to queue and specified "
- + "queue is invalid : " + asc.getQueue();
- LOG.error(msg);
- throw new YarnException(msg);
- }
-
return placement;
} finally {
readLock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e65057e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index 88c19a1..a821b0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -39,8 +39,12 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityMockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.placement
+ .ApplicationPlacementContext;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
@@ -105,6 +109,8 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
import static org.apache.hadoop.yarn.server.resourcemanager.webapp
.RMWebServices.DEFAULT_QUEUE;
import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -1555,6 +1561,48 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
}
@Test(timeout = 30000)
+ public void testUnknownUserOnRecovery() throws Exception {
+
+ MockRM rm1 = new MockRM(conf);
+ rm1.start();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ // create app and launch the UAM
+ RMApp app0 = rm1.submitApp(200, true);
+ MockAM am0 = MockRM.launchUAM(app0, rm1, nm1);
+ am0.registerAppAttempt();
+ rm1.killApp(app0.getApplicationId());
+ PlacementManager placementMgr = mock(PlacementManager.class);
+ doThrow(new YarnException("No groups for user")).when(placementMgr)
+ .placeApplication(any(ApplicationSubmissionContext.class),
+ any(String.class));
+ MockRM rm2 = new MockRM(conf, memStore) {
+ @Override
+ protected RMAppManager createRMAppManager() {
+ return new RMAppManager(this.rmContext, this.scheduler,
+ this.masterService, this.applicationACLsManager, conf) {
+ @Override
+ ApplicationPlacementContext placeApplication(
+ PlacementManager placementManager,
+ ApplicationSubmissionContext context, String user,
+ boolean isRecovery) throws YarnException {
+ return super
+ .placeApplication(placementMgr, context, user, isRecovery);
+ }
+ };
+ }
+ };
+ rm2.start();
+ RMApp recoveredApp =
+ rm2.getRMContext().getRMApps().get(app0.getApplicationId());
+ Assert.assertEquals(RMAppState.KILLED, recoveredApp.getState());
+ }
+
+ @Test(timeout = 30000)
public void testDynamicAutoCreatedQueueRecoveryWithDefaultQueue()
throws Exception {
//if queue name is not specified, it should submit to 'default' queue
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] hadoop git commit: YARN-8515. container-executor can crash
with SIGPIPE after nodemanager restart. Contributed by Jim Brennan
Posted by ey...@apache.org.
YARN-8515. container-executor can crash with SIGPIPE after nodemanager restart. Contributed by Jim Brennan
(cherry picked from commit 17118f446c2387aa796849da8b69a845d9d307d3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7cbb9597
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7cbb9597
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7cbb9597
Branch: refs/remotes/origin/branch-3.1
Commit: 7cbb9597c43d0e4270a64d28b5521941ce940a1a
Parents: d5d9875
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jul 13 10:05:25 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jul 13 10:06:38 2018 -0500
----------------------------------------------------------------------
.../src/main/native/container-executor/impl/main.c | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7cbb9597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index c54fd3e..3d7b19a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -31,6 +31,7 @@
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
+#include <signal.h>
static void display_usage(FILE *stream) {
fprintf(stream,
@@ -112,6 +113,11 @@ static void open_log_files() {
if (ERRORFILE == NULL) {
ERRORFILE = stderr;
}
+
+ // There may be a process reading from stdout/stderr, and if it
+ // exits, we will crash on a SIGPIPE when we try to write to them.
+ // By ignoring SIGPIPE, we can handle the EPIPE instead of crashing.
+ signal(SIGPIPE, SIG_IGN);
}
/* Flushes and closes log files */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] hadoop git commit: YARN-8301. Added YARN service upgrade
instructions. Contributed by Chandni Singh
Posted by ey...@apache.org.
YARN-8301. Added YARN service upgrade instructions.
Contributed by Chandni Singh
(cherry picked from commit 10014a4d88f239d3c072e51bc0739cba1fca9406)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f2a129f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f2a129f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f2a129f
Branch: refs/remotes/origin/branch-3.1
Commit: 4f2a129f2e2326ca28659d93b412cf8649ed5025
Parents: 004e1f2
Author: Eric Yang <ey...@apache.org>
Authored: Fri Jul 20 19:46:35 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Jul 20 19:48:19 2018 -0400
----------------------------------------------------------------------
.../src/site/markdown/yarn-service/Overview.md | 4 +-
.../markdown/yarn-service/ServiceUpgrade.md | 197 +++++++++++++++++++
2 files changed, 198 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f2a129f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
index 8e2bf9a..041b0ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
@@ -56,6 +56,4 @@ The benefits of combining these workloads are two-fold:
* [Registry DNS](RegistryDNS.html): Deep dives into the Registry DNS internals.
* [Examples](Examples.html): List some example service definitions (`Yarnfile`).
* [Configurations](Configurations.html): Describes how to configure the custom services on YARN.
-
-
-
+* [Service Upgrade](ServiceUpgrade.html): Describes how to upgrade a YARN service which is an experimental feature.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f2a129f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceUpgrade.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceUpgrade.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceUpgrade.md
new file mode 100644
index 0000000..839be22
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceUpgrade.md
@@ -0,0 +1,197 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Service Upgrade (Experimental Feature - Tech Preview)
+
+Yarn service provides a way of upgrading/downgrading long running applications without
+shutting down the application to minimize the downtime during this process. This is
+an experimental feature which is currently not enabled by default.
+
+## Overview
+
+Upgrading a Yarn Service is a 3 steps (or 2 steps when auto-finalization of
+upgrade is chosen) process:
+
+1. Initiate service upgrade.\
+This step involves providing the service spec of the newer version of the service.
+Once, the service upgrade is initiated, the state of the service is changed to
+`UPGRADING`.
+
+2. Upgrade component instances.\
+This step involves triggering upgrade of individual component instance.
+By providing an API to upgrade at instance level, users can orchestrate upgrade
+of the entire service in any order which is relevant for the service.\
+In addition, there are APIs to upgrade multiple instances, all instances of a
+component, and all instances of multiple components.
+
+3. Finalize upgrade.\
+This step involves finalization of upgrade. With an explicit step to finalize the
+upgrade, users have a chance to cancel current upgrade in progress. When the
+user chose to cancel, the service will make the best effort to revert to the
+previous version.\
+\
+When the upgrade is finalized, the old service definition is
+overwritten by the new service definition and the service state changes to `STABLE`.\
+A service can be auto-finalized when the upgrade is initialized with
+`-autoFinalize` option. With auto-finalization, when all the component-instances of
+the service have been upgraded, finalization will be performed automatically by the
+service framework.\
+\
+**NOTE**: Cancel of upgrade is not implemented yet.
+
+## Upgrade Example
+This example shows upgrade of sleeper service. Below is the sleeper service
+definition
+
+```
+{
+ "name": "sleeper-service",
+ "components" :
+ [
+ {
+ "name": "sleeper",
+ "version": "1.0.0",
+ "number_of_containers": 1,
+ "launch_command": "sleep 900000",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+Assuming, user launched an instance of sleeper service named as `my-sleeper`:
+```
+{
+ "components":
+ [
+ {
+ "configuration": {...},
+ "containers":
+ [
+ {
+ "bare_host": "0.0.0.0",
+ "component_instance_name": "sleeper-0",
+ "hostname": "example.local",
+ "id": "container_1531508836237_0002_01_000002",
+ "ip": "0.0.0.0",
+ "launch_time": 1531941023675,
+ "state": "READY"
+ },
+ {
+ "bare_host": "0.0.0.0",
+ "component_instance_name": "sleeper-1",
+ "hostname": "example.local",
+ "id": "container_1531508836237_0002_01_000003",
+ "ip": "0.0.0.0",
+ "launch_time": 1531941024680,
+ "state": "READY"
+ }
+ ],
+ "dependencies": [],
+ "launch_command": "sleep 900000",
+ "name": "sleeper",
+ "number_of_containers": 2,
+ "quicklinks": [],
+ "resource": {...},
+ "restart_policy": "ALWAYS",
+ "run_privileged_container": false,
+ "state": "STABLE"
+ }
+ ],
+ "configuration": {...},
+ "id": "application_1531508836237_0002",
+ "kerberos_principal": {},
+ "lifetime": -1,
+ "name": "my-sleeper",
+ "quicklinks": {},
+ "state": "STABLE",
+ "version": "1.0.0"
+}
+```
+
+### Enable Service Upgrade
+Below is the configuration in `yarn-site.xml` required for enabling service
+upgrade.
+
+```
+ <property>
+ <name>yarn.service.upgrade.enabled</name>
+ <value>true</value>
+ </property>
+```
+
+### Initiate Upgrade
+User can initiate upgrade using the below command:
+```
+yarn app -upgrade ${service_name} -initate ${path_to_new_service_def_file} [-autoFinalize]
+```
+
+e.g. To upgrade `my-sleeper` to sleep for *1200000* instead of *900000*, the user
+can upgrade the service to version 1.0.1. Below is the service definition for
+version 1.0.1 of sleeper-service:
+
+```
+{
+ "components" :
+ [
+ {
+ "name": "sleeper",
+ "version": "1.0.1",
+ "number_of_containers": 1,
+ "launch_command": "sleep 1200000",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+The command below initiates the upgrade to version 1.0.1.
+```
+yarn app -upgrade my-sleeper -initiate sleeper_v101.json
+```
+
+### Upgrade Instance
+User can upgrade a component instance using the below command:
+```
+yarn app -upgrade ${service_name} -instances ${comma_separated_list_of_instance_names}
+```
+e.g. The command below upgrades `sleeper-0` and `sleeper-1` instances of `my-service`:
+```
+yarn app -upgrade my-sleeper -instances sleeper-0,sleeper-1
+```
+
+### Upgrade Component
+User can upgrade a component, that is, all the instances of a component with
+one command:
+```
+yarn app -upgrade ${service_name} -components ${comma_separated_list_of_component_names}
+```
+e.g. The command below upgrades all the instances of `sleeper` component of `my-service`:
+```
+yarn app -ugrade my-sleeper -components sleeper
+```
+
+### Finalize Upgrade
+User must finalize the upgrade using the below command (since autoFinalize was not specified during initiate):
+```
+yarn app -upgrade ${service_name} -finalize
+```
+e.g. The command below finalizes the upgrade of `my-sleeper`:
+```
+yarn app -upgrade my-sleeper -finalize
+```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] hadoop git commit: HDFS-13726. RBF: Fix RBF configuration
links. Contributed by Takanobu Asanuma.
Posted by ey...@apache.org.
HDFS-13726. RBF: Fix RBF configuration links. Contributed by Takanobu Asanuma.
(cherry picked from commit 2ae13d41dcd4f49e6b4ebc099e5f8bb8280b9872)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99e98bf1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99e98bf1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99e98bf1
Branch: refs/remotes/origin/branch-3.1
Commit: 99e98bf19a70711935823bc82861cd67c1e89c1c
Parents: caf3853
Author: Yiqun Lin <yq...@apache.org>
Authored: Wed Jul 11 22:11:59 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Wed Jul 11 22:15:07 2018 +0800
----------------------------------------------------------------------
.../hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99e98bf1/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
index 70c6226..73e0f4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
@@ -175,7 +175,7 @@ Deployment
By default, the Router is ready to take requests and monitor the NameNode in the local machine.
It needs to know the State Store endpoint by setting `dfs.federation.router.store.driver.class`.
-The rest of the options are documented in [hdfs-default.xml](../hadoop-hdfs/hdfs-default.xml).
+The rest of the options are documented in [hdfs-rbf-default.xml](../hadoop-hdfs-rbf/hdfs-rbf-default.xml).
Once the Router is configured, it can be started:
@@ -290,7 +290,7 @@ Router configuration
--------------------
One can add the configurations for Router-based federation to **hdfs-site.xml**.
-The main options are documented in [hdfs-default.xml](../hadoop-hdfs/hdfs-default.xml).
+The main options are documented in [hdfs-rbf-default.xml](../hadoop-hdfs-rbf/hdfs-rbf-default.xml).
The configuration values are described in this section.
### RPC server
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] hadoop git commit: YARN-8528. Final states in
ContainerAllocation might be modified externally causing unexpected
allocation results. Contributed by Xintong Song.
Posted by ey...@apache.org.
YARN-8528. Final states in ContainerAllocation might be modified externally causing unexpected allocation results. Contributed by Xintong Song.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/004e1f24
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/004e1f24
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/004e1f24
Branch: refs/remotes/origin/branch-3.1
Commit: 004e1f248ef20b78f9d12d6f1fe04f66d8c56158
Parents: 823d576
Author: Weiwei Yang <ww...@apache.org>
Authored: Fri Jul 20 22:32:11 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Fri Jul 20 22:43:47 2018 +0800
----------------------------------------------------------------------
.../capacity/allocator/ContainerAllocation.java | 2 +-
.../allocator/RegularContainerAllocator.java | 10 ++--
.../capacity/TestCapacityScheduler.java | 48 ++++++++++++++++++++
3 files changed, 54 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/004e1f24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocation.java
index f408508..b9b9bcf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocation.java
@@ -56,7 +56,7 @@ public class ContainerAllocation {
RMContainer containerToBeUnreserved;
private Resource resourceToBeAllocated = Resources.none();
- AllocationState state;
+ private AllocationState state;
NodeType containerNodeType = NodeType.NODE_LOCAL;
NodeType requestLocalityType = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/004e1f24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index 99deb1a..adc27f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -263,7 +263,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
reservedContainer, schedulingMode, resourceLimits);
if (null == reservedContainer) {
- if (result.state == AllocationState.PRIORITY_SKIPPED) {
+ if (result.getAllocationState() == AllocationState.PRIORITY_SKIPPED) {
// Don't count 'skipped nodes' as a scheduling opportunity!
application.subtractSchedulingOpportunity(schedulerKey);
}
@@ -487,8 +487,8 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
// When a returned allocation is LOCALITY_SKIPPED, since we're in
// off-switch request now, we will skip this app w.r.t priorities
- if (allocation.state == AllocationState.LOCALITY_SKIPPED) {
- allocation.state = AllocationState.APP_SKIPPED;
+ if (allocation.getAllocationState() == AllocationState.LOCALITY_SKIPPED) {
+ allocation = ContainerAllocation.APP_SKIPPED;
}
allocation.requestLocalityType = requestLocalityType;
@@ -836,8 +836,8 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
result = tryAllocateOnNode(clusterResource, node, schedulingMode,
resourceLimits, schedulerKey, reservedContainer);
- if (AllocationState.ALLOCATED == result.state
- || AllocationState.RESERVED == result.state) {
+ if (AllocationState.ALLOCATED == result.getAllocationState()
+ || AllocationState.RESERVED == result.getAllocationState()) {
result = doAllocation(result, node, schedulerKey, reservedContainer);
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/004e1f24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 0b54010..79cdcfe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -134,6 +134,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AllocationState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.ContainerAllocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
@@ -4930,4 +4932,50 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
spyCs.handle(new NodeUpdateSchedulerEvent(
spyCs.getNode(nm.getNodeId()).getRMNode()));
}
+
+ // Testcase for YARN-8528
+ // This is to test whether ContainerAllocation constants are holding correct
+ // values during scheduling.
+ @Test
+ public void testContainerAllocationLocalitySkipped() throws Exception {
+ Assert.assertEquals(AllocationState.APP_SKIPPED,
+ ContainerAllocation.APP_SKIPPED.getAllocationState());
+ Assert.assertEquals(AllocationState.LOCALITY_SKIPPED,
+ ContainerAllocation.LOCALITY_SKIPPED.getAllocationState());
+ Assert.assertEquals(AllocationState.PRIORITY_SKIPPED,
+ ContainerAllocation.PRIORITY_SKIPPED.getAllocationState());
+ Assert.assertEquals(AllocationState.QUEUE_SKIPPED,
+ ContainerAllocation.QUEUE_SKIPPED.getAllocationState());
+
+ // init RM & NMs & Nodes
+ final MockRM rm = new MockRM(new CapacitySchedulerConfiguration());
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ rm.start();
+ final MockNM nm1 = rm.registerNode("h1:1234", 4 * GB);
+ final MockNM nm2 = rm.registerNode("h2:1234", 6 * GB); // maximum-allocation-mb = 6GB
+
+ // submit app and request resource
+ // container2 is larger than nm1 total resource, will trigger locality skip
+ final RMApp app = rm.submitApp(1 * GB, "app", "user");
+ final MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
+ am.addRequests(new String[] {"*"}, 5 * GB, 1, 1, 2);
+ am.schedule();
+
+ // container1 (am) should be acquired, container2 should not
+ RMNode node1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+ cs.handle(new NodeUpdateSchedulerEvent(node1));
+ ContainerId cid = ContainerId.newContainerId(am.getApplicationAttemptId(), 1l);
+ Assert.assertEquals(cs.getRMContainer(cid).getState(), RMContainerState.ACQUIRED);
+ cid = ContainerId.newContainerId(am.getApplicationAttemptId(), 2l);
+ Assert.assertNull(cs.getRMContainer(cid));
+
+ Assert.assertEquals(AllocationState.APP_SKIPPED,
+ ContainerAllocation.APP_SKIPPED.getAllocationState());
+ Assert.assertEquals(AllocationState.LOCALITY_SKIPPED,
+ ContainerAllocation.LOCALITY_SKIPPED.getAllocationState());
+ Assert.assertEquals(AllocationState.PRIORITY_SKIPPED,
+ ContainerAllocation.PRIORITY_SKIPPED.getAllocationState());
+ Assert.assertEquals(AllocationState.QUEUE_SKIPPED,
+ ContainerAllocation.QUEUE_SKIPPED.getAllocationState());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] hadoop git commit: Only mount non-empty directories for
cgroups (miklos.szegedi@cloudera.com via rkanter)
Posted by ey...@apache.org.
Only mount non-empty directories for cgroups (miklos.szegedi@cloudera.com via rkanter)
(cherry picked from commit 0838fe833738e04f5e6f6408e97866d77bebbf30)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c7d9163
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c7d9163
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c7d9163
Branch: refs/remotes/origin/branch-3.1
Commit: 1c7d916347d1c68ad32b592d764890b40b66e558
Parents: 27e2b4b
Author: Robert Kanter <rk...@apache.org>
Authored: Mon Jul 9 10:37:20 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Jul 18 16:07:48 2018 -0700
----------------------------------------------------------------------
.../impl/container-executor.c | 30 +++++++++++++++++++-
.../test/test-container-executor.c | 20 +++++++++++++
2 files changed, 49 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c7d9163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index baf0e8b..effeeee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -2379,6 +2379,28 @@ void chown_dir_contents(const char *dir_path, uid_t uid, gid_t gid) {
free(path_tmp);
}
+int is_empty(char *target_dir) {
+ DIR *dir = NULL;
+ struct dirent *entry = NULL;
+ dir = opendir(target_dir);
+ if (!dir) {
+ fprintf(LOGFILE, "Could not open directory %s - %s\n", target_dir,
+ strerror(errno));
+ return 0;
+ }
+ while ((entry = readdir(dir)) != NULL) {
+ if (strcmp(entry->d_name, ".") == 0) {
+ continue;
+ }
+ if (strcmp(entry->d_name, "..") == 0) {
+ continue;
+ }
+ fprintf(LOGFILE, "Directory is not empty %s\n", target_dir);
+ return 0;
+ }
+ return 1;
+}
+
/**
* Mount a cgroup controller at the requested mount point and create
* a hierarchy for the Hadoop NodeManager to manage.
@@ -2413,7 +2435,13 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
result = -1;
} else {
if (strstr(mount_path, "..") != NULL) {
- fprintf(LOGFILE, "Unsupported cgroup mount path detected.\n");
+ fprintf(LOGFILE, "Unsupported cgroup mount path detected. %s\n",
+ mount_path);
+ result = INVALID_COMMAND_PROVIDED;
+ goto cleanup;
+ }
+ if (!is_empty(mount_path)) {
+ fprintf(LOGFILE, "cgroup mount path is not empty. %s\n", mount_path);
result = INVALID_COMMAND_PROVIDED;
goto cleanup;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c7d9163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 3d32883..a199d84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -1203,6 +1203,23 @@ void test_trim_function() {
free(trimmed);
}
+void test_is_empty() {
+ printf("\nTesting is_empty function\n");
+ if (is_empty("/")) {
+ printf("FAIL: / should not be empty\n");
+ exit(1);
+ }
+ if (is_empty("/tmp/2938rf2983hcqnw8ud/noexist")) {
+ printf("FAIL: /tmp/2938rf2983hcqnw8ud/noexist should not exist\n");
+ exit(1);
+ }
+ mkdir("/tmp/2938rf2983hcqnw8ud/emptydir", S_IRWXU);
+ if (!is_empty("/tmp/2938rf2983hcqnw8ud/emptydir")) {
+ printf("FAIL: /tmp/2938rf2983hcqnw8ud/emptydir be empty\n");
+ exit(1);
+ }
+}
+
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -1264,6 +1281,9 @@ int main(int argc, char **argv) {
printf("\nStarting tests\n");
+ printf("\ntest_is_empty()\n");
+ test_is_empty();
+
printf("\nTesting recursive_unlink_children()\n");
test_recursive_unlink_children();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] hadoop git commit: YARN-8548. AllocationRespose proto
setNMToken initBuilder not done. Contributed by Bilwa S T.
Posted by ey...@apache.org.
YARN-8548. AllocationRespose proto setNMToken initBuilder not done. Contributed by Bilwa S T.
(cherry picked from commit ff7c2eda34c2c40ad71b50df6462a661bd213fbd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a684a2ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a684a2ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a684a2ef
Branch: refs/remotes/origin/branch-3.1
Commit: a684a2efb855e1933b0d808363c3c1fe69778867
Parents: 0710107
Author: bibinchundatt <bi...@apache.org>
Authored: Tue Jul 24 16:17:20 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Tue Jul 24 16:30:31 2018 +0530
----------------------------------------------------------------------
.../impl/pb/AllocateResponsePBImpl.java | 1 +
.../resourcemanager/recovery/TestProtos.java | 20 ++++++++++++++++++++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a684a2ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
index 3ab5563..8df56b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
@@ -347,6 +347,7 @@ public class AllocateResponsePBImpl extends AllocateResponse {
@Override
public synchronized void setNMTokens(List<NMToken> nmTokens) {
+ maybeInitBuilder();
if (nmTokens == null || nmTokens.isEmpty()) {
if (this.nmTokens != null) {
this.nmTokens.clear();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a684a2ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestProtos.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestProtos.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestProtos.java
index cc96412..d42b411 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestProtos.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestProtos.java
@@ -18,7 +18,15 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl;
+import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.EpochProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProto;
import org.junit.Assert;
import org.junit.Test;
@@ -33,4 +41,16 @@ public class TestProtos {
String protoString = proto.toString();
Assert.assertNotNull(protoString);
}
+
+ @Test
+ public void testProtoAllocateResponse() {
+ AllocateResponseProto proto = AllocateResponseProto.getDefaultInstance();
+ AllocateResponsePBImpl alloc = new AllocateResponsePBImpl(proto);
+ List<NMToken> nmTokens = new ArrayList<NMToken>();
+ try {
+ alloc.setNMTokens(nmTokens);
+ } catch (Exception ex) {
+ fail();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] hadoop git commit: YARN-8512. ATSv2 entities are not
published to HBase from second attempt onwards. Contributed by Rohith Sharma
K S.
Posted by ey...@apache.org.
YARN-8512. ATSv2 entities are not published to HBase from second attempt onwards. Contributed by Rohith Sharma K S.
(cherry picked from commit 7f1d3d0e9dbe328fae0d43421665e0b6907b33fe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b4ead92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b4ead92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b4ead92
Branch: refs/remotes/origin/branch-3.1
Commit: 9b4ead92c811b02bdfc62acf00fc364533eecab8
Parents: 6f10491
Author: Sunil G <su...@apache.org>
Authored: Wed Jul 11 12:26:32 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Jul 11 12:27:12 2018 +0530
----------------------------------------------------------------------
.../containermanager/ContainerManagerImpl.java | 69 ++++++++----
.../application/ApplicationImpl.java | 7 +-
.../BaseContainerManagerTest.java | 25 +++++
.../TestContainerManagerRecovery.java | 106 +++++++++++++++++--
4 files changed, 180 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b4ead92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 3470910..ad63720 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -1102,24 +1102,8 @@ public class ContainerManagerImpl extends CompositeService implements
// Create the application
// populate the flow context from the launch context if the timeline
// service v.2 is enabled
- FlowContext flowContext = null;
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
- String flowName = launchContext.getEnvironment()
- .get(TimelineUtils.FLOW_NAME_TAG_PREFIX);
- String flowVersion = launchContext.getEnvironment()
- .get(TimelineUtils.FLOW_VERSION_TAG_PREFIX);
- String flowRunIdStr = launchContext.getEnvironment()
- .get(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX);
- long flowRunId = 0L;
- if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) {
- flowRunId = Long.parseLong(flowRunIdStr);
- }
- flowContext = new FlowContext(flowName, flowVersion, flowRunId);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Flow context: " + flowContext
- + " created for an application " + applicationID);
- }
- }
+ FlowContext flowContext =
+ getFlowContext(launchContext, applicationID);
Application application =
new ApplicationImpl(dispatcher, user, flowContext,
@@ -1138,6 +1122,31 @@ public class ContainerManagerImpl extends CompositeService implements
dispatcher.getEventHandler().handle(new ApplicationInitEvent(
applicationID, appAcls, logAggregationContext));
}
+ } else if (containerTokenIdentifier.getContainerType()
+ == ContainerType.APPLICATION_MASTER) {
+ FlowContext flowContext =
+ getFlowContext(launchContext, applicationID);
+ if (flowContext != null) {
+ ApplicationImpl application =
+ (ApplicationImpl) context.getApplications().get(applicationID);
+
+ // update flowContext reference in ApplicationImpl
+ application.setFlowContext(flowContext);
+
+ // Required to update state store for recovery.
+ context.getNMStateStore().storeApplication(applicationID,
+ buildAppProto(applicationID, user, credentials,
+ container.getLaunchContext().getApplicationACLs(),
+ containerTokenIdentifier.getLogAggregationContext(),
+ flowContext));
+
+ LOG.info(
+ "Updated application reference with flowContext " + flowContext
+ + " for app " + applicationID);
+ } else {
+ LOG.info("TimelineService V2.0 is not enabled. Skipping updating "
+ + "flowContext for application " + applicationID);
+ }
}
this.context.getNMStateStore().storeContainer(containerId,
@@ -1163,6 +1172,30 @@ public class ContainerManagerImpl extends CompositeService implements
}
}
+ private FlowContext getFlowContext(ContainerLaunchContext launchContext,
+ ApplicationId applicationID) {
+ FlowContext flowContext = null;
+ if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ String flowName = launchContext.getEnvironment()
+ .get(TimelineUtils.FLOW_NAME_TAG_PREFIX);
+ String flowVersion = launchContext.getEnvironment()
+ .get(TimelineUtils.FLOW_VERSION_TAG_PREFIX);
+ String flowRunIdStr = launchContext.getEnvironment()
+ .get(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX);
+ long flowRunId = 0L;
+ if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) {
+ flowRunId = Long.parseLong(flowRunIdStr);
+ }
+ flowContext = new FlowContext(flowName, flowVersion, flowRunId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Flow context: " + flowContext + " created for an application "
+ + applicationID);
+ }
+ }
+ return flowContext;
+ }
+
protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier(
org.apache.hadoop.yarn.api.records.Token token,
ContainerTokenIdentifier containerTokenIdentifier) throws YarnException,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b4ead92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 6d84fb2..ad995fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -25,6 +25,8 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,7 +68,6 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
-import com.google.common.annotations.VisibleForTesting;
/**
* The state machine for the representation of an Application
@@ -688,4 +689,8 @@ public class ApplicationImpl implements Application {
public long getFlowRunId() {
return flowContext == null ? 0L : flowContext.getFlowRunId();
}
+
+ public void setFlowContext(FlowContext fc) {
+ this.flowContext = fc;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b4ead92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 93d0afb..b31601c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -429,6 +429,16 @@ public abstract class BaseContainerManagerTest {
}
public static Token createContainerToken(ContainerId cId, long rmIdentifier,
+ NodeId nodeId, String user,
+ NMContainerTokenSecretManager containerTokenSecretManager,
+ LogAggregationContext logAggregationContext, ContainerType containerType)
+ throws IOException {
+ Resource r = BuilderUtils.newResource(1024, 1);
+ return createContainerToken(cId, rmIdentifier, nodeId, user, r,
+ containerTokenSecretManager, logAggregationContext, containerType);
+ }
+
+ public static Token createContainerToken(ContainerId cId, long rmIdentifier,
NodeId nodeId, String user, Resource resource,
NMContainerTokenSecretManager containerTokenSecretManager,
LogAggregationContext logAggregationContext)
@@ -442,6 +452,21 @@ public abstract class BaseContainerManagerTest {
containerTokenIdentifier);
}
+ public static Token createContainerToken(ContainerId cId, long rmIdentifier,
+ NodeId nodeId, String user, Resource resource,
+ NMContainerTokenSecretManager containerTokenSecretManager,
+ LogAggregationContext logAggregationContext, ContainerType continerType)
+ throws IOException {
+ ContainerTokenIdentifier containerTokenIdentifier =
+ new ContainerTokenIdentifier(cId, nodeId.toString(), user, resource,
+ System.currentTimeMillis() + 100000L, 123, rmIdentifier,
+ Priority.newInstance(0), 0, logAggregationContext, null,
+ continerType);
+ return BuilderUtils.newContainerToken(nodeId,
+ containerTokenSecretManager.retrievePassword(containerTokenIdentifier),
+ containerTokenIdentifier);
+ }
+
public static Token createContainerToken(ContainerId cId, int version,
long rmIdentifier, NodeId nodeId, String user, Resource resource,
NMContainerTokenSecretManager containerTokenSecretManager,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b4ead92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index bf8b500..0a834af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.mock;
@@ -74,6 +75,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
@@ -205,7 +207,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
"includePatternInRollingAggregation",
"excludePatternInRollingAggregation");
StartContainersResponse startResponse = startContainer(context, cm, cid,
- clc, logAggregationContext);
+ clc, logAggregationContext, ContainerType.TASK);
assertTrue(startResponse.getFailedRequests().isEmpty());
assertEquals(1, context.getApplications().size());
Application app = context.getApplications().get(appId);
@@ -342,7 +344,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
null, null);
StartContainersResponse startResponse = startContainer(context, cm, cid,
- clc, null);
+ clc, null, ContainerType.TASK);
assertTrue(startResponse.getFailedRequests().isEmpty());
assertEquals(1, context.getApplications().size());
Application app = context.getApplications().get(appId);
@@ -579,7 +581,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
cm.init(conf);
cm.start();
StartContainersResponse startResponse = startContainer(context, cm, cid,
- clc, logAggregationContext);
+ clc, logAggregationContext, ContainerType.TASK);
assertEquals(1, startResponse.getSuccessfullyStartedContainers().size());
cm.stop();
verify(cm).handle(isA(CMgrCompletedAppsEvent.class));
@@ -595,7 +597,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
cm.init(conf);
cm.start();
startResponse = startContainer(context, cm, cid,
- clc, logAggregationContext);
+ clc, logAggregationContext, ContainerType.TASK);
assertEquals(1, startResponse.getSuccessfullyStartedContainers().size());
cm.stop();
memStore.close();
@@ -612,7 +614,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
cm.init(conf);
cm.start();
startResponse = startContainer(context, cm, cid,
- clc, logAggregationContext);
+ clc, logAggregationContext, ContainerType.TASK);
assertEquals(1, startResponse.getSuccessfullyStartedContainers().size());
cm.stop();
memStore.close();
@@ -661,7 +663,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
localResources, containerEnv, commands, serviceData,
containerTokens, acls);
StartContainersResponse startResponse = startContainer(
- context, cm, cid, clc, null);
+ context, cm, cid, clc, null, ContainerType.TASK);
assertTrue(startResponse.getFailedRequests().isEmpty());
assertEquals(1, context.getApplications().size());
// make sure the container reaches RUNNING state
@@ -736,14 +738,15 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
private StartContainersResponse startContainer(Context context,
final ContainerManagerImpl cm, ContainerId cid,
- ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
+ ContainerLaunchContext clc, LogAggregationContext logAggregationContext,
+ ContainerType containerType)
throws Exception {
UserGroupInformation user = UserGroupInformation.createRemoteUser(
cid.getApplicationAttemptId().toString());
StartContainerRequest scReq = StartContainerRequest.newInstance(
clc, TestContainerManager.createContainerToken(cid, 0,
context.getNodeId(), user.getShortUserName(),
- context.getContainerTokenSecretManager(), logAggregationContext));
+ context.getContainerTokenSecretManager(), logAggregationContext, containerType));
final List<StartContainerRequest> scReqList =
new ArrayList<StartContainerRequest>();
scReqList.add(scReq);
@@ -910,4 +913,91 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
}
}
+ @Test
+ public void testApplicationRecoveryAfterFlowContextUpdated()
+ throws Exception {
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
+ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, "yarn_admin_user");
+ NMStateStoreService stateStore = new NMMemoryStateStoreService();
+ stateStore.init(conf);
+ stateStore.start();
+ Context context = createContext(conf, stateStore);
+ ContainerManagerImpl cm = createContainerManager(context);
+ cm.init(conf);
+ cm.start();
+
+ // add an application by starting a container
+ String appName = "app_name1";
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+
+ // create 1nd attempt container with containerId 2
+ ContainerId cid = ContainerId.newContainerId(attemptId, 2);
+ Map<String, LocalResource> localResources = Collections.emptyMap();
+ Map<String, String> containerEnv = new HashMap<>();
+
+ List<String> containerCmds = Collections.emptyList();
+ Map<String, ByteBuffer> serviceData = Collections.emptyMap();
+ Credentials containerCreds = new Credentials();
+ DataOutputBuffer dob = new DataOutputBuffer();
+ containerCreds.writeTokenStorageToStream(dob);
+ ByteBuffer containerTokens =
+ ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ Map<ApplicationAccessType, String> acls =
+ new HashMap<ApplicationAccessType, String>();
+ ContainerLaunchContext clc = ContainerLaunchContext
+ .newInstance(localResources, containerEnv, containerCmds, serviceData,
+ containerTokens, acls);
+ // create the logAggregationContext
+ LogAggregationContext logAggregationContext = LogAggregationContext
+ .newInstance("includePattern", "excludePattern",
+ "includePatternInRollingAggregation",
+ "excludePatternInRollingAggregation");
+
+ StartContainersResponse startResponse =
+ startContainer(context, cm, cid, clc, logAggregationContext,
+ ContainerType.TASK);
+ assertTrue(startResponse.getFailedRequests().isEmpty());
+ assertEquals(1, context.getApplications().size());
+ ApplicationImpl app =
+ (ApplicationImpl) context.getApplications().get(appId);
+ assertNotNull(app);
+ waitForAppState(app, ApplicationState.INITING);
+ assertNull(app.getFlowName());
+
+ // 2nd attempt
+ ApplicationAttemptId attemptId2 =
+ ApplicationAttemptId.newInstance(appId, 2);
+ // create 2nd attempt master container
+ ContainerId cid2 = ContainerId.newContainerId(attemptId, 1);
+ setFlowContext(containerEnv, appName, appId);
+ // once again create for updating launch context
+ clc = ContainerLaunchContext
+ .newInstance(localResources, containerEnv, containerCmds, serviceData,
+ containerTokens, acls);
+ // start container with container type AM.
+ startResponse =
+ startContainer(context, cm, cid2, clc, logAggregationContext,
+ ContainerType.APPLICATION_MASTER);
+ assertTrue(startResponse.getFailedRequests().isEmpty());
+ assertEquals(1, context.getApplications().size());
+ waitForAppState(app, ApplicationState.INITING);
+ assertEquals(appName, app.getFlowName());
+
+ // reset container manager and verify flow context information
+ cm.stop();
+ context = createContext(conf, stateStore);
+ cm = createContainerManager(context);
+ cm.init(conf);
+ cm.start();
+ assertEquals(1, context.getApplications().size());
+ app = (ApplicationImpl) context.getApplications().get(appId);
+ assertNotNull(app);
+ assertEquals(appName, app.getFlowName());
+ waitForAppState(app, ApplicationState.INITING);
+
+ cm.stop();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] hadoop git commit: HDFS-11060. make
DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED configurable. Contributed by Lantao
Jin.
Posted by ey...@apache.org.
HDFS-11060. make DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED configurable. Contributed by Lantao Jin.
(cherry picked from commit e95c5e9f62452ee848875ec2f8642eab4992cd23)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/964f3454
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/964f3454
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/964f3454
Branch: refs/remotes/origin/branch-3.1
Commit: 964f3454d1237fa25aa828504612b4a11d8e76ff
Parents: 830ef12
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Wed Jul 25 11:04:18 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Wed Jul 25 11:05:18 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +++
.../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 8 ++++++--
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 9 +++++++++
.../hdfs/server/namenode/TestListCorruptFileBlocks.java | 2 +-
4 files changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/964f3454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index dde7eb7..ea3abb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -238,6 +238,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT
= 1;
+ public static final String DFS_NAMENODE_MAX_CORRUPT_FILE_BLOCKS_RETURNED_KEY = "dfs.namenode.max-corrupt-file-blocks-returned";
+ public static final int DFS_NAMENODE_MAX_CORRUPT_FILE_BLOCKS_RETURNED_DEFAULT = 100;
+
public static final String DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY;
public static final int DFS_NAMENODE_REPLICATION_MAX_STREAMS_DEFAULT = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/964f3454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 19ff08d..2098252 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -425,7 +425,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
public static final Log auditLog = LogFactory.getLog(
FSNamesystem.class.getName() + ".audit");
- static final int DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED = 100;
+ private final int maxCorruptFileBlocksReturn;
static int BLOCK_DELETION_INCREMENT = 1000;
private final boolean isPermissionEnabled;
private final UserGroupInformation fsOwner;
@@ -831,6 +831,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
DFSConfigKeys.DFS_NAMENODE_FILE_CLOSE_NUM_COMMITTED_ALLOWED_KEY,
DFSConfigKeys.DFS_NAMENODE_FILE_CLOSE_NUM_COMMITTED_ALLOWED_DEFAULT);
+ this.maxCorruptFileBlocksReturn = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_MAX_CORRUPT_FILE_BLOCKS_RETURNED_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MAX_CORRUPT_FILE_BLOCKS_RETURNED_DEFAULT);
+
this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);
this.standbyShouldCheckpoint = conf.getBoolean(
@@ -5497,7 +5501,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (src.startsWith(path)){
corruptFiles.add(new CorruptFileBlockInfo(src, blk));
count++;
- if (count >= DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED)
+ if (count >= maxCorruptFileBlocksReturn)
break;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/964f3454/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index c092bff..4bdeef6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -613,6 +613,15 @@
</property>
<property>
+ <name>dfs.namenode.max-corrupt-file-blocks-returned</name>
+ <value>100</value>
+ <description>
+ The maximum number of corrupt file blocks listed by NameNode Web UI,
+ JMX and other client request.
+ </description>
+</property>
+
+<property>
<name>dfs.blocksize</name>
<value>134217728</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/964f3454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java
index 1f31bdc..e1c8ae3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java
@@ -452,7 +452,7 @@ public class TestListCorruptFileBlocks {
cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem();
final int maxCorruptFileBlocks =
- FSNamesystem.DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED;
+ conf.getInt(DFSConfigKeys.DFS_NAMENODE_MAX_CORRUPT_FILE_BLOCKS_RETURNED_KEY, 100);
// create 110 files with one block each
DFSTestUtil util = new DFSTestUtil.Builder().setName("testMaxCorruptFiles").
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org