You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2018/01/13 01:41:13 UTC
hadoop git commit: YARN-7705. Create the container log directory with
correct sticky bit in C code. Contributed by Yufei Gu.
Repository: hadoop
Updated Branches:
refs/heads/trunk d5d6a0353 -> 2dcfc1876
YARN-7705. Create the container log directory with correct sticky bit in C code. Contributed by Yufei Gu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2dcfc187
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2dcfc187
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2dcfc187
Branch: refs/heads/trunk
Commit: 2dcfc1876e4d73cf85a6b1b7de694b1b4cc54494
Parents: d5d6a03
Author: Miklos Szegedi <sz...@apache.org>
Authored: Fri Jan 12 15:03:29 2018 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Fri Jan 12 17:00:01 2018 -0800
----------------------------------------------------------------------
.../nodemanager/LinuxContainerExecutor.java | 1 +
.../localizer/ContainerLocalizer.java | 27 ---------
.../impl/container-executor.c | 58 ++++++++++++++++++++
.../impl/container-executor.h | 3 +
.../main/native/container-executor/impl/main.c | 6 +-
.../test/test-container-executor.c | 11 +++-
.../TestLinuxContainerExecutorWithMocks.java | 40 ++++++++------
7 files changed, 98 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dcfc187/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index fc5a04d..d359f31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -365,6 +365,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
PrivilegedOperation.RunAsUserCommand.INITIALIZE_CONTAINER
.getValue()),
appId,
+ locId,
nmPrivateContainerTokensPath.toUri().getPath().toString(),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
localDirs),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dcfc187/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index 541911a..639a69d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -63,7 +63,6 @@ import org.apache.hadoop.util.DiskValidatorFactory;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -447,7 +446,6 @@ public class ContainerLocalizer {
// MKDIR $x/$user/appcache/$appid/filecache
// LOAD $x/$user/appcache/$appid/appTokens
try {
- createLogDir();
String user = argv[0];
String appId = argv[1];
String locId = argv[2];
@@ -483,31 +481,6 @@ public class ContainerLocalizer {
}
}
- /**
- * Create the log directory, if the directory exists, make sure its permission
- * is 750.
- */
- private static void createLogDir() {
- FileContext localFs;
- try {
- localFs = FileContext.getLocalFSFileContext(new Configuration());
-
- String logDir = System.getProperty(
- YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR);
-
- if (logDir != null && !logDir.trim().isEmpty()) {
- Path containerLogPath = new Path(logDir);
- FsPermission containerLogDirPerm= new FsPermission((short)0750);
- localFs.mkdir(containerLogPath, containerLogDirPerm, true);
- // set permission again to make sure the permission is correct
- // in case the directory is already there.
- localFs.setPermission(containerLogPath, containerLogDirPerm);
- }
- } catch (IOException e) {
- throw new YarnRuntimeException("Unable to create the log dir", e);
- }
- }
-
private static void initDirs(Configuration conf, String user, String appId,
FileContext lfs, List<Path> localDirs) throws IOException {
if (null == localDirs || 0 == localDirs.size()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dcfc187/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index c1977e5..64052fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1094,11 +1094,62 @@ int create_log_dirs(const char *app_id, char * const * log_dirs) {
return 0;
}
+char* get_container_log_directory(const char *log_root, const char* app_id,
+ const char *container_id) {
+ return concatenate("%s/%s/%s", "container log dir", 3, log_root, app_id,
+ container_id);
+}
+
+int create_container_log_dirs(const char *container_id, const char *app_id,
+ char * const * log_dirs) {
+ char* const* log_root;
+ int created_any_dir = 0;
+ for(log_root=log_dirs; *log_root != NULL; ++log_root) {
+ char *container_log_dir = get_container_log_directory(*log_root, app_id,
+ container_id);
+
+ if (container_log_dir == NULL) {
+ fprintf(LOGFILE,
+ "Failed to get container log directory name! Log root directory: %s, App id: %s, Container id: %s\n",
+ *log_root, app_id, container_id);
+ continue;
+ }
+
+ int result = check_nm_local_dir(nm_uid, *log_root);
+ if (result != 0 && container_log_dir != NULL) {
+ fprintf(LOGFILE, "Unsupported container log directory path (%s) detected.\n",
+ container_log_dir);
+ free(container_log_dir);
+ container_log_dir = NULL;
+ continue;
+ }
+
+ if (create_directory_for_user(container_log_dir) != 0) {
+ fprintf(LOGFILE, "Failed to create container log directory (%s)!\n",
+ container_log_dir);
+ free(container_log_dir);
+ return -1;
+ }
+
+ if (!created_any_dir) {
+ created_any_dir = 1;
+ }
+
+ free(container_log_dir);
+ }
+
+ if (!created_any_dir) {
+ fprintf(LOGFILE, "Did not create any container log directory.\n");
+ return -1;
+ }
+ return 0;
+}
/**
* Function to prepare the application directories for the container.
*/
int initialize_app(const char *user, const char *app_id,
+ const char *container_id,
const char* nmPrivate_credentials_file,
char* const* local_dirs, char* const* log_roots,
char* const* args) {
@@ -1119,6 +1170,13 @@ int initialize_app(const char *user, const char *app_id,
return log_create_result;
}
+ // create the log directories for the container on all disks
+ int container_log_create_result = create_container_log_dirs(container_id,
+ app_id, log_roots);
+ if (container_log_create_result != 0) {
+ return container_log_create_result;
+ }
+
// open up the credentials file
int cred_file = open_file_as_nm(nmPrivate_credentials_file);
if (cred_file == -1) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dcfc187/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index c5f3d99..5835214 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -91,6 +91,7 @@ void free_executor_configurations();
// initialize the application directory
int initialize_app(const char *user, const char *app_id,
+ const char *container_id,
const char *credentials, char* const* local_dirs,
char* const* log_dirs, char* const* args);
@@ -195,6 +196,8 @@ char *get_container_credentials_file(const char* work_dir);
*/
char* get_app_log_directory(const char* log_root, const char* appid);
+char* get_container_log_directory(const char *log_root, const char* app_id,
+ const char *container_id);
/**
* Ensure that the given path and all of the parent directories are created
* with the desired permissions.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dcfc187/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index ca84d40..259719a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -360,13 +360,14 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
char * resources_value = NULL;
switch (command) {
case INITIALIZE_CONTAINER:
- if (argc < 9) {
- fprintf(ERRORFILE, "Too few arguments (%d vs 9) for initialize container\n",
+ if (argc < 10) {
+ fprintf(ERRORFILE, "Too few arguments (%d vs 10) for initialize container\n",
argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
cmd_input.app_id = argv[optind++];
+ cmd_input.container_id = argv[optind++];
cmd_input.cred_file = argv[optind++];
cmd_input.local_dirs = argv[optind++];// good local dirs as a comma separated list
cmd_input.log_dirs = argv[optind++];// good log dirs as a comma separated list
@@ -563,6 +564,7 @@ int main(int argc, char **argv) {
exit_code = initialize_app(cmd_input.yarn_user_name,
cmd_input.app_id,
+ cmd_input.container_id,
cmd_input.cred_file,
split(cmd_input.local_dirs),
split(cmd_input.log_dirs),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dcfc187/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 1a6d4e6..8b0c0fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -798,7 +798,8 @@ void test_init_app() {
exit(1);
} else if (child == 0) {
char *final_pgm[] = {"touch", "my-touch-file", 0};
- if (initialize_app(yarn_username, "app_4", TEST_ROOT "/creds.txt",
+ if (initialize_app(yarn_username, "app_4", "container_1",
+ TEST_ROOT "/creds.txt",
local_dirs, log_dirs, final_pgm) != 0) {
printf("FAIL: failed in child\n");
exit(42);
@@ -839,6 +840,14 @@ void test_init_app() {
exit(1);
}
free(app_dir);
+
+ char *container_dir = get_container_log_directory(TEST_ROOT "/logs/userlogs",
+ "app_4", "container_1");
+ if (container_dir != NULL && access(container_dir, R_OK) != 0) {
+ printf("FAIL: failed to create container log directory %s\n", container_dir);
+ exit(1);
+ }
+ free(container_dir);
}
void test_run_container() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dcfc187/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index 9a56656..0632482 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -253,8 +253,6 @@ public class TestLinuxContainerExecutorWithMocks {
@Test (timeout = 5000)
public void testStartLocalizer() throws IOException {
-
-
InetSocketAddress address = InetSocketAddress.createUnresolved("localhost", 8040);
Path nmPrivateCTokensPath= new Path("file:///bin/nmPrivateCTokensPath");
@@ -269,25 +267,31 @@ public class TestLinuxContainerExecutorWithMocks {
.build());
List<String> result=readMockParams();
- Assert.assertEquals(result.size(), 24);
+ Assert.assertEquals(result.size(), 25);
Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);
Assert.assertEquals(result.get(1), "test");
Assert.assertEquals(result.get(2), "0" );
- Assert.assertEquals(result.get(3),"application_0" );
- Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath");
- Assert.assertEquals(result.get(8), "-classpath" );
- Assert.assertEquals(result.get(11), "-Xmx256m" );
- Assert.assertEquals(result.get(12), "-Dlog4j.configuration=container-log4j.properties" );
- Assert.assertEquals(result.get(13), "-Dyarn.app.container.log.dir=${yarn.log.dir}/userlogs/application_0/12345");
- Assert.assertEquals(result.get(14), "-Dyarn.app.container.log.filesize=0");
- Assert.assertEquals(result.get(15), "-Dhadoop.root.logger=INFO,CLA");
- Assert.assertEquals(result.get(16), "-Dhadoop.root.logfile=container-localizer-syslog");
- Assert.assertEquals(result.get(17),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
- Assert.assertEquals(result.get(18), "test");
- Assert.assertEquals(result.get(19), "application_0");
- Assert.assertEquals(result.get(20),"12345" );
- Assert.assertEquals(result.get(21),"localhost" );
- Assert.assertEquals(result.get(22),"8040" );
+ Assert.assertEquals(result.get(3), "application_0");
+ Assert.assertEquals(result.get(4), "12345");
+ Assert.assertEquals(result.get(5), "/bin/nmPrivateCTokensPath");
+ Assert.assertEquals(result.get(9), "-classpath" );
+ Assert.assertEquals(result.get(12), "-Xmx256m" );
+ Assert.assertEquals(result.get(13),
+ "-Dlog4j.configuration=container-log4j.properties" );
+ Assert.assertEquals(result.get(14),
+ "-Dyarn.app.container.log.dir=${yarn.log.dir}/userlogs/application_0/12345");
+ Assert.assertEquals(result.get(15),
+ "-Dyarn.app.container.log.filesize=0");
+ Assert.assertEquals(result.get(16), "-Dhadoop.root.logger=INFO,CLA");
+ Assert.assertEquals(result.get(17),
+ "-Dhadoop.root.logfile=container-localizer-syslog");
+ Assert.assertEquals(result.get(18),
+ "org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer");
+ Assert.assertEquals(result.get(19), "test");
+ Assert.assertEquals(result.get(20), "application_0");
+ Assert.assertEquals(result.get(21), "12345");
+ Assert.assertEquals(result.get(22), "localhost");
+ Assert.assertEquals(result.get(23), "8040");
} catch (InterruptedException e) {
LOG.error("Error:"+e.getMessage(),e);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org