You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2018/01/18 21:47:28 UTC
[1/2] hadoop git commit: YARN-7590. Improve container-executor
validation check. Contributed by Eric Yang.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 f638ff904 -> 0a8a79988
YARN-7590. Improve container-executor validation check. Contributed by Eric Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27c2ade4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27c2ade4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27c2ade4
Branch: refs/heads/branch-2.6
Commit: 27c2ade4d2ae9697dcb6b4db8ea2b15c0b507bf3
Parents: f638ff9
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Jan 17 22:20:39 2018 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Thu Jan 18 12:50:49 2018 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../impl/container-executor.c | 39 +++++++++++++++++
.../impl/container-executor.h | 5 +++
.../test/test-container-executor.c | 46 ++++++++++++++++----
4 files changed, 85 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27c2ade4/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ac28b0c..9a5770c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -19,6 +19,9 @@ Release 2.6.6 - UNRELEASED
HADOOP-14474. Use OpenJDK 7 instead of Oracle JDK 7 to avoid
oracle-java7-installer failures. (Akira Ajisaka via xiao)
+ YARN-7590. Improve container-executor validation check.
+ Contributed by Eric Yang.
+
Release 2.6.5 - 2016-10-08
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27c2ade4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 2345594..d07de2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -406,10 +406,32 @@ char *get_app_directory(const char * nm_root, const char *user,
* Get the user directory of a particular user
*/
char *get_user_directory(const char *nm_root, const char *user) {
+ int result = check_nm_local_dir(nm_uid, nm_root);
+ if (result != 0) {
+ return NULL;
+ }
return concatenate(USER_DIR_PATTERN, "user_dir_path", 2, nm_root, user);
}
/**
+ * Check node manager local dir permission.
+ */
+int check_nm_local_dir(uid_t caller_uid, const char *nm_root) {
+ struct stat info;
+ errno = 0;
+ int err = stat(nm_root, &info);
+ if (err < 0) {
+ fprintf(LOGFILE, "Error checking file stats for %s %d %s.\n", nm_root, err, strerror(errno));
+ return 1;
+ }
+ if (caller_uid != info.st_uid) {
+ fprintf(LOGFILE, "Permission mismatch for %s for caller uid: %d, owner uid: %d.\n", nm_root, caller_uid, info.st_uid);
+ return 1;
+ }
+ return 0;
+}
+
+/**
* Get the container directory for the given container_id
*/
char *get_container_work_directory(const char *nm_root, const char *user,
@@ -552,6 +574,11 @@ static int create_container_directories(const char* user, const char *app_id,
for(local_dir_ptr = local_dir; *local_dir_ptr != NULL; ++local_dir_ptr) {
char *container_dir = get_container_work_directory(*local_dir_ptr, user, app_id,
container_id);
+ int check = check_nm_local_dir(nm_uid, *local_dir_ptr);
+ if (check != 0) {
+ free(container_dir);
+ continue;
+ }
if (container_dir == NULL) {
return -1;
}
@@ -578,6 +605,14 @@ static int create_container_directories(const char* user, const char *app_id,
char* const* log_dir_ptr;
for(log_dir_ptr = log_dir; *log_dir_ptr != NULL; ++log_dir_ptr) {
char *container_log_dir = get_app_log_directory(*log_dir_ptr, combined_name);
+ int check = check_nm_local_dir(nm_uid, *log_dir_ptr);
+ if (check != 0) {
+ container_log_dir = NULL;
+ }
+ if (strstr(container_log_dir, "..") != 0) {
+ fprintf(LOGFILE, "Unsupported container log directory path detected.\n");
+ container_log_dir = NULL;
+ }
if (container_log_dir == NULL) {
free(combined_name);
return -1;
@@ -891,6 +926,10 @@ int create_log_dirs(const char *app_id, char * const * log_dirs) {
char *any_one_app_log_dir = NULL;
for(log_root=log_dirs; *log_root != NULL; ++log_root) {
char *app_log_dir = get_app_log_directory(*log_root, app_id);
+ int result = check_nm_local_dir(nm_uid, *log_root);
+ if (result != 0) {
+ app_log_dir = NULL;
+ }
if (app_log_dir == NULL) {
// try the next one
} else if (create_directory_for_user(app_log_dir) != 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27c2ade4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index b1efd6a..d994bae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -168,6 +168,11 @@ char *get_user_directory(const char *nm_root, const char *user);
char *get_app_directory(const char * nm_root, const char *user,
const char *app_id);
+/**
+ * Check node manager local dir permission.
+ */
+int check_nm_local_dir(uid_t caller_uid, const char *nm_root);
+
char *get_container_work_directory(const char *nm_root, const char *user,
const char *app_id, const char *container_id);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27c2ade4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index ad6d740..25cfc5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -39,6 +39,7 @@ static char* username = NULL;
static char* yarn_username = NULL;
static char** local_dirs = NULL;
static char** log_dirs = NULL;
+static uid_t nm_uid = -1;
/**
* Run the command using the effective user id.
@@ -149,8 +150,8 @@ void check_pid_file(const char* pid_file, pid_t mypid) {
}
void test_get_user_directory() {
- char *user_dir = get_user_directory("/tmp", "user");
- char *expected = "/tmp/usercache/user";
+ char *user_dir = get_user_directory(TEST_ROOT, "user");
+ char *expected = TEST_ROOT "/usercache/user";
if (strcmp(user_dir, expected) != 0) {
printf("test_get_user_directory expected %s got %s\n", expected, user_dir);
exit(1);
@@ -158,9 +159,32 @@ void test_get_user_directory() {
free(user_dir);
}
+void test_check_nm_local_dir() {
+ // check filesystem is same as running user.
+ int expected = 0;
+ char *local_path = TEST_ROOT "target";
+ char *root_path = "/";
+ if (mkdirs(local_path, 0700) != 0) {
+ printf("FAIL: unble to create node manager local directory: %s\n", local_path);
+ exit(1);
+ }
+ int actual = check_nm_local_dir(nm_uid, local_path);
+ if (expected != actual) {
+ printf("test_nm_local_dir expected %d got %d\n", expected, actual);
+ exit(1);
+ }
+ // check filesystem is different from running user.
+ expected = 1;
+ actual = check_nm_local_dir(nm_uid, root_path);
+ if (expected != actual && nm_uid != 0) {
+ printf("test_nm_local_dir expected %d got %d\n", expected, actual);
+ exit(1);
+ }
+}
+
void test_get_app_directory() {
- char *expected = "/tmp/usercache/user/appcache/app_200906101234_0001";
- char *app_dir = (char *) get_app_directory("/tmp", "user",
+ char *expected = TEST_ROOT "/usercache/user/appcache/app_200906101234_0001";
+ char *app_dir = (char *) get_app_directory(TEST_ROOT, "user",
"app_200906101234_0001");
if (strcmp(app_dir, expected) != 0) {
printf("test_get_app_directory expected %s got %s\n", expected, app_dir);
@@ -170,9 +194,9 @@ void test_get_app_directory() {
}
void test_get_container_directory() {
- char *container_dir = get_container_work_directory("/tmp", "owen", "app_1",
+ char *container_dir = get_container_work_directory(TEST_ROOT, "owen", "app_1",
"container_1");
- char *expected = "/tmp/usercache/owen/appcache/app_1/container_1";
+ char *expected = TEST_ROOT "/usercache/owen/appcache/app_1/container_1";
if (strcmp(container_dir, expected) != 0) {
printf("Fail get_container_work_directory got %s expected %s\n",
container_dir, expected);
@@ -182,9 +206,9 @@ void test_get_container_directory() {
}
void test_get_container_launcher_file() {
- char *expected_file = ("/tmp/usercache/user/appcache/app_200906101234_0001"
+ char *expected_file = (TEST_ROOT "/usercache/user/appcache/app_200906101234_0001"
"/launch_container.sh");
- char *app_dir = get_app_directory("/tmp", "user",
+ char *app_dir = get_app_directory(TEST_ROOT, "user",
"app_200906101234_0001");
char *container_file = get_container_launcher_file(app_dir);
if (strcmp(container_file, expected_file) != 0) {
@@ -691,6 +715,9 @@ int main(int argc, char **argv) {
LOGFILE = stdout;
ERRORFILE = stderr;
+ nm_uid = getuid();
+
+ printf("Attempting to clean up from any previous runs\n");
// clean up any junk from previous run
if (system("chmod -R u=rwx " TEST_ROOT "; rm -fr " TEST_ROOT)) {
exit(1);
@@ -734,6 +761,9 @@ int main(int argc, char **argv) {
printf("\nTesting get_user_directory()\n");
test_get_user_directory();
+ printf("\nTesting check_nm_local_dir()\n");
+ test_check_nm_local_dir();
+
printf("\nTesting get_app_directory()\n");
test_get_app_directory();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HADOOP-14842. Hadoop 2.8.2 release build
process get stuck due to java issue. Contributed by Junping Du.
Posted by sz...@apache.org.
HADOOP-14842. Hadoop 2.8.2 release build process get stuck due to java issue. Contributed by Junping Du.
(cherry picked from commit d0a0f24abc42957c885d5076f8c8e7945e074ba8)
(cherry picked from commit eaf5c66f7ec8c34e34541b7398fe59228f5ef2d8)
Conflicts:
dev-support/bin/create-release
dev-support/docker/Dockerfile
(cherry picked from commit ea57d107b630cabbeadc50f6ef5506095b2abb58)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a8a7998
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a8a7998
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a8a7998
Branch: refs/heads/branch-2.6
Commit: 0a8a799884f5e1e9cd6cca9946f98e869a33776c
Parents: 27c2ade
Author: Junping Du <ju...@apache.org>
Authored: Fri Sep 8 13:07:52 2017 -0700
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Thu Jan 18 12:55:03 2018 -0800
----------------------------------------------------------------------
dev-support/docker/Dockerfile | 7 +++----
hadoop-common-project/hadoop-common/CHANGES.txt | 5 ++++-
2 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a8a7998/dev-support/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index e1fff8a..627451c 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -46,16 +46,15 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
#######
-# Oracle Java
+# Java OpenJDK
#######
RUN apt-get install -y software-properties-common
RUN add-apt-repository -y ppa:webupd8team/java
RUN apt-get update
-# Auto-accept the Oracle JDK license
-RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
-RUN apt-get install -y oracle-java8-installer
+# Install OpenJDK 7
+RUN apt-get install -y openjdk-7-jdk
######
# Install findbugs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a8a7998/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9a5770c..248bfb6 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -22,6 +22,9 @@ Release 2.6.6 - UNRELEASED
YARN-7590. Improve container-executor validation check.
Contributed by Eric Yang.
+ HADOOP-14842. Hadoop 2.8.2 release build process get stuck due to java
+ issue. Contributed by Junping Du.
+
Release 2.6.5 - 2016-10-08
INCOMPATIBLE CHANGES
@@ -4368,7 +4371,7 @@ Release 0.23.1 - 2012-02-17
HADOOP-7841. Run tests with non-secure random. (tlipcon)
- HADOOP-7851. Configuration.getClasses() never returns the default value.
+ HADOOP-7851. Configuration.getClasses() never returns the default value.
(Uma Maheswara Rao G via amarrk)
HADOOP-7787. Make source tarball use conventional name.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org