You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/12/20 06:21:33 UTC
[1/8] hadoop git commit: YARN-9129. Ensure flush after printing to
log plus additional cleanup. Contributed by Eric Yang
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12943 2753d6c93 -> f7072e070
YARN-9129. Ensure flush after printing to log plus additional cleanup. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e544dc9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e544dc9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e544dc9
Branch: refs/heads/HDFS-12943
Commit: 2e544dc921afeaa02e731cb273ac7776eec6e49d
Parents: e815fd9
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Dec 19 16:45:23 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Wed Dec 19 16:45:23 2018 -0800
----------------------------------------------------------------------
.../client/api/ContainerShellWebSocket.java | 7 +
.../yarn/client/api/impl/YarnClientImpl.java | 9 +-
.../impl/container-executor.c | 132 +++++++++++++++++--
.../main/native/container-executor/impl/main.c | 3 +
.../test/test-container-executor.c | 4 +-
5 files changed, 139 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e544dc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java
index 4b7b2ac..efcc2ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java
@@ -52,9 +52,16 @@ public class ContainerShellWebSocket {
private Session mySession;
private Terminal terminal;
private LineReader reader;
+ private boolean sttySet = false;
@OnWebSocketMessage
public void onText(Session session, String message) throws IOException {
+ if (!sttySet) {
+ session.getRemote().sendString("stty -echo");
+ session.getRemote().sendString("\r");
+ session.getRemote().flush();
+ sttySet = true;
+ }
terminal.output().write(message.getBytes(Charset.forName("UTF-8")));
terminal.output().flush();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e544dc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index a7487f6..28a7f85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -1121,13 +1121,10 @@ public class YarnClientImpl extends YarnClient {
}
// Attempt Connect
Future<Session> fut = client.connect(socket, uri, upgradeRequest);
- // Wait for Connect
Session session = fut.get();
- // Send a message
- session.getRemote().sendString("stty -echo");
- session.getRemote().sendString("\r");
- session.getRemote().flush();
- socket.run();
+ if (session.isOpen()) {
+ socket.run();
+ }
} finally {
client.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e544dc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 65bb803..cba84ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -143,6 +143,7 @@ int check_executor_permissions(char *executable_file) {
fprintf(ERRORFILE,
"Error resolving the canonical name for the executable : %s!",
strerror(errno));
+ fflush(ERRORFILE);
return -1;
}
@@ -151,6 +152,7 @@ int check_executor_permissions(char *executable_file) {
if (stat(resolved_path, &filestat) != 0) {
fprintf(ERRORFILE,
"Could not stat the executable : %s!.\n", strerror(errno));
+ fflush(ERRORFILE);
return -1;
}
@@ -161,12 +163,14 @@ int check_executor_permissions(char *executable_file) {
if (binary_euid != 0) {
fprintf(LOGFILE,
"The container-executor binary should be user-owned by root.\n");
+ fflush(LOGFILE);
return -1;
}
if (binary_gid != getgid()) {
fprintf(LOGFILE, "The configured nodemanager group %d is different from"
" the group of the executable %d\n", getgid(), binary_gid);
+ fflush(LOGFILE);
return -1;
}
@@ -176,12 +180,14 @@ int check_executor_permissions(char *executable_file) {
fprintf(LOGFILE,
"The container-executor binary should not have write or execute "
"for others.\n");
+ fflush(LOGFILE);
return -1;
}
// Binary should be setuid executable
if ((filestat.st_mode & S_ISUID) == 0) {
fprintf(LOGFILE, "The container-executor binary should be set setuid.\n");
+ fflush(LOGFILE);
return -1;
}
@@ -201,11 +207,13 @@ static int change_effective_user(uid_t user, gid_t group) {
if (setegid(group) != 0) {
fprintf(LOGFILE, "Failed to set effective group id %d - %s\n", group,
strerror(errno));
+ fflush(LOGFILE);
return -1;
}
if (seteuid(user) != 0) {
fprintf(LOGFILE, "Failed to set effective user id %d - %s\n", user,
strerror(errno));
+ fflush(LOGFILE);
return -1;
}
return 0;
@@ -230,6 +238,7 @@ static int write_pid_to_cgroup_as_root(const char* cgroup_file, pid_t pid) {
if (cgroup_fd == -1) {
fprintf(LOGFILE, "Can't open file %s as node manager - %s\n", cgroup_file,
strerror(errno));
+ fflush(LOGFILE);
rc = -1;
goto cleanup;
}
@@ -242,6 +251,7 @@ static int write_pid_to_cgroup_as_root(const char* cgroup_file, pid_t pid) {
if (written == -1) {
fprintf(LOGFILE, "Failed to write pid to file %s - %s\n",
cgroup_file, strerror(errno));
+ fflush(LOGFILE);
rc = -1;
goto cleanup;
}
@@ -347,6 +357,7 @@ static int write_exit_code_file_as_nm(const char* exit_code_file,
if (ecode_fd == -1) {
fprintf(LOGFILE, "Can't open file %s - %s\n", tmp_ecode_file,
strerror(errno));
+ fflush(LOGFILE);
rc = -1;
goto cleanup;
}
@@ -358,6 +369,7 @@ static int write_exit_code_file_as_nm(const char* exit_code_file,
if (written == -1) {
fprintf(LOGFILE, "Failed to write exit code to file %s - %s\n",
tmp_ecode_file, strerror(errno));
+ fflush(LOGFILE);
rc = -1;
goto cleanup;
}
@@ -367,6 +379,7 @@ static int write_exit_code_file_as_nm(const char* exit_code_file,
if (rename(tmp_ecode_file, exit_code_file)) {
fprintf(LOGFILE, "Can't move exit code file from %s to %s - %s\n",
tmp_ecode_file, exit_code_file, strerror(errno));
+ fflush(LOGFILE);
unlink(tmp_ecode_file);
rc = -1;
goto cleanup;
@@ -396,6 +409,7 @@ static int wait_and_get_exit_code(pid_t pid) {
if (waitpid_result < 0) {
fprintf(LOGFILE, "error waiting for process %" PRId64 " - %s\n", (int64_t)pid, strerror(errno));
+ fflush(LOGFILE);
return -1;
}
@@ -405,6 +419,7 @@ static int wait_and_get_exit_code(pid_t pid) {
exit_code = 0x80 + WTERMSIG(child_status);
} else {
fprintf(LOGFILE, "Unable to determine exit status for pid %" PRId64 "\n", (int64_t)pid);
+ fflush(LOGFILE);
}
return exit_code;
@@ -440,6 +455,7 @@ int change_user(uid_t user, gid_t group) {
fprintf(LOGFILE, "unable to reacquire root - %s\n", strerror(errno));
fprintf(LOGFILE, "Real: %d:%d; Effective: %d:%d\n",
getuid(), getgid(), geteuid(), getegid());
+ fflush(LOGFILE);
return SETUID_OPER_FAILED;
}
if (setgid(group) != 0) {
@@ -447,12 +463,14 @@ int change_user(uid_t user, gid_t group) {
strerror(errno));
fprintf(LOGFILE, "Real: %d:%d; Effective: %d:%d\n",
getuid(), getgid(), geteuid(), getegid());
+ fflush(LOGFILE);
return SETUID_OPER_FAILED;
}
if (setuid(user) != 0) {
fprintf(LOGFILE, "unable to set user to %d - %s\n", user, strerror(errno));
fprintf(LOGFILE, "Real: %d:%d; Effective: %d:%d\n",
getuid(), getgid(), geteuid(), getegid());
+ fflush(LOGFILE);
return SETUID_OPER_FAILED;
}
@@ -533,6 +551,7 @@ char *concatenate(char *concat_pattern, char *return_path_name,
if (arg == NULL) {
fprintf(LOGFILE, "One of the arguments passed for %s is null.\n",
return_path_name);
+ fflush(LOGFILE);
return NULL;
}
strlen_args += strlen(arg);
@@ -545,6 +564,7 @@ char *concatenate(char *concat_pattern, char *return_path_name,
return_path = (char *) malloc(str_len);
if (return_path == NULL) {
fprintf(LOGFILE, "Unable to allocate memory for %s.\n", return_path_name);
+ fflush(LOGFILE);
return NULL;
}
va_start(ap, numArgs);
@@ -594,10 +614,12 @@ int check_nm_local_dir(uid_t caller_uid, const char *nm_root) {
int err = stat(nm_root, &info);
if (err < 0) {
fprintf(LOGFILE, "Error checking file stats for %s %d %s.\n", nm_root, err, strerror(errno));
+ fflush(LOGFILE);
return 1;
}
if (caller_uid != info.st_uid) {
fprintf(LOGFILE, "Permission mismatch for %s for caller uid: %d, owner uid: %d.\n", nm_root, caller_uid, info.st_uid);
+ fflush(LOGFILE);
return 1;
}
return 0;
@@ -664,6 +686,7 @@ int mkdirs(const char* path, mode_t perm) {
npath = strdup(path);
if (npath == NULL) {
fprintf(LOGFILE, "Not enough memory to copy path string");
+ fflush(LOGFILE);
return -1;
}
/* Skip leading slashes. */
@@ -706,6 +729,7 @@ int create_validate_dir(const char* npath, mode_t perm, const char* path,
if (errno != EEXIST || stat(npath, &sb) != 0) {
fprintf(LOGFILE, "Can't create directory %s - %s\n", npath,
strerror(errno));
+ fflush(LOGFILE);
return -1;
}
// The directory npath should exist.
@@ -726,12 +750,14 @@ int create_validate_dir(const char* npath, mode_t perm, const char* path,
int check_dir(const char* npath, mode_t st_mode, mode_t desired, int finalComponent) {
if (!S_ISDIR(st_mode)) {
fprintf(LOGFILE, "Path %s is file not dir\n", npath);
+ fflush(LOGFILE);
return -1;
} else if (finalComponent == 1) {
int filePermInt = st_mode & (S_IRWXU | S_IRWXG | S_IRWXO);
int desiredInt = desired & (S_IRWXU | S_IRWXG | S_IRWXO);
if (filePermInt != desiredInt) {
fprintf(LOGFILE, "Path %s has permission %o but needs permission %o.\n", npath, filePermInt, desiredInt);
+ fflush(LOGFILE);
return -1;
}
}
@@ -751,6 +777,7 @@ static int create_container_directories(const char* user, const char *app_id,
user_detail == NULL || user_detail->pw_name == NULL) {
fprintf(LOGFILE,
"Either app_id, container_id or the user passed is null.\n");
+ fflush(LOGFILE);
return ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS;
}
@@ -783,6 +810,7 @@ static int create_container_directories(const char* user, const char *app_id,
char *combined_name = malloc(strlen(app_id) + strlen(container_id) + 2);
if (combined_name == NULL) {
fprintf(LOGFILE, "Malloc of combined name failed\n");
+ fflush(LOGFILE);
result = OUT_OF_MEMORY;
} else {
sprintf(combined_name, "%s/%s", app_id, container_id);
@@ -795,6 +823,7 @@ static int create_container_directories(const char* user, const char *app_id,
}
if (strstr(container_log_dir, "..") != 0) {
fprintf(LOGFILE, "Unsupported container log directory path detected.\n");
+ fflush(LOGFILE);
container_log_dir = NULL;
}
if (container_log_dir == NULL) {
@@ -842,6 +871,7 @@ static struct passwd* get_user_info(const char* user) {
struct passwd* buffer = malloc(sizeof(struct passwd) + string_size);
if (NULL == buffer) {
fprintf(LOGFILE, "Failed malloc in get_user_info");
+ fflush(LOGFILE);
return NULL;
}
if (getpwnam_r(user, buffer, ((char*)buffer) + sizeof(struct passwd),
@@ -849,6 +879,7 @@ static struct passwd* get_user_info(const char* user) {
free(buffer);
fprintf(LOGFILE, "Can't get user information %s - %s\n", user,
strerror(errno));
+ fflush(LOGFILE);
return NULL;
}
return result;
@@ -921,6 +952,7 @@ struct passwd* check_user(const char *user) {
free_values(banned_users);
}
fprintf(LOGFILE, "Requested user %s is banned\n", user);
+ fflush(LOGFILE);
return NULL;
}
}
@@ -951,6 +983,7 @@ int set_user(const char *user) {
if (initgroups(user, user_detail->pw_gid) != 0) {
fprintf(LOGFILE, "Error setting supplementary groups for user %s: %s\n",
user, strerror(errno));
+ fflush(LOGFILE);
return -1;
}
@@ -972,6 +1005,7 @@ static int change_owner(const char* path, uid_t user, gid_t group) {
if (chown(path, user, group) != 0) {
fprintf(LOGFILE, "Can't chown %s to %d:%d - %s\n", path, user, group,
strerror(errno));
+ fflush(LOGFILE);
return -1;
}
return 0;
@@ -988,6 +1022,7 @@ static int change_owner(const char* path, uid_t user, gid_t group) {
if (chown(path, user, group) != 0) {
fprintf(LOGFILE, "Can't chown %s to %d:%d - %s\n", path, user, group,
strerror(errno));
+ fflush(LOGFILE);
return -1;
}
return change_effective_user(old_user, old_group);
@@ -1019,20 +1054,24 @@ int create_directory_for_user(const char* path) {
if (change_owner(path, user, nm_gid) != 0) {
fprintf(LOGFILE, "Failed to chown %s to %d:%d: %s\n", path, user, nm_gid,
strerror(errno));
+ fflush(LOGFILE);
ret = -1;
} else if (chmod(path, permissions) != 0) {
fprintf(LOGFILE, "Can't chmod %s to add the sticky bit - %s\n",
path, strerror(errno));
+ fflush(LOGFILE);
ret = -1;
}
} else {
fprintf(LOGFILE, "Failed to create directory %s - %s\n", path,
strerror(errno));
+ fflush(LOGFILE);
ret = -1;
}
}
if (change_effective_user(user, group) != 0) {
fprintf(LOGFILE, "Failed to change user to %i - %i\n", user, group);
+ fflush(LOGFILE);
ret = -1;
}
@@ -1053,6 +1092,7 @@ static int open_file_as_nm(const char* filename) {
if (result == -1) {
fprintf(LOGFILE, "Can't open file %s as node manager - %s\n", filename,
strerror(errno));
+ fflush(LOGFILE);
}
if (change_effective_user(user, group)) {
result = -1;
@@ -1094,6 +1134,7 @@ static int copy_file(int input, const char* in_filename,
if (write_result <= 0) {
fprintf(LOGFILE, "Error writing to %s - %s\n", out_filename,
strerror(errno));
+ fflush(LOGFILE);
close(out_fd);
free(buffer);
return -1;
@@ -1107,12 +1148,14 @@ static int copy_file(int input, const char* in_filename,
if (len < 0) {
fprintf(LOGFILE, "Failed to read file %s - %s\n", in_filename,
strerror(errno));
+ fflush(LOGFILE);
close(out_fd);
return -1;
}
if (close(out_fd) != 0) {
fprintf(LOGFILE, "Failed to close file %s - %s\n", out_filename,
strerror(errno));
+ fflush(LOGFILE);
return -1;
}
close(input);
@@ -1131,6 +1174,7 @@ int initialize_user(const char *user, char* const* local_dirs) {
user_dir = get_user_directory(*local_dir_ptr, user);
if (user_dir == NULL) {
fprintf(LOGFILE, "Couldn't get userdir directory for %s.\n", user);
+ fflush(LOGFILE);
failed = 1;
break;
}
@@ -1166,6 +1210,7 @@ int create_log_dirs(const char *app_id, char * const * log_dirs) {
if (any_one_app_log_dir == NULL) {
fprintf(LOGFILE, "Did not create any app-log directories\n");
+ fflush(LOGFILE);
return -1;
}
free(any_one_app_log_dir);
@@ -1189,17 +1234,20 @@ char *init_log_path(const char *container_log_dir, const char *logfile) {
if (change_owner(tmp_buffer, user_detail->pw_uid, nm_gid) != 0) {
fprintf(ERRORFILE, "Failed to chown %s to %d:%d: %s\n", tmp_buffer, user_detail->pw_uid, nm_gid,
strerror(errno));
+ fflush(ERRORFILE);
free(tmp_buffer);
tmp_buffer = NULL;
} else if (chmod(tmp_buffer, permissions) != 0) {
fprintf(ERRORFILE, "Can't chmod %s - %s\n",
tmp_buffer, strerror(errno));
+ fflush(ERRORFILE);
free(tmp_buffer);
tmp_buffer = NULL;
}
} else {
fprintf(ERRORFILE, "Failed to create file %s - %s\n", tmp_buffer,
strerror(errno));
+ fflush(ERRORFILE);
free(tmp_buffer);
tmp_buffer = NULL;
}
@@ -1217,7 +1265,8 @@ int create_container_log_dirs(const char *container_id, const char *app_id,
if (container_log_dir == NULL) {
fprintf(LOGFILE,
"Failed to get container log directory name! Log root directory: %s, App id: %s, Container id: %s\n",
- *log_root, app_id, container_id);
+ *log_root, app_id, container_id);
+ fflush(LOGFILE);
continue;
}
@@ -1225,6 +1274,7 @@ int create_container_log_dirs(const char *container_id, const char *app_id,
if (result != 0 && container_log_dir != NULL) {
fprintf(LOGFILE, "Unsupported container log directory path (%s) detected.\n",
container_log_dir);
+ fflush(LOGFILE);
free(container_log_dir);
container_log_dir = NULL;
continue;
@@ -1233,6 +1283,7 @@ int create_container_log_dirs(const char *container_id, const char *app_id,
if (create_directory_for_user(container_log_dir) != 0) {
fprintf(LOGFILE, "Failed to create container log directory (%s)!\n",
container_log_dir);
+ fflush(LOGFILE);
free(container_log_dir);
return -1;
}
@@ -1246,6 +1297,7 @@ int create_container_log_dirs(const char *container_id, const char *app_id,
if (!created_any_dir) {
fprintf(LOGFILE, "Did not create any container log directory.\n");
+ fflush(LOGFILE);
return -1;
}
return 0;
@@ -1261,6 +1313,7 @@ int initialize_app(const char *user, const char *app_id,
char* const* args) {
if (app_id == NULL || user == NULL || user_detail == NULL || user_detail->pw_name == NULL) {
fprintf(LOGFILE, "Either app_id is null or the user passed is null.\n");
+ fflush(LOGFILE);
return INVALID_ARGUMENT_NUMBER;
}
@@ -1313,6 +1366,7 @@ int initialize_app(const char *user, const char *app_id,
if (primary_app_dir == NULL) {
fprintf(LOGFILE, "Did not create any app directories\n");
+ fflush(LOGFILE);
return -1;
}
@@ -1343,11 +1397,13 @@ int initialize_app(const char *user, const char *app_id,
}
if (chdir(primary_app_dir) != 0) {
fprintf(LOGFILE, "Failed to chdir to app dir - %s\n", strerror(errno));
+ fflush(LOGFILE);
return -1;
}
execvp(args[0], args);
fprintf(ERRORFILE, "Failure to exec app initialization process - %s\n",
strerror(errno));
+ fflush(ERRORFILE);
return -1;
}
@@ -1389,13 +1445,12 @@ int run_docker(const char *command_file) {
docker_binary, strerror(errno));
fflush(LOGFILE);
fflush(ERRORFILE);
- free(docker_binary);
- free_values(args);
exit_code = DOCKER_RUN_FAILED;
} else {
- free_values(args);
exit_code = 0;
}
+ free(docker_binary);
+ free_values(args);
return exit_code;
}
@@ -1415,13 +1470,14 @@ int exec_container(const char *command_file) {
int ret = read_config(command_file, &command_config);
if (ret != 0) {
free_configuration(&command_config);
+ free(docker_binary);
return INVALID_COMMAND_FILE;
}
char *value = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (value != NULL && strcasecmp(value, "exec") == 0) {
args = construct_docker_command(command_file);
- binary = docker_binary;
+ binary = strdup(docker_binary);
docker = 1;
} else {
value = get_configuration_value("command", COMMAND_FILE_SECTION, &command_config);
@@ -1452,19 +1508,22 @@ int exec_container(const char *command_file) {
fdm = posix_openpt(O_RDWR);
if (fdm < 0) {
fprintf(stderr, "Error %d on posix_openpt()\n", errno);
- return DOCKER_EXEC_FAILED;
+ exit_code = DOCKER_EXEC_FAILED;
+ goto cleanup;
}
rc = grantpt(fdm);
if (rc != 0) {
fprintf(stderr, "Error %d on grantpt()\n", errno);
- return DOCKER_EXEC_FAILED;
+ exit_code = DOCKER_EXEC_FAILED;
+ goto cleanup;
}
rc = unlockpt(fdm);
if (rc != 0) {
fprintf(stderr, "Error %d on unlockpt()\n", errno);
- return DOCKER_EXEC_FAILED;
+ exit_code = DOCKER_EXEC_FAILED;
+ goto cleanup;
}
// Open the slave PTY
@@ -1602,6 +1661,7 @@ int exec_container(const char *command_file) {
if (ret != 0) {
fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s\n",
binary, strerror(errno));
+ fflush(ERRORFILE);
exit_code = DOCKER_EXEC_FAILED;
} else {
exit_code = 0;
@@ -1614,6 +1674,7 @@ cleanup:
free(user);
free(workdir);
free_values(args);
+ free_values(env);
free_configuration(&command_config);
return exit_code;
}
@@ -1834,12 +1895,14 @@ int create_user_filecache_dirs(const char * user, char* const* local_dirs) {
char* filecache_dir = get_user_filecache_directory(*ldir_p, user);
if (filecache_dir == NULL) {
fprintf(LOGFILE, "Couldn't get user filecache directory for %s.\n", user);
+ fflush(LOGFILE);
rc = INITIALIZE_USER_FAILED;
break;
}
if (0 != mkdir(filecache_dir, permissions) && EEXIST != errno) {
fprintf(LOGFILE, "Failed to create directory %s - %s\n", filecache_dir,
strerror(errno));
+ fflush(LOGFILE);
free(filecache_dir);
rc = INITIALIZE_USER_FAILED;
break;
@@ -2082,6 +2145,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
}
fprintf(LOGFILE, "Waiting for docker container to finish.\n");
+ fflush(LOGFILE);
// wait for pid to finish
#ifdef __linux
@@ -2124,6 +2188,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
count++;
}
fprintf(LOGFILE, "Exit code from docker inspect: %d\n", exit_code);
+ fflush(LOGFILE);
cleanup:
@@ -2133,6 +2198,7 @@ cleanup:
fflush(ERRORFILE);
}
fprintf(LOGFILE, "Wrote the exit code %d to %s\n", exit_code, exit_code_file);
+ fflush(LOGFILE);
// Drop root privileges
if (change_effective_user(prev_uid, user_gid) != 0) {
@@ -2251,6 +2317,7 @@ int launch_container_as_user(const char *user, const char *app_id,
}
fprintf(LOGFILE, "Launching container...\n");
+ fflush(LOGFILE);
#if HAVE_FCLOSEALL
fcloseall();
@@ -2267,6 +2334,7 @@ int launch_container_as_user(const char *user, const char *app_id,
if (execlp(script_file_dest, script_file_dest, NULL) != 0) {
fprintf(LOGFILE, "Couldn't execute the container launch file %s - %s",
script_file_dest, strerror(errno));
+ fflush(LOGFILE);
exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT;
goto cleanup;
}
@@ -2294,6 +2362,7 @@ int signal_container_as_user(const char *user, int pid, int sig) {
if (kill(-pid,0) < 0) {
fprintf(LOGFILE, "Error signalling not exist process group %d "
"with signal %d\n", pid, sig);
+ fflush(LOGFILE);
return INVALID_CONTAINER_PID;
}
@@ -2309,6 +2378,7 @@ int signal_container_as_user(const char *user, int pid, int sig) {
}
}
fprintf(LOGFILE, "Killing process group %d with %d\n", pid, sig);
+ fflush(LOGFILE);
return 0;
}
@@ -2322,6 +2392,7 @@ static int rmdir_as_nm(const char* path) {
if (ret == 0) {
if (rmdir(path) != 0 && errno != ENOENT) {
fprintf(LOGFILE, "rmdir of %s failed - %s\n", path, strerror(errno));
+ fflush(LOGFILE);
ret = -1;
}
}
@@ -2418,6 +2489,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
ret = -ret;
fprintf(LOGFILE, "is_dir_helper(%s) failed: %s\n",
fullpath, strerror(ret));
+ fflush(LOGFILE);
goto done;
} else if (ret == 0) {
// is_dir_helper determined that the path is not a directory.
@@ -2425,6 +2497,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
if (ret) {
fprintf(LOGFILE, "failed to unlink %s: %s\n",
fullpath, strerror(ret));
+ fflush(LOGFILE);
}
goto done;
}
@@ -2441,6 +2514,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
fprintf(LOGFILE, "chmod(%s) failed: %s\n", fullpath, strerror(ret));
+ fflush(LOGFILE);
goto done;
}
fd = open_helper(dirfd, name);
@@ -2452,6 +2526,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
fprintf(LOGFILE, "error opening %s: %s\n", fullpath, strerror(ret));
+ fflush(LOGFILE);
goto done;
}
if (fstat(fd, &stat) < 0) {
@@ -2461,12 +2536,14 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
fprintf(LOGFILE, "failed to stat %s: %s\n", fullpath, strerror(ret));
+ fflush(LOGFILE);
goto done;
}
if (!(S_ISDIR(stat.st_mode))) {
ret = unlink_helper(dirfd, name, 0);
if (ret) {
fprintf(LOGFILE, "failed to unlink %s: %s\n", fullpath, strerror(ret));
+ fflush(LOGFILE);
goto done;
}
} else {
@@ -2480,6 +2557,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
fprintf(LOGFILE, "chmod(%s) failed: %s\n", fullpath, strerror(ret));
+ fflush(LOGFILE);
goto done;
}
}
@@ -2491,6 +2569,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
fprintf(LOGFILE, "fopendir(%s) failed: %s\n", fullpath, strerror(ret));
+ fflush(LOGFILE);
goto done;
}
while (1) {
@@ -2503,6 +2582,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
ret = errno;
if (ret && ret != ENOENT) {
fprintf(LOGFILE, "readdir(%s) failed: %s\n", fullpath, strerror(ret));
+ fflush(LOGFILE);
goto done;
}
break;
@@ -2516,6 +2596,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
if (asprintf(&new_fullpath, "%s/%s", fullpath, de->d_name) < 0) {
fprintf(LOGFILE, "Failed to allocate string for %s/%s.\n",
fullpath, de->d_name);
+ fflush(LOGFILE);
ret = ENOMEM;
goto done;
}
@@ -2529,6 +2610,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
ret = unlink_helper(dirfd, name, AT_REMOVEDIR);
if (ret) {
fprintf(LOGFILE, "failed to rmdir %s: %s\n", name, strerror(ret));
+ fflush(LOGFILE);
goto done;
}
}
@@ -2561,12 +2643,14 @@ static int delete_path(const char *full_path,
/* Return an error if the path is null. */
if (full_path == NULL) {
fprintf(LOGFILE, "Path is null\n");
+ fflush(LOGFILE);
return PATH_TO_DELETE_IS_NULL;
}
ret = recursive_unlink_children(full_path);
if (ret != 0) {
fprintf(LOGFILE, "Error while deleting %s: %d (%s)\n",
full_path, ret, strerror(ret));
+ fflush(LOGFILE);
return -1;
}
@@ -2584,6 +2668,7 @@ static int delete_path(const char *full_path,
if (ret != ENOENT) {
fprintf(LOGFILE, "Couldn't delete directory %s - %s\n",
full_path, strerror(ret));
+ fflush(LOGFILE);
return -1;
}
}
@@ -2619,12 +2704,14 @@ int delete_as_user(const char *user,
continue;
} else {
fprintf(LOGFILE, "Could not stat %s - %s\n", *ptr, strerror(errno));
+ fflush(LOGFILE);
return -1;
}
}
if (!S_ISDIR(sb.st_mode)) {
if (!subDirEmptyStr) {
fprintf(LOGFILE, "baseDir \"%s\" is a file and cannot contain subdir \"%s\".\n", *ptr, subdir);
+ fflush(LOGFILE);
return -1;
}
full_path = strdup(*ptr);
@@ -2659,6 +2746,7 @@ int list_as_user(const char *target_dir) {
// If directory doesn't exist or can't be accessed, error out
fprintf(LOGFILE, "Could not stat %s - %s\n", target_dir,
strerror(errno));
+ fflush(LOGFILE);
ret = -1;
} else if (!S_ISDIR(sb.st_mode)) {
// If it's not a directory, list it as the only file
@@ -2686,11 +2774,13 @@ int list_as_user(const char *target_dir) {
if (errno != 0) {
fprintf(LOGFILE, "Could not read directory %s - %s\n", target_dir,
strerror(errno));
+ fflush(LOGFILE);
ret = -1;
}
} else {
fprintf(LOGFILE, "Could not open directory %s - %s\n", target_dir,
strerror(errno));
+ fflush(LOGFILE);
ret = -1;
}
}
@@ -2719,6 +2809,7 @@ void chown_dir_contents(const char *dir_path, uid_t uid, gid_t gid) {
change_owner(path_tmp, uid, gid);
} else {
fprintf(LOGFILE, "Ignored %s/%s due to length", dir_path, ep->d_name);
+ fflush(LOGFILE);
}
}
}
@@ -2735,6 +2826,7 @@ int is_empty(char *target_dir) {
if (!dir) {
fprintf(LOGFILE, "Could not open directory %s - %s\n", target_dir,
strerror(errno));
+ fflush(LOGFILE);
return 0;
}
while ((entry = readdir(dir)) != NULL) {
@@ -2745,6 +2837,7 @@ int is_empty(char *target_dir) {
continue;
}
fprintf(LOGFILE, "Directory is not empty %s\n", target_dir);
+ fflush(LOGFILE);
return 0;
}
return 1;
@@ -2759,6 +2852,7 @@ int is_empty(char *target_dir) {
int mount_cgroup(const char *pair, const char *hierarchy) {
#ifndef __linux
fprintf(LOGFILE, "Failed to mount cgroup controller, not supported\n");
+ fflush(LOGFILE);
return -1;
#else
size_t len = strlen(pair);
@@ -2769,11 +2863,13 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
if (controller == NULL || mount_path == NULL) {
fprintf(LOGFILE, "Failed to mount cgroup controller; not enough memory\n");
+ fflush(LOGFILE);
result = OUT_OF_MEMORY;
goto cleanup;
}
if (hierarchy == NULL || strstr(hierarchy, "..") != NULL) {
fprintf(LOGFILE, "Unsupported cgroup hierarhy path detected.\n");
+ fflush(LOGFILE);
result = INVALID_COMMAND_PROVIDED;
goto cleanup;
}
@@ -2781,11 +2877,13 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
get_kv_value(pair, mount_path, len) < 0) {
fprintf(LOGFILE, "Failed to mount cgroup controller; invalid option: %s\n",
pair);
+ fflush(LOGFILE);
result = -1;
} else {
if (strstr(mount_path, "..") != NULL) {
fprintf(LOGFILE, "Unsupported cgroup mount path detected. %s\n",
mount_path);
+ fflush(LOGFILE);
result = INVALID_COMMAND_PROVIDED;
goto cleanup;
}
@@ -2805,6 +2903,7 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
if (stat(hier_path, &sb) == 0 &&
(sb.st_uid != nm_uid || sb.st_gid != nm_gid)) {
fprintf(LOGFILE, "cgroup hierarchy %s already owned by another user %d\n", hier_path, sb.st_uid);
+ fflush(LOGFILE);
result = INVALID_COMMAND_PROVIDED;
goto cleanup;
}
@@ -2815,6 +2914,7 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
} else {
fprintf(LOGFILE, "Failed to mount cgroup controller %s at %s - %s\n",
controller, mount_path, strerror(errno));
+ fflush(LOGFILE);
// if controller is already mounted, don't stop trying to mount others
if (errno != EBUSY) {
result = -1;
@@ -2843,6 +2943,7 @@ static int run_traffic_control(const char *opts[], char *command_file) {
//too many args to tc
if (i == max_tc_args - 1) {
fprintf(LOGFILE, "too many args to tc");
+ fflush(LOGFILE);
return TRAFFIC_CONTROL_EXECUTION_FAILED;
}
args[i++] = command_file;
@@ -2853,6 +2954,7 @@ static int run_traffic_control(const char *opts[], char *command_file) {
int exit_code = wait_and_get_exit_code(child_pid);
if (exit_code != 0) {
fprintf(LOGFILE, "failed to execute tc command!\n");
+ fflush(LOGFILE);
return TRAFFIC_CONTROL_EXECUTION_FAILED;
}
return 0;
@@ -2860,6 +2962,7 @@ static int run_traffic_control(const char *opts[], char *command_file) {
execv(TC_BIN, (char**)args);
//if we reach here, exec failed
fprintf(LOGFILE, "failed to execute tc command! error: %s\n", strerror(errno));
+ fflush(LOGFILE);
return TRAFFIC_CONTROL_EXECUTION_FAILED;
}
}
@@ -2985,6 +3088,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
const char* container_id) {
#ifndef __linux
fprintf(LOGFILE, "Failed to clean cgroups, not supported\n");
+ fflush(LOGFILE);
return -1;
#else
const char * cgroup_mount_type = "cgroup";
@@ -2998,17 +3102,20 @@ int clean_docker_cgroups_internal(const char *mount_table,
if (!mount_table || mount_table[0] == 0) {
fprintf(ERRORFILE, "clean_docker_cgroups: Invalid mount table\n");
+ fflush(ERRORFILE);
rc = -1;
goto cleanup;
}
if (!yarn_hierarchy || yarn_hierarchy[0] == 0) {
fprintf(ERRORFILE, "clean_docker_cgroups: Invalid yarn_hierarchy\n");
+ fflush(ERRORFILE);
rc = -1;
goto cleanup;
}
if (!validate_container_id(container_id)) {
fprintf(ERRORFILE, "clean_docker_cgroups: Invalid container_id: %s\n",
(container_id == NULL) ? "null" : container_id);
+ fflush(ERRORFILE);
rc = -1;
goto cleanup;
}
@@ -3016,6 +3123,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
if (fp == NULL) {
fprintf(ERRORFILE, "clean_docker_cgroups: failed to open %s, error %d: %s\n",
mount_table, errno, strerror(errno));
+ fflush(ERRORFILE);
rc = -1;
goto cleanup;
}
@@ -3029,6 +3137,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
ret = sscanf(lineptr, " %ms %ms %*s %*s %*s %*s", &mnt_type, &mnt_dir);
if (ret != 2) {
fprintf(ERRORFILE, "clean_docker_cgroups: Failed to parse line: %s\n", lineptr);
+ fflush(ERRORFILE);
rc = -1;
break;
}
@@ -3037,6 +3146,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
}
if ((mnt_dir == NULL) || (mnt_dir[0] == 0)) {
fprintf(ERRORFILE, "clean_docker_cgroups: skipping mount entry with invalid mnt_dir\n");
+ fflush(ERRORFILE);
continue;
}
@@ -3044,6 +3154,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
full_path = make_string("%s/%s/%s", mnt_dir, yarn_hierarchy, container_id);
if (full_path == NULL) {
fprintf(ERRORFILE, "clean_docker_cgroups: Failed to allocate cgroup path.\n");
+ fflush(ERRORFILE);
rc = -1;
break;
}
@@ -3052,13 +3163,15 @@ int clean_docker_cgroups_internal(const char *mount_table,
if (!verify_path_safety(full_path)) {
fprintf(ERRORFILE,
"clean_docker_cgroups: skipping invalid path: %s\n", full_path);
- continue;
+ fflush(ERRORFILE);
+ continue;
}
ret = rmdir(full_path);
if ((ret == -1) && (errno != ENOENT)) {
fprintf(ERRORFILE, "clean_docker_cgroups: Failed to rmdir cgroup, %s (error=%s)\n",
full_path, strerror(errno));
+ fflush(ERRORFILE);
rc = -1;
continue;
}
@@ -3066,6 +3179,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
if (ferror(fp)) {
fprintf(ERRORFILE, "clean_docker_cgroups: Error reading %s, error=%d (%s) \n",
mount_table, errno, strerror(errno));
+ fflush(ERRORFILE);
rc = -1;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e544dc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 8cdb8a3..af540fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -122,6 +122,7 @@ static void display_usage(FILE *stream) {
"[DISABLED] sync yarn sysfs: %2d app-id nm-local-dirs\n",
SYNC_YARN_SYSFS);
}
+ fflush(stream);
}
/* Sets up log files for normal/error logging */
@@ -230,6 +231,7 @@ static void assert_valid_setup(char *argv0) {
static void display_feature_disabled_message(const char* name) {
fprintf(ERRORFILE, "Feature disabled: %s\n", name);
+ fflush(ERRORFILE);
}
/* Use to store parsed input parmeters for various operations */
@@ -458,6 +460,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
cmd_input.container_id = argv[optind++];
if (!validate_container_id(cmd_input.container_id)) {
fprintf(ERRORFILE, "Invalid container id %s\n", cmd_input.container_id);
+ fflush(ERRORFILE);
return INVALID_CONTAINER_ID;
}
cmd_input.cred_file = argv[optind++];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e544dc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 279d1c7..86a5d78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -550,7 +550,7 @@ void test_yarn_sysfs() {
exit(1);
}
free(user_dir);
- char *app_dir = make_string("%s/usercache/%s/appcache/%s/%s", *local_dir_ptr, username, app_id);
+ char *app_dir = make_string("%s/usercache/%s/appcache/%s", *local_dir_ptr, username, app_id);
if (mkdirs(app_dir, 0750) != 0) {
printf("Can not make app directories: %s\n", app_dir);
exit(1);
@@ -1154,6 +1154,8 @@ static void test_delete_race_internal() {
printf("FAIL: return code from delete_as_user is %d\n", ret);
exit(1);
}
+ free(app_dir);
+ free(container_dir);
exit(0);
} else {
// delete application directory
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/8] hadoop git commit: MAPREDUCE-7166. map-only job should ignore
node lost event when task is already succeeded. Contributed by Lei Li.
Posted by su...@apache.org.
MAPREDUCE-7166. map-only job should ignore node lost event when task is already succeeded. Contributed by Lei Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/499c70ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/499c70ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/499c70ed
Branch: refs/heads/HDFS-12943
Commit: 499c70eda5f6315cf7e13a4b3bbefc7e187bc457
Parents: d963575
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Dec 20 10:09:50 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Dec 20 10:09:50 2018 +0900
----------------------------------------------------------------------
.../v2/app/job/impl/TaskAttemptImpl.java | 15 ++++
.../v2/app/job/impl/TestTaskAttempt.java | 84 +++++++++++++++++++-
2 files changed, 95 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/499c70ed/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 63e7456..d912b60 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -2196,6 +2196,14 @@ public abstract class TaskAttemptImpl implements
taskAttempt.getID().toString());
return TaskAttemptStateInternal.SUCCEEDED;
}
+ if (taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP
+ && taskAttempt.conf.getNumReduceTasks() == 0) {
+ // same reason as above for map only job after map task has succeeded.
+ // ignore this for map only tasks
+ LOG.info("Ignoring killed event for successful map only task attempt" +
+ taskAttempt.getID().toString());
+ return TaskAttemptStateInternal.SUCCEEDED;
+ }
if(event instanceof TaskAttemptKillEvent) {
TaskAttemptKillEvent msgEvent = (TaskAttemptKillEvent) event;
//add to diagnostic
@@ -2246,6 +2254,13 @@ public abstract class TaskAttemptImpl implements
LOG.info("Ignoring killed event for successful reduce task attempt" +
taskAttempt.getID().toString());
return TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP;
+ } else if (taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP
+ && taskAttempt.conf.getNumReduceTasks() == 0) {
+ // same reason as above for map only job after map task has succeeded.
+ // ignore this for map only tasks
+ LOG.info("Ignoring killed event for successful map only task attempt" +
+ taskAttempt.getID().toString());
+ return TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP;
} else {
// Store reschedule flag so that after clean up is completed, new
// attempt is scheduled/rescheduled based on it.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/499c70ed/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index d62a6cc..11f16a8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -1323,6 +1323,42 @@ public class TestTaskAttempt{
}
@Test
+ public void testKillMapOnlyTaskWhileSuccessFinishing() throws Exception {
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptImpl taImpl = createMapOnlyTaskAttemptImpl(eventHandler);
+
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_DONE));
+
+ assertEquals("Task attempt is not in SUCCEEDED state",
+ TaskAttemptState.SUCCEEDED, taImpl.getState());
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_FINISHING_CONTAINER",
+ TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
+ taImpl.getInternalState());
+
+ // If the map only task is killed when it is in SUCCESS_FINISHING_CONTAINER
+ // state, the state will move to SUCCESS_CONTAINER_CLEANUP
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_KILL));
+ assertEquals("Task attempt is not in SUCCEEDED state",
+ TaskAttemptState.SUCCEEDED, taImpl.getState());
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_CONTAINER_CLEANUP",
+ TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
+ taImpl.getInternalState());
+
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_CONTAINER_CLEANED));
+ assertEquals("Task attempt is not in SUCCEEDED state",
+ TaskAttemptState.SUCCEEDED, taImpl.getState());
+ assertEquals("Task attempt's internal state is not SUCCEEDED state",
+ TaskAttemptStateInternal.SUCCEEDED, taImpl.getInternalState());
+
+ assertFalse("InternalError occurred", eventHandler.internalError);
+ }
+
+ @Test
public void testKillMapTaskAfterSuccess() throws Exception {
MockEventHandler eventHandler = new MockEventHandler();
TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler);
@@ -1340,7 +1376,7 @@ public class TestTaskAttempt{
TaskAttemptEventType.TA_CONTAINER_CLEANED));
// Send a map task attempt kill event indicating next map attempt has to be
// reschedule
- taImpl.handle(new TaskAttemptKillEvent(taImpl.getID(),"", true));
+ taImpl.handle(new TaskAttemptKillEvent(taImpl.getID(), "", true));
assertEquals("Task attempt is not in KILLED state", taImpl.getState(),
TaskAttemptState.KILLED);
assertEquals("Task attempt's internal state is not KILLED",
@@ -1354,6 +1390,34 @@ public class TestTaskAttempt{
}
@Test
+ public void testKillMapOnlyTaskAfterSuccess() throws Exception {
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptImpl taImpl = createMapOnlyTaskAttemptImpl(eventHandler);
+
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_DONE));
+
+ assertEquals("Task attempt is not in SUCCEEDED state",
+ TaskAttemptState.SUCCEEDED, taImpl.getState());
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_FINISHING_CONTAINER",
+ TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
+ taImpl.getInternalState());
+
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_CONTAINER_CLEANED));
+ // Succeeded
+ taImpl.handle(new TaskAttemptKillEvent(taImpl.getID(),"", true));
+ assertEquals("Task attempt is not in SUCCEEDED state",
+ TaskAttemptState.SUCCEEDED, taImpl.getState());
+ assertEquals("Task attempt's internal state is not SUCCEEDED",
+ TaskAttemptStateInternal.SUCCEEDED, taImpl.getInternalState());
+ assertFalse("InternalError occurred", eventHandler.internalError);
+ TaskEvent event = eventHandler.lastTaskEvent;
+ assertEquals(TaskEventType.T_ATTEMPT_SUCCEEDED, event.getType());
+ }
+
+ @Test
public void testKillMapTaskWhileFailFinishing() throws Exception {
MockEventHandler eventHandler = new MockEventHandler();
TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler);
@@ -1765,8 +1829,8 @@ public class TestTaskAttempt{
thenReturn(taskAttemptFinishingMonitor);
}
- private TaskAttemptImpl createTaskAttemptImpl(
- MockEventHandler eventHandler) {
+ private TaskAttemptImpl createCommonTaskAttemptImpl(
+ MockEventHandler eventHandler, JobConf jobConf) {
ApplicationId appId = ApplicationId.newInstance(1, 2);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 0);
@@ -1778,7 +1842,6 @@ public class TestTaskAttempt{
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
- JobConf jobConf = new JobConf();
jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
jobConf.setBoolean("fs.file.impl.disable.cache", true);
jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
@@ -1813,6 +1876,19 @@ public class TestTaskAttempt{
return taImpl;
}
+ private TaskAttemptImpl createTaskAttemptImpl(
+ MockEventHandler eventHandler) {
+ JobConf jobConf = new JobConf();
+ return createCommonTaskAttemptImpl(eventHandler, jobConf);
+ }
+
+ private TaskAttemptImpl createMapOnlyTaskAttemptImpl(
+ MockEventHandler eventHandler) {
+ JobConf jobConf = new JobConf();
+ jobConf.setInt(MRJobConfig.NUM_REDUCES, 0);
+ return createCommonTaskAttemptImpl(eventHandler, jobConf);
+ }
+
public static class MockEventHandler implements EventHandler {
public boolean internalError;
public TaskEvent lastTaskEvent;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/8] hadoop git commit: HDFS-13661. Ls command with e option fails
when the filesystem is not HDFS.
Posted by su...@apache.org.
HDFS-13661. Ls command with e option fails when the filesystem is not HDFS.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9635759
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9635759
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9635759
Branch: refs/heads/HDFS-12943
Commit: d9635759182b614a1dd5034c30978e7c4be8d0dd
Parents: 2e544dc
Author: Takanobu Asanuma <ta...@apache.org>
Authored: Thu Dec 20 10:03:33 2018 +0900
Committer: Takanobu Asanuma <ta...@apache.org>
Committed: Thu Dec 20 10:03:33 2018 +0900
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/shell/Ls.java | 40 ++++++++----
.../java/org/apache/hadoop/fs/shell/TestLs.java | 67 +++++++++++++++++++-
2 files changed, 93 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9635759/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index 56504e4..19383d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -25,6 +25,7 @@ import java.util.Comparator;
import java.util.Date;
import java.util.LinkedList;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
@@ -155,7 +156,7 @@ class Ls extends FsCommand {
* Should display only paths of files and directories.
* @return true display paths only, false display all fields
*/
- @InterfaceAudience.Private
+ @VisibleForTesting
boolean isPathOnly() {
return this.pathOnly;
}
@@ -164,7 +165,7 @@ class Ls extends FsCommand {
* Should the contents of the directory be shown or just the directory?
* @return true if directory contents, false if just directory
*/
- @InterfaceAudience.Private
+ @VisibleForTesting
boolean isDirRecurse() {
return this.dirRecurse;
}
@@ -173,12 +174,12 @@ class Ls extends FsCommand {
* Should file sizes be returned in human readable format rather than bytes?
* @return true is human readable, false if bytes
*/
- @InterfaceAudience.Private
+ @VisibleForTesting
boolean isHumanReadable() {
return this.humanReadable;
}
- @InterfaceAudience.Private
+ @VisibleForTesting
private boolean isHideNonPrintable() {
return hideNonPrintable;
}
@@ -187,7 +188,7 @@ class Ls extends FsCommand {
* Should directory contents be displayed in reverse order
* @return true reverse order, false default order
*/
- @InterfaceAudience.Private
+ @VisibleForTesting
boolean isOrderReverse() {
return this.orderReverse;
}
@@ -196,7 +197,7 @@ class Ls extends FsCommand {
* Should directory contents be displayed in mtime order.
* @return true mtime order, false default order
*/
- @InterfaceAudience.Private
+ @VisibleForTesting
boolean isOrderTime() {
return this.orderTime;
}
@@ -205,7 +206,7 @@ class Ls extends FsCommand {
* Should directory contents be displayed in size order.
* @return true size order, false default order
*/
- @InterfaceAudience.Private
+ @VisibleForTesting
boolean isOrderSize() {
return this.orderSize;
}
@@ -214,13 +215,28 @@ class Ls extends FsCommand {
* Should access time be used rather than modification time.
* @return true use access time, false use modification time
*/
- @InterfaceAudience.Private
+ @VisibleForTesting
boolean isUseAtime() {
return this.useAtime;
}
+ /**
+ * Should EC policies be displayed.
+ * @return true display EC policies, false doesn't display EC policies
+ */
+ @VisibleForTesting
+ boolean isDisplayECPolicy() {
+ return this.displayECPolicy;
+ }
+
@Override
protected void processPathArgument(PathData item) throws IOException {
+ if (isDisplayECPolicy() && item.fs.getContentSummary(item.path)
+ .getErasureCodingPolicy() == null) {
+ throw new UnsupportedOperationException("FileSystem "
+ + item.fs.getUri() + " does not support Erasure Coding");
+ }
+
// implicitly recurse once for cmdline directories
if (dirRecurse && item.stat.isDirectory()) {
recursePath(item);
@@ -320,19 +336,19 @@ class Ls extends FsCommand {
StringBuilder fmt = new StringBuilder();
fmt.append("%s%s"); // permission string
fmt.append("%" + maxRepl + "s ");
+ fmt.append((maxOwner > 0) ? "%-" + maxOwner + "s " : "%s");
+ fmt.append((maxGroup > 0) ? "%-" + maxGroup + "s " : "%s");
// Do not use '%-0s' as a formatting conversion, since it will throw a
// a MissingFormatWidthException if it is used in String.format().
// http://docs.oracle.com/javase/1.5.0/docs/api/java/util/Formatter.html#intFlags
if(displayECPolicy){
- int maxEC=0;
+ int maxEC = 0;
for (PathData item : items) {
ContentSummary contentSummary = item.fs.getContentSummary(item.path);
maxEC=maxLength(maxEC,contentSummary.getErasureCodingPolicy().length());
}
- fmt.append(" %"+maxEC+"s ");
+ fmt.append((maxEC > 0) ? "%-" + maxEC + "s " : "%s");
}
- fmt.append((maxOwner > 0) ? "%-" + maxOwner + "s " : "%s");
- fmt.append((maxGroup > 0) ? "%-" + maxGroup + "s " : "%s");
fmt.append("%" + maxLen + "s ");
fmt.append("%s %s"); // mod time & path
lineFormat = fmt.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9635759/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
index 0df1a6a..2e5c3e3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs.shell;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
import static org.junit.Assert.*;
import static org.mockito.Matchers.eq;
@@ -26,6 +27,7 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
+import java.net.URISyntaxException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -57,17 +59,18 @@ public class TestLs {
@BeforeClass
public static void setup() throws IOException {
conf = new Configuration();
- conf.set("fs.defaultFS", "mockfs:///");
+ conf.set(FS_DEFAULT_NAME_KEY, "mockfs:///");
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
mockFs = mock(FileSystem.class);
}
@Before
- public void resetMock() throws IOException {
+ public void resetMock() throws IOException, URISyntaxException {
reset(mockFs);
AclStatus mockAclStatus = mock(AclStatus.class);
when(mockAclStatus.getEntries()).thenReturn(new ArrayList<AclEntry>());
when(mockFs.getAclStatus(any(Path.class))).thenReturn(mockAclStatus);
+ when(mockFs.getUri()).thenReturn(new URI(conf.get(FS_DEFAULT_NAME_KEY)));
}
// check that default options are correct
@@ -84,6 +87,7 @@ public class TestLs {
assertFalse(ls.isOrderSize());
assertFalse(ls.isOrderTime());
assertFalse(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
}
// check the -C option is recognised
@@ -101,6 +105,7 @@ public class TestLs {
assertFalse(ls.isOrderSize());
assertFalse(ls.isOrderTime());
assertFalse(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
}
// check the -d option is recognised
@@ -118,6 +123,7 @@ public class TestLs {
assertFalse(ls.isOrderSize());
assertFalse(ls.isOrderTime());
assertFalse(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
}
// check the -h option is recognised
@@ -135,6 +141,7 @@ public class TestLs {
assertFalse(ls.isOrderSize());
assertFalse(ls.isOrderTime());
assertFalse(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
}
// check the -R option is recognised
@@ -152,6 +159,7 @@ public class TestLs {
assertFalse(ls.isOrderSize());
assertFalse(ls.isOrderTime());
assertFalse(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
}
// check the -r option is recognised
@@ -169,6 +177,7 @@ public class TestLs {
assertFalse(ls.isOrderSize());
assertFalse(ls.isOrderTime());
assertFalse(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
}
// check the -S option is recognised
@@ -186,6 +195,7 @@ public class TestLs {
assertTrue(ls.isOrderSize());
assertFalse(ls.isOrderTime());
assertFalse(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
}
// check the -t option is recognised
@@ -203,6 +213,7 @@ public class TestLs {
assertFalse(ls.isOrderSize());
assertTrue(ls.isOrderTime());
assertFalse(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
}
// check the precedence of the -t and -S options
@@ -221,6 +232,7 @@ public class TestLs {
assertFalse(ls.isOrderSize());
assertTrue(ls.isOrderTime());
assertFalse(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
}
// check the precedence of the -t, -S and -r options
@@ -240,6 +252,7 @@ public class TestLs {
assertFalse(ls.isOrderSize());
assertTrue(ls.isOrderTime());
assertFalse(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
}
// chheck the -u option is recognised
@@ -257,6 +270,25 @@ public class TestLs {
assertFalse(ls.isOrderSize());
assertFalse(ls.isOrderTime());
assertTrue(ls.isUseAtime());
+ assertFalse(ls.isDisplayECPolicy());
+ }
+
+ // chheck the -e option is recognised
+ @Test
+ public void processOptionsDisplayECPolicy() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-e");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertFalse(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ assertTrue(ls.isDisplayECPolicy());
}
// check all options is handled correctly
@@ -271,6 +303,7 @@ public class TestLs {
options.add("-t"); // time order
options.add("-S"); // size order
options.add("-u"); // show atime
+ options.add("-e"); // show EC policies
Ls ls = new Ls();
ls.processOptions(options);
assertTrue(ls.isPathOnly());
@@ -281,6 +314,7 @@ public class TestLs {
assertFalse(ls.isOrderSize()); // -t overrules -S
assertTrue(ls.isOrderTime());
assertTrue(ls.isUseAtime());
+ assertTrue(ls.isDisplayECPolicy());
}
// check listing of a single file
@@ -1100,6 +1134,35 @@ public class TestLs {
assertEquals("Ls.getName", expected, actual);
}
+ @Test(expected = UnsupportedOperationException.class)
+ public void processPathFileDisplayECPolicyWhenUnsupported()
+ throws IOException {
+ TestFile testFile = new TestFile("testDirectory", "testFile");
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testFile.getPathData());
+ Ls ls = new Ls();
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-e");
+ ls.processOptions(options);
+ ls.processArguments(pathData);
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void processPathDirDisplayECPolicyWhenUnsupported()
+ throws IOException {
+ TestFile testFile = new TestFile("testDirectory", "testFile");
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testFile);
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+ Ls ls = new Ls();
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-e");
+ ls.processOptions(options);
+ ls.processArguments(pathData);
+ }
+
// test class representing a file to be listed
static class TestFile {
private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[8/8] hadoop git commit: HDFS-14154. [SBN read] Document
dfs.ha.tail-edits.period in user guide. Contributed by Chao Sun.
Posted by su...@apache.org.
HDFS-14154. [SBN read] Document dfs.ha.tail-edits.period in user guide. Contributed by Chao Sun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7072e07
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7072e07
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7072e07
Branch: refs/heads/HDFS-12943
Commit: f7072e070135469d78e7563489785c7a2d7923e8
Parents: 8fc4cce
Author: Chao Sun <su...@apache.org>
Authored: Wed Dec 19 22:20:31 2018 -0800
Committer: Chao Sun <su...@apache.org>
Committed: Wed Dec 19 22:20:31 2018 -0800
----------------------------------------------------------------------
.../site/markdown/HDFSHighAvailabilityWithQJM.md | 7 ++++++-
.../src/site/markdown/ObserverNameNode.md | 18 ++++++++++++++++--
2 files changed, 22 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7072e07/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
index 76a9837..4f3df27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
@@ -27,7 +27,12 @@ This document assumes that the reader has a general understanding of general com
Note: Using the Quorum Journal Manager or Conventional Shared Storage
---------------------------------------------------------------------
-This guide discusses how to configure and use HDFS HA using the Quorum Journal Manager (QJM) to share edit logs between the Active and Standby NameNodes. For information on how to configure HDFS HA using NFS for shared storage instead of the QJM, please see [this alternative guide.](./HDFSHighAvailabilityWithNFS.html)
+This guide discusses how to configure and use HDFS HA using the Quorum
+Journal Manager (QJM) to share edit logs between the Active and Standby
+NameNodes. For information on how to configure HDFS HA using NFS for
+shared storage instead of the QJM, please see [this alternative
+guide.](./HDFSHighAvailabilityWithNFS.html). For information on how to
+configure HDFS HA with Observer NameNode, please see [this guide](./ObserverNameNode.html)
Background
----------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7072e07/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ObserverNameNode.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ObserverNameNode.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ObserverNameNode.md
index 2548315..d93256c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ObserverNameNode.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ObserverNameNode.md
@@ -23,13 +23,13 @@ Purpose
This guide provides an overview of the HDFS Observer NameNode feature
and how to configure/install it in a typical HA-enabled cluster. For a
detailed technical design overview, please check the doc attached to
-HDFS-12943.
+[HDFS-12943](https://issues.apache.org/jira/browse/HDFS-12943).
Background
-----------
In a HA-enabled HDFS cluster (for more information, check
-[HDFSHighAvailabilityWithQJM](./HDFSHighAvailabilityWithQJM.md)), there
+[HDFSHighAvailabilityWithQJM](./HDFSHighAvailabilityWithQJM.html)), there
is a single Active NameNode and one or more Standby NameNode(s). The
Active NameNode is responsible for serving all client requests, while
Standby NameNode just keep the up-to-date information regarding the
@@ -104,6 +104,20 @@ few configurations to your **hdfs-site.xml**:
<value>true</value>
</property>
+* **dfs.ha.tail-edits.period** - how often Standby/Observer NameNodes
+ should fetch edits from JournalNodes.
+
+ This determines the staleness of Observer NameNode w.r.t the Active.
+ If too large, RPC time will increase as client requests will wait
+ longer in the RPC queue before Observer tails edit logs and catches
+ up the latest state of Active. The default value is 1min. It is
+ **highly recommend** to configure this to a much lower value.
+
+ <property>
+ <name>dfs.ha.tail-edits.period</name>
+ <value>0ms</value>
+ </property>
+
* **dfs.journalnode.edit-cache-size.bytes** - the in-memory cache size,
in bytes, on the JournalNodes.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/8] hadoop git commit: YARN-9130. Add Bind_HOST configuration for
Yarn Web Proxy. Contributed by Rong Tang.
Posted by su...@apache.org.
YARN-9130. Add Bind_HOST configuration for Yarn Web Proxy. Contributed by Rong Tang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5df9fb16
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5df9fb16
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5df9fb16
Branch: refs/heads/HDFS-12943
Commit: 5df9fb16b9871618f19f3081b7006d3e77106737
Parents: 499c70e
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Dec 19 18:58:00 2018 -0800
Committer: Inigo Goiri <in...@apache.org>
Committed: Wed Dec 19 18:58:00 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +++-
.../src/main/resources/yarn-default.xml | 9 ++++++
.../yarn/server/webproxy/WebAppProxy.java | 14 +++++++--
.../yarn/server/webproxy/WebAppProxyServer.java | 10 ++++---
.../server/webproxy/TestWebAppProxyServer.java | 31 ++++++++++++++++----
5 files changed, 57 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 72cc0d6..d84d11a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2302,13 +2302,17 @@ public class YarnConfiguration extends Configuration {
/** Keytab for Proxy.*/
public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab";
-
+
/** The address for the web proxy.*/
public static final String PROXY_ADDRESS =
PROXY_PREFIX + "address";
public static final int DEFAULT_PROXY_PORT = 9099;
public static final String DEFAULT_PROXY_ADDRESS =
"0.0.0.0:" + DEFAULT_PROXY_PORT;
+
+ /** Binding address for the web proxy. */
+ public static final String PROXY_BIND_HOST =
+ PROXY_PREFIX + "bind-host";
/**
* YARN Service Level Authorization
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index ccceaab..12c5ff6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2157,6 +2157,15 @@
<value/>
</property>
+ <property>
+ <description>The actual address the web proxy will bind to. If this optional
+ address is set, it overrides only the hostname portion of yarn.web-proxy.address.
+ This is useful for making the web proxy server listen on all interfaces by setting
+ it to 0.0.0.0 </description>
+ <name>yarn.web-proxy.bind-host</name>
+ <value/>
+ </property>
+
<!-- Applications' Configuration -->
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
index f205cf7..1382fc6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
@@ -74,16 +74,26 @@ public class WebAppProxy extends AbstractService {
fetcher = new AppReportFetcher(conf);
bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
if(bindAddress == null || bindAddress.isEmpty()) {
- throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS +
+ throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS +
" is not set so the proxy will not run.");
}
- LOG.info("Instantiating Proxy at " + bindAddress);
+
String[] parts = StringUtils.split(bindAddress, ':');
port = 0;
if (parts.length == 2) {
bindAddress = parts[0];
port = Integer.parseInt(parts[1]);
}
+
+ String bindHost = conf.getTrimmed(YarnConfiguration.PROXY_BIND_HOST, null);
+ if (bindHost != null) {
+ LOG.debug("{} is set, will be used to run proxy.",
+ YarnConfiguration.PROXY_BIND_HOST);
+ bindAddress = bindHost;
+ }
+
+ LOG.info("Instantiating Proxy at {}:{}", bindAddress, port);
+
acl = new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
super.serviceInit(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
index a5dd8f6..6be6039 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
@@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory;
/**
* ProxyServer will sit in between the end user and AppMaster
- * web interfaces.
+ * web interfaces.
*/
public class WebAppProxyServer extends CompositeService {
@@ -103,9 +103,11 @@ public class WebAppProxyServer extends CompositeService {
* @return InetSocketAddress
*/
public static InetSocketAddress getBindAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.PROXY_ADDRESS,
- YarnConfiguration.DEFAULT_PROXY_ADDRESS,
- YarnConfiguration.DEFAULT_PROXY_PORT);
+ return conf.getSocketAddr(
+ YarnConfiguration.PROXY_BIND_HOST,
+ YarnConfiguration.PROXY_ADDRESS,
+ YarnConfiguration.DEFAULT_PROXY_ADDRESS,
+ YarnConfiguration.DEFAULT_PROXY_PORT);
}
public static void main(String[] args) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
index 93479a9..87907a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.assertEquals;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -33,14 +32,15 @@ import java.net.InetSocketAddress;
public class TestWebAppProxyServer {
private WebAppProxyServer webAppProxy = null;
- private final String proxyAddress = "0.0.0.0:8888";
+ private final String port = "8888";
+ private final String proxyAddress = "localhost:" + port;
+ private YarnConfiguration conf = null;
@Before
public void setUp() throws Exception {
- YarnConfiguration conf = new YarnConfiguration();
+ conf = new YarnConfiguration();
conf.set(YarnConfiguration.PROXY_ADDRESS, proxyAddress);
webAppProxy = new WebAppProxyServer();
- webAppProxy.init(conf);
}
@After
@@ -50,19 +50,38 @@ public class TestWebAppProxyServer {
@Test
public void testStart() {
+ webAppProxy.init(conf);
assertEquals(STATE.INITED, webAppProxy.getServiceState());
webAppProxy.start();
for (Service service : webAppProxy.getServices()) {
if (service instanceof WebAppProxy) {
- assertEquals(((WebAppProxy) service).getBindAddress(), proxyAddress);
+ assertEquals(proxyAddress, ((WebAppProxy) service).getBindAddress());
}
}
assertEquals(STATE.STARTED, webAppProxy.getServiceState());
}
@Test
+ public void testStartWithBindHost() {
+ String bindHost = "0.0.0.0";
+ conf.set(YarnConfiguration.PROXY_BIND_HOST, bindHost);
+ webAppProxy.init(conf);
+
+ assertEquals(STATE.INITED, webAppProxy.getServiceState());
+ webAppProxy.start();
+ for (Service service : webAppProxy.getServices()) {
+ if (service instanceof WebAppProxy) {
+ assertEquals(bindHost + ":" + port,
+ ((WebAppProxy) service).getBindAddress());
+ }
+ }
+ assertEquals(STATE.STARTED, webAppProxy.getServiceState());
+ }
+
+
+ @Test
public void testBindAddress() {
- YarnConfiguration conf = new YarnConfiguration();
+ conf = new YarnConfiguration();
InetSocketAddress defaultBindAddress = WebAppProxyServer.getBindAddress(conf);
Assert.assertEquals("Web Proxy default bind address port is incorrect",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[6/8] hadoop git commit: HDDS-934. Freon run hung on non-functional
pipeline. Contributed by Nanda kumar.
Posted by su...@apache.org.
HDDS-934. Freon run hung on non-functional pipeline.
Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea621fa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea621fa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea621fa0
Branch: refs/heads/HDFS-12943
Commit: ea621fa0877441ec5d9488702d455616e4df49ba
Parents: 2af90cc
Author: Nanda kumar <na...@apache.org>
Authored: Thu Dec 20 11:10:08 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Thu Dec 20 11:13:15 2018 +0530
----------------------------------------------------------------------
.../hadoop/ozone/freon/RandomKeyGenerator.java | 17 +++++++----------
1 file changed, 7 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea621fa0/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index d85b829..5871b49 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -270,23 +270,20 @@ public final class RandomKeyGenerator implements Callable<Void> {
processor.shutdown();
processor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
completed = true;
- progressbar.shutdown();
- if (validateWrites) {
+ if (exception) {
+ progressbar.terminate();
+ } else {
+ progressbar.shutdown();
+ }
+
+ if (validator != null) {
validator.join();
}
ozoneClient.close();
return null;
}
- private void parseOptions(CommandLine cmdLine) {
- if (keySize < 1024) {
- throw new IllegalArgumentException(
- "keySize can not be less than 1024 bytes");
- }
-
- }
-
/**
* Adds ShutdownHook to print statistics.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[7/8] hadoop git commit: Merge branch 'trunk' into HDFS-12943
Posted by su...@apache.org.
Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8fc4cce1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8fc4cce1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8fc4cce1
Branch: refs/heads/HDFS-12943
Commit: 8fc4cce1e615ffe263c9ca020889becf16f45347
Parents: 2753d6c ea621fa
Author: Chao Sun <su...@apache.org>
Authored: Wed Dec 19 22:19:31 2018 -0800
Committer: Chao Sun <su...@apache.org>
Committed: Wed Dec 19 22:19:31 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/shell/Ls.java | 40 ++++--
.../java/org/apache/hadoop/fs/shell/TestLs.java | 67 +++++++++-
.../v2/app/job/impl/TaskAttemptImpl.java | 15 +++
.../v2/app/job/impl/TestTaskAttempt.java | 84 +++++++++++-
hadoop-maven-plugins/pom.xml | 2 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 17 +--
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +-
.../client/api/ContainerShellWebSocket.java | 7 +
.../yarn/client/api/impl/YarnClientImpl.java | 9 +-
.../src/main/resources/yarn-default.xml | 9 ++
.../impl/container-executor.c | 132 +++++++++++++++++--
.../main/native/container-executor/impl/main.c | 3 +
.../test/test-container-executor.c | 4 +-
.../yarn/server/webproxy/WebAppProxy.java | 14 +-
.../yarn/server/webproxy/WebAppProxyServer.java | 10 +-
.../server/webproxy/TestWebAppProxyServer.java | 31 ++++-
16 files changed, 392 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[5/8] hadoop git commit: HADOOP-15935. [JDK 11] Update
maven.plugin-tools.version to 3.6.0. Contributed by Dinesh Chitlangia.
Posted by su...@apache.org.
HADOOP-15935. [JDK 11] Update maven.plugin-tools.version to 3.6.0. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2af90ccb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2af90ccb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2af90ccb
Branch: refs/heads/HDFS-12943
Commit: 2af90ccbc313f36aec6807232a2ddf08b4b09eec
Parents: 5df9fb1
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Dec 20 14:17:32 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Dec 20 14:17:32 2018 +0900
----------------------------------------------------------------------
hadoop-maven-plugins/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2af90ccb/hadoop-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index b0f6ef4..3c8b0ea 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -27,7 +27,7 @@
<name>Apache Hadoop Maven Plugins</name>
<properties>
<maven.dependency.version>3.0.5</maven.dependency.version>
- <maven.plugin-tools.version>3.5.1</maven.plugin-tools.version>
+ <maven.plugin-tools.version>3.6.0</maven.plugin-tools.version>
</properties>
<dependencies>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org