You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/08/26 20:35:55 UTC
[51/52] [abbrv] hadoop git commit: YARN-5373. NPE listing wildcard
directory in containerLaunch. (Daniel Templeton via kasha)
YARN-5373. NPE listing wildcard directory in containerLaunch. (Daniel Templeton via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cde3a005
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cde3a005
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cde3a005
Branch: refs/heads/HADOOP-13341
Commit: cde3a00526c562a500308232e2b93498d22c90d7
Parents: 9ef632f
Author: Karthik Kambatla <ka...@apache.org>
Authored: Fri Aug 26 11:04:33 2016 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Fri Aug 26 11:04:33 2016 -0700
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 42 +++--
.../nodemanager/DockerContainerExecutor.java | 4 +-
.../nodemanager/LinuxContainerExecutor.java | 40 ++++-
.../launcher/ContainerLaunch.java | 2 +-
.../linux/privileged/PrivilegedOperation.java | 6 +-
.../impl/container-executor.c | 54 ++++++-
.../impl/container-executor.h | 10 +-
.../main/native/container-executor/impl/main.c | 40 +++--
.../test/test-container-executor.c | 157 +++++++++++++++++++
.../launcher/TestContainerLaunch.java | 12 +-
10 files changed, 325 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index c73c4c7..818b0ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -67,8 +67,8 @@ import org.apache.hadoop.util.StringUtils;
* underlying OS. All executor implementations must extend ContainerExecutor.
*/
public abstract class ContainerExecutor implements Configurable {
- private static final String WILDCARD = "*";
private static final Log LOG = LogFactory.getLog(ContainerExecutor.class);
+ protected static final String WILDCARD = "*";
/**
* The permissions to use when creating the launch script.
@@ -274,15 +274,16 @@ public abstract class ContainerExecutor implements Configurable {
* @param environment the environment variables and their values
* @param resources the resources which have been localized for this
* container. Symlinks will be created to these localized resources
- * @param command the command that will be run.
- * @param logDir the log dir to copy debugging information to
+ * @param command the command that will be run
+ * @param logDir the log dir to which to copy debugging information
+ * @param user the username of the job owner
* @throws IOException if any errors happened writing to the OutputStream,
* while creating symlinks
*/
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
- Map<Path, List<String>> resources, List<String> command, Path logDir)
- throws IOException {
- this.writeLaunchEnv(out, environment, resources, command, logDir,
+ Map<Path, List<String>> resources, List<String> command, Path logDir,
+ String user) throws IOException {
+ this.writeLaunchEnv(out, environment, resources, command, logDir, user,
ContainerLaunch.CONTAINER_SCRIPT);
}
@@ -295,17 +296,17 @@ public abstract class ContainerExecutor implements Configurable {
* @param environment the environment variables and their values
* @param resources the resources which have been localized for this
* container. Symlinks will be created to these localized resources
- * @param command the command that will be run.
- * @param logDir the log dir to copy debugging information to
+ * @param command the command that will be run
+ * @param logDir the log dir to which to copy debugging information
+ * @param user the username of the job owner
* @param outFilename the path to which to write the launch environment
* @throws IOException if any errors happened writing to the OutputStream,
* while creating symlinks
*/
@VisibleForTesting
- public void writeLaunchEnv(OutputStream out,
- Map<String, String> environment, Map<Path, List<String>> resources,
- List<String> command, Path logDir, String outFilename)
- throws IOException {
+ public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
+ Map<Path, List<String>> resources, List<String> command, Path logDir,
+ String user, String outFilename) throws IOException {
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
Set<String> whitelist = new HashSet<>();
@@ -334,9 +335,7 @@ public abstract class ContainerExecutor implements Configurable {
if (new Path(linkName).getName().equals(WILDCARD)) {
// If this is a wildcarded path, link to everything in the
// directory from the working directory
- File directory = new File(resourceEntry.getKey().toString());
-
- for (File wildLink : directory.listFiles()) {
+ for (File wildLink : readDirAsUser(user, resourceEntry.getKey())) {
sb.symlink(new Path(wildLink.toString()),
new Path(wildLink.getName()));
}
@@ -371,6 +370,19 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
+ * Return the files in the target directory. If retrieving the list of files
+ * requires specific access rights, that access will happen as the
+ * specified user. The list will not include entries for "." or "..".
+ *
+ * @param user the user as whom to access the target directory
+ * @param dir the target directory
+ * @return a list of files in the target directory
+ */
+ protected File[] readDirAsUser(String user, Path dir) {
+ return new File(dir.toString()).listFiles();
+ }
+
+ /**
* The container exit code.
*/
public enum ExitCode {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index 7bf6989..ebf9566 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -330,8 +330,8 @@ public class DockerContainerExecutor extends ContainerExecutor {
* the docker image and write them out to an OutputStream.
*/
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
- Map<Path, List<String>> resources, List<String> command, Path logDir)
- throws IOException {
+ Map<Path, List<String>> resources, List<String> command, Path logDir,
+ String user) throws IOException {
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 6c658bc..6890b25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler;
import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler;
import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler;
-
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -645,6 +644,45 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ protected File[] readDirAsUser(String user, Path dir) {
+ List<File> files = new ArrayList<>();
+ PrivilegedOperation listAsUserOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.LIST_AS_USER, (String)null);
+ String runAsUser = getRunAsUser(user);
+ String dirString = "";
+
+ if (dir != null) {
+ dirString = dir.toUri().getPath();
+ }
+
+ listAsUserOp.appendArgs(runAsUser, user,
+ Integer.toString(
+ PrivilegedOperation.RunAsUserCommand.LIST_AS_USER.getValue()),
+ dirString);
+
+ try {
+ PrivilegedOperationExecutor privOpExecutor =
+ PrivilegedOperationExecutor.getInstance(super.getConf());
+
+ String results =
+ privOpExecutor.executePrivilegedOperation(listAsUserOp, true);
+
+ for (String file: results.split("\n")) {
+ // The container-executor always dumps its log output to stdout, which
+ // includes 3 lines that start with "main : "
+ if (!file.startsWith("main :")) {
+ files.add(new File(new File(dirString), file));
+ }
+ }
+ } catch (PrivilegedOperationException e) {
+ LOG.error("ListAsUser for " + dir + " returned with exit code: "
+ + e.getExitCode(), e);
+ }
+
+ return files.toArray(new File[files.size()]);
+ }
+
+ @Override
public boolean isContainerAlive(ContainerLivenessContext ctx)
throws IOException {
String user = ctx.getUser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 81b6c1f..14190fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -267,7 +267,7 @@ public class ContainerLaunch implements Callable<Integer> {
// Write out the environment
exec.writeLaunchEnv(containerScriptOutStream, environment,
localResources, launchContext.getCommands(),
- new Path(containerLogDirs.get(0)));
+ new Path(containerLogDirs.get(0)), user);
// /////////// End of writing out container-script
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
index 259dee8..8402a16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
@@ -50,7 +50,8 @@ public class PrivilegedOperation {
TC_READ_STATE("--tc-read-state"),
TC_READ_STATS("--tc-read-stats"),
ADD_PID_TO_CGROUP(""), //no CLI switch supported yet.
- RUN_DOCKER_CMD("--run-docker");
+ RUN_DOCKER_CMD("--run-docker"),
+ LIST_AS_USER(""); //no CLI switch supported yet.
private final String option;
@@ -146,7 +147,8 @@ public class PrivilegedOperation {
LAUNCH_CONTAINER(1),
SIGNAL_CONTAINER(2),
DELETE_AS_USER(3),
- LAUNCH_DOCKER_CONTAINER(4);
+ LAUNCH_DOCKER_CONTAINER(4),
+ LIST_AS_USER(5);
private int value;
RunAsUserCommand(int value) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index f8c7f6e..ca3847e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -439,7 +439,7 @@ char *concatenate(char *concat_pattern, char *return_path_name,
for (j = 0; j < numArgs; j++) {
arg = va_arg(ap, char*);
if (arg == NULL) {
- fprintf(LOGFILE, "One of the arguments passed for %s in null.\n",
+ fprintf(LOGFILE, "One of the arguments passed for %s is null.\n",
return_path_name);
return NULL;
}
@@ -1929,6 +1929,58 @@ int delete_as_user(const char *user,
return ret;
}
+/**
+ * List the files in the given directory as the user.
+ * user: the user listing the files
+ * target_dir: the directory from which to list files
+ */
+int list_as_user(const char *target_dir) {
+ int ret = 0;
+ struct stat sb;
+
+ if (stat(target_dir, &sb) != 0) {
+ // If directory doesn't exist or can't be accessed, error out
+ fprintf(LOGFILE, "Could not stat %s - %s\n", target_dir,
+ strerror(errno));
+ ret = -1;
+ } else if (!S_ISDIR(sb.st_mode)) {
+ // If it's not a directory, list it as the only file
+ printf("%s\n", target_dir);
+ } else {
+ DIR *dir = opendir(target_dir);
+
+ if (dir != NULL) {
+ struct dirent *file;
+
+ errno = 0;
+
+ do {
+ file = readdir(dir);
+
+ // Ignore the . and .. entries
+ if ((file != NULL) &&
+ (strcmp(".", file->d_name) != 0) &&
+ (strcmp("..", file->d_name) != 0)) {
+ printf("%s\n", file->d_name);
+ }
+ } while (file != NULL);
+
+ // If we ended the directory read early on an error, then error out
+ if (errno != 0) {
+ fprintf(LOGFILE, "Could not read directory %s - %s\n", target_dir,
+ strerror(errno));
+ ret = -1;
+ }
+ } else {
+ fprintf(LOGFILE, "Could not open directory %s - %s\n", target_dir,
+ strerror(errno));
+ ret = -1;
+ }
+ }
+
+ return ret;
+}
+
void chown_dir_contents(const char *dir_path, uid_t uid, gid_t gid) {
DIR *dp;
struct dirent *ep;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index be78283..1858555 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -31,7 +31,8 @@ enum command {
LAUNCH_CONTAINER = 1,
SIGNAL_CONTAINER = 2,
DELETE_AS_USER = 3,
- LAUNCH_DOCKER_CONTAINER = 4
+ LAUNCH_DOCKER_CONTAINER = 4,
+ LIST_AS_USER = 5
};
enum errorcodes {
@@ -79,7 +80,8 @@ enum operations {
RUN_AS_USER_SIGNAL_CONTAINER = 8,
RUN_AS_USER_DELETE = 9,
RUN_AS_USER_LAUNCH_DOCKER_CONTAINER = 10,
- RUN_DOCKER = 11
+ RUN_DOCKER = 11,
+ RUN_AS_USER_LIST = 12
};
#define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group"
@@ -189,6 +191,10 @@ int delete_as_user(const char *user,
const char *dir_to_be_deleted,
char* const* baseDirs);
+// List the files in the given directory on stdout. The target_dir is always
+// assumed to be an absolute path.
+int list_as_user(const char *target_dir);
+
// set the uid and gid of the node manager. This is used when doing some
// priviledged operations for setting the effective uid and gid.
void set_nm_uid(uid_t user, gid_t group);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 5fc92d6..56215ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -58,11 +58,12 @@ static void display_usage(FILE *stream) {
" launch docker container: %2d appid containerid workdir container-script " \
"tokens pidfile nm-local-dirs nm-log-dirs docker-command-file resources optional-tc-command-file\n" \
" signal container: %2d container-pid signal\n" \
- " delete as user: %2d relative-path\n" ;
+ " delete as user: %2d relative-path\n" \
+ " list as user: %2d relative-path\n" ;
fprintf(stream, usage_template, INITIALIZE_CONTAINER, LAUNCH_CONTAINER, LAUNCH_DOCKER_CONTAINER,
- SIGNAL_CONTAINER, DELETE_AS_USER);
+ SIGNAL_CONTAINER, DELETE_AS_USER, LIST_AS_USER);
}
/* Sets up log files for normal/error logging */
@@ -169,20 +170,20 @@ static void assert_valid_setup(char *argv0) {
static struct {
char *cgroups_hierarchy;
char *traffic_control_command_file;
- const char * run_as_user_name;
- const char * yarn_user_name;
+ const char *run_as_user_name;
+ const char *yarn_user_name;
char *local_dirs;
char *log_dirs;
char *resources_key;
char *resources_value;
char **resources_values;
- const char * app_id;
- const char * container_id;
- const char * cred_file;
- const char * script_file;
- const char * current_dir;
- const char * pid_file;
- const char *dir_to_be_deleted;
+ const char *app_id;
+ const char *container_id;
+ const char *cred_file;
+ const char *script_file;
+ const char *current_dir;
+ const char *pid_file;
+ const char *target_dir;
int container_pid;
int signal;
const char *docker_command_file;
@@ -417,9 +418,13 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
return 0;
case DELETE_AS_USER:
- cmd_input.dir_to_be_deleted = argv[optind++];
+ cmd_input.target_dir = argv[optind++];
*operation = RUN_AS_USER_DELETE;
return 0;
+ case LIST_AS_USER:
+ cmd_input.target_dir = argv[optind++];
+ *operation = RUN_AS_USER_LIST;
+ return 0;
default:
fprintf(ERRORFILE, "Invalid command %d not supported.",command);
fflush(ERRORFILE);
@@ -554,9 +559,18 @@ int main(int argc, char **argv) {
}
exit_code = delete_as_user(cmd_input.yarn_user_name,
- cmd_input.dir_to_be_deleted,
+ cmd_input.target_dir,
argv + optind);
break;
+ case RUN_AS_USER_LIST:
+ exit_code = set_user(cmd_input.run_as_user_name);
+
+ if (exit_code != 0) {
+ break;
+ }
+
+ exit_code = list_as_user(cmd_input.target_dir);
+ break;
}
flush_and_close_log_files();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index cad7c3f..f174a9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -454,6 +454,163 @@ void test_delete_user() {
free(app_dir);
}
+/**
+ * Read a file and tokenize it on newlines. Place up to max lines into lines.
+ * The max+1st element of lines will be set to NULL.
+ *
+ * @param file the name of the file to open
+ * @param lines the pointer array into which to place the lines
+ * @param max the max number of lines to add to lines
+ */
+void read_lines(const char* file, char **lines, size_t max) {
+ char buf[4096];
+ size_t nread;
+
+ int fd = open(file, O_RDONLY);
+
+ if (fd < 0) {
+ printf("FAIL: failed to open directory listing file: %s\n", file);
+ exit(1);
+ } else {
+ char *cur = buf;
+ size_t count = sizeof buf;
+
+ while ((nread = read(fd, cur, count)) > 0) {
+ cur += nread;
+ count -= nread;
+ }
+
+ if (nread < 0) {
+ printf("FAIL: failed to read directory listing file: %s\n", file);
+ exit(1);
+ }
+
+ close(fd);
+ }
+
+ char* entity = strtok(buf, "\n");
+ int i;
+
+ for (i = 0; i < max; i++) {
+ if (entity == NULL) {
+ break;
+ }
+
+ lines[i] = (char *)malloc(sizeof(char) * (strlen(entity) + 1));
+ strcpy(lines[i], entity);
+ entity = strtok(NULL, "\n");
+ }
+
+ lines[i] = NULL;
+}
+
+void test_list_as_user() {
+ printf("\nTesting list_as_user\n");
+ char buffer[4096];
+
+ char *app_dir =
+ get_app_directory(TEST_ROOT "/local-1", "yarn", "app_4");
+
+ if (mkdirs(app_dir, 0700) != 0) {
+ printf("FAIL: unble to create application directory: %s\n", app_dir);
+ exit(1);
+ }
+
+ // Test with empty dir string
+ sprintf(buffer, "");
+ int ret = list_as_user(buffer);
+
+ if (ret == 0) {
+ printf("FAIL: did not fail on empty directory string\n");
+ exit(1);
+ }
+
+ // Test with a non-existent directory
+ sprintf(buffer, "%s/output", app_dir);
+
+ ret = list_as_user(buffer);
+
+ if (ret == 0) {
+ printf("FAIL: did not fail on non-existent directory\n");
+ exit(1);
+ }
+
+ // Write a couple files to list
+ sprintf(buffer, "%s/file1", app_dir);
+
+ if (write_config_file(buffer, 1) != 0) {
+ exit(1);
+ }
+
+ sprintf(buffer, "%s/.file2", app_dir);
+
+ if (write_config_file(buffer, 1) != 0) {
+ exit(1);
+ }
+
+ // Also create a directory
+ sprintf(buffer, "%s/output", app_dir);
+
+ if (mkdirs(buffer, 0700) != 0) {
+ exit(1);
+ }
+
+ // Test the regular case
+ // Store a copy of stdout, then redirect it to a file
+ sprintf(buffer, "%s/output/files", app_dir);
+
+ int oldout = dup(STDOUT_FILENO);
+ int fd = open(buffer, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
+
+ dup2(fd, STDOUT_FILENO);
+
+ // Now list the files
+ ret = list_as_user(app_dir);
+
+ if (ret != 0) {
+ printf("FAIL: unable to list files in regular case\n");
+ exit(1);
+ }
+
+ // Restore stdout
+ close(fd);
+ dup2(oldout, STDOUT_FILENO);
+
+ // Check the output -- shouldn't be more than a couple lines
+ char *lines[16];
+
+ read_lines(buffer, lines, 15);
+
+ int got_file1 = 0;
+ int got_file2 = 0;
+ int got_output = 0;
+ int i;
+
+ for (i = 0; i < sizeof lines; i++) {
+ if (lines[i] == NULL) {
+ break;
+ } else if (strcmp("file1", lines[i]) == 0) {
+ got_file1 = 1;
+ } else if (strcmp(".file2", lines[i]) == 0) {
+ got_file2 = 1;
+ } else if (strcmp("output", lines[i]) == 0) {
+ got_output = 1;
+ } else {
+ printf("FAIL: listed extraneous file: %s\n", lines[i]);
+ exit(1);
+ }
+
+ free(lines[i]);
+ }
+
+ if (!got_file1 || !got_file2 || !got_output) {
+ printf("FAIL: missing files in listing\n");
+ exit(1);
+ }
+
+ free(app_dir);
+}
+
void run_test_in_child(const char* test_name, void (*func)()) {
printf("\nRunning test %s in child process\n", test_name);
fflush(stdout);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index a06822a..be6eadb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -174,7 +174,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
new DefaultContainerExecutor()
.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), tempFile.getName());
+ new Path(localLogDir.getAbsolutePath()), "user",
+ tempFile.getName());
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -243,7 +244,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
new DefaultContainerExecutor()
.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()));
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -298,7 +299,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
List<String> commands = new ArrayList<String>();
new DefaultContainerExecutor()
.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()));
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
@@ -377,7 +378,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
commands.add(command);
ContainerExecutor exec = new DefaultContainerExecutor();
exec.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()));
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
@@ -1400,7 +1401,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
ContainerExecutor exec = new DefaultContainerExecutor();
exec.setConf(conf);
exec.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), tempFile.getName());
+ new Path(localLogDir.getAbsolutePath()), "user",
+ tempFile.getName());
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org