You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by yh...@apache.org on 2010/01/31 12:46:23 UTC
svn commit: r905008 - in /hadoop/mapreduce/trunk: ./ conf/
src/c++/task-controller/ src/docs/src/documentation/content/xdocs/
src/java/ src/java/org/apache/hadoop/mapred/
src/java/org/apache/hadoop/mapreduce/server/tasktracker/
src/test/mapred/org/apac...
Author: yhemanth
Date: Sun Jan 31 11:46:20 2010
New Revision: 905008
URL: http://svn.apache.org/viewvc?rev=905008&view=rev
Log:
MAPREDUCE-899. Modified LinuxTaskController to check that task-controller has right permissions and ownership before performing any actions. Contributed by Amareshwari Sriramadasu.
Added:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestLinuxTaskController.java
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/conf/taskcontroller.cfg
hadoop/mapreduce/trunk/src/c++/task-controller/main.c
hadoop/mapreduce/trunk/src/c++/task-controller/task-controller.h
hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/cluster_setup.xml
hadoop/mapreduce/trunk/src/java/mapred-default.xml
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LinuxTaskController.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskController.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestLocalizationWithLinuxTaskController.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTrackerDistributedCacheManagerWithLinuxTaskController.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Sun Jan 31 11:46:20 2010
@@ -285,6 +285,10 @@
based fixing a failing streaming test.
(Devaraj Das and Amareshwari Sriramadasu via yhemanth)
+ MAPREDUCE-899. Modified LinuxTaskController to check that task-controller
+ has right permissions and ownership before performing any actions.
+ (Amareshwari Sriramadasu via yhemanth)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/conf/taskcontroller.cfg
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/conf/taskcontroller.cfg?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/conf/taskcontroller.cfg (original)
+++ hadoop/mapreduce/trunk/conf/taskcontroller.cfg Sun Jan 31 11:46:20 2010
@@ -1,2 +1,3 @@
mapreduce.cluster.local.dir=#configured value of mapreduce.cluster.local.dir. It can be a list of comma separated paths.
-hadoop.log.dir=#configured value of hadoop.log.dir.
\ No newline at end of file
+hadoop.log.dir=#configured value of hadoop.log.dir.
+mapreduce.tasktracker.group=#configured value of mapreduce.tasktracker.group
Modified: hadoop/mapreduce/trunk/src/c++/task-controller/main.c
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/c%2B%2B/task-controller/main.c?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/c++/task-controller/main.c (original)
+++ hadoop/mapreduce/trunk/src/c++/task-controller/main.c Sun Jan 31 11:46:20 2010
@@ -44,6 +44,80 @@
"Usage: task-controller [-l logfile] user command command-args\n");
}
+/**
+ * Check the permissions on taskcontroller to make sure that security is
+ * promisable. For this, we need task-controller binary to
+ * * be user-owned by root
+ * * be group-owned by a configured special group.
+ * * others do not have any permissions
+ * * be setuid/setgid
+ */
+int check_taskcontroller_permissions(char *executable_file) {
+
+ errno = 0;
+ char * resolved_path = (char *) canonicalize_file_name(executable_file);
+ if (resolved_path == NULL) {
+ fprintf(LOGFILE,
+ "Error resolving the canonical name for the executable : %s!",
+ strerror(errno));
+ return -1;
+ }
+
+ struct stat filestat;
+ errno = 0;
+ if (stat(resolved_path, &filestat) != 0) {
+ fprintf(LOGFILE, "Could not stat the executable : %s!.\n", strerror(errno));
+ return -1;
+ }
+
+ uid_t binary_euid = filestat.st_uid; // Binary's user owner
+ gid_t binary_egid = filestat.st_gid; // Binary's group owner
+
+ // Effective uid should be root
+ if (binary_euid != 0) {
+ fprintf(LOGFILE,
+ "The task-controller binary should be user-owned by root.\n");
+ return -1;
+ }
+
+ // Get the group entry for the special_group
+ errno = 0;
+ struct group *special_group_entry = getgrgid(binary_egid);
+ if (special_group_entry == NULL) {
+ fprintf(LOGFILE,
+ "Unable to get information for effective group of the binary : %s\n",
+ strerror(errno));
+ return -1;
+ }
+
+ char * binary_group = special_group_entry->gr_name;
+ // verify that the group name of the special group
+ // is same as the one in configuration
+ if (check_variable_against_config(TT_GROUP_KEY, binary_group) != 0) {
+ fprintf(LOGFILE,
+ "Group of the binary does not match with that in configuration\n");
+ return -1;
+ }
+
+ // check others do not have read/write/execute permissions
+ if ((filestat.st_mode & S_IROTH) == S_IROTH || (filestat.st_mode & S_IWOTH)
+ == S_IWOTH || (filestat.st_mode & S_IXOTH) == S_IXOTH) {
+ fprintf(LOGFILE,
+ "The task-controller binary should not have read or write or execute for others.\n");
+ return -1;
+ }
+
+ // Binary should be setuid/setgid executable
+ if ((filestat.st_mode & S_ISUID) != S_ISUID || (filestat.st_mode & S_ISGID)
+ != S_ISGID) {
+ fprintf(LOGFILE,
+ "The task-controller binary should be set setuid and setgid bits.\n");
+ return -1;
+ }
+
+ return 0;
+}
+
int main(int argc, char **argv) {
int command;
int next_option = 0;
@@ -62,24 +136,19 @@
char * dir_to_be_deleted = NULL;
int conf_dir_len = 0;
- //Minimum number of arguments required to run the task-controller
- //command-name user command tt-root
- if (argc < 3) {
- display_usage(stdout);
- return INVALID_ARGUMENT_NUMBER;
- }
-
+ char *executable_file = argv[0];
#ifndef HADOOP_CONF_DIR
- conf_dir_len = (strlen(argv[0]) - strlen(EXEC_PATTERN)) + 1;
+ conf_dir_len = (strlen(executable_file) - strlen(EXEC_PATTERN)) + 1;
if (conf_dir_len < 1) {
- // We didn't get an absolute path to our argv[0]; bail.
+ // We didn't get an absolute path to our executable_file; bail.
printf("Cannot find configuration directory.\n");
printf("This program must be run with its full absolute path.\n");
return INVALID_CONF_DIR;
} else {
hadoop_conf_dir = (char *) malloc (sizeof(char) * conf_dir_len);
- strncpy(hadoop_conf_dir,argv[0],(strlen(argv[0]) - strlen(EXEC_PATTERN)));
- hadoop_conf_dir[(strlen(argv[0]) - strlen(EXEC_PATTERN))] = '\0';
+ strncpy(hadoop_conf_dir, executable_file,
+ (strlen(executable_file) - strlen(EXEC_PATTERN)));
+ hadoop_conf_dir[(strlen(executable_file) - strlen(EXEC_PATTERN))] = '\0';
}
#endif
do {
@@ -94,6 +163,18 @@
open_log_file(log_file);
+ if (check_taskcontroller_permissions(executable_file) != 0) {
+ fprintf(LOGFILE, "Invalid permissions on task-controller binary.\n");
+ return INVALID_TASKCONTROLLER_PERMISSIONS;
+ }
+
+ //Minimum number of arguments required to run the task-controller
+ //command-name user command tt-root
+ if (argc < 3) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+
//checks done for user name
//checks done if the user is root or not.
if (argv[optind] == NULL) {
Modified: hadoop/mapreduce/trunk/src/c++/task-controller/task-controller.h
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/c%2B%2B/task-controller/task-controller.h?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/c++/task-controller/task-controller.h (original)
+++ hadoop/mapreduce/trunk/src/c++/task-controller/task-controller.h Sun Jan 31 11:46:20 2010
@@ -72,7 +72,8 @@
INITIALIZE_USER_FAILED, //20
UNABLE_TO_EXECUTE_DEBUG_SCRIPT, //21
INVALID_CONF_DIR, //22
- UNABLE_TO_BUILD_PATH //23
+ UNABLE_TO_BUILD_PATH, //23
+ INVALID_TASKCONTROLLER_PERMISSIONS //24
};
#define USER_DIR_PATTERN "%s/taskTracker/%s"
@@ -95,6 +96,8 @@
#define TT_LOG_DIR_KEY "hadoop.log.dir"
+#define TT_GROUP_KEY "mapreduce.tasktracker.group"
+
#ifndef HADOOP_CONF_DIR
#define EXEC_PATTERN "/bin/task-controller"
extern char * hadoop_conf_dir;
Modified: hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/cluster_setup.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/cluster_setup.xml?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/cluster_setup.xml (original)
+++ hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/cluster_setup.xml Sun Jan 31 11:46:20 2010
@@ -955,21 +955,35 @@
<p>
The executable must have specific permissions as follows. The
executable should have <em>6050 or --Sr-s---</em> permissions
- user-owned by root(super-user) and group-owned by a group
- of which only the TaskTracker's user is the sole group member.
+ user-owned by root(super-user) and group-owned by a special group
+ of which the TaskTracker's user is the group member and no job
+ submitter is. If any job submitter belongs to this special group,
+ security will be compromised. This special group name should be
+ specified for the configuration property
+ <em>"mapreduce.tasktracker.group"</em> in both mapred-site.xml and
+ <a href="#task-controller.cfg">task-controller.cfg</a>.
For example, let's say that the TaskTracker is run as user
<em>mapred</em> who is part of the groups <em>users</em> and
- <em>mapredGroup</em> any of them being the primary group.
+ <em>specialGroup</em> any of them being the primary group.
Let also be that <em>users</em> has both <em>mapred</em> and
- another user <em>X</em> as its members, while <em>mapredGroup</em>
- has only <em>mapred</em> as its member. Going by the above
+ another user (job submitter) <em>X</em> as its members, and X does
+ not belong to <em>specialGroup</em>. Going by the above
description, the setuid/setgid executable should be set
<em>6050 or --Sr-s---</em> with user-owner as <em>mapred</em> and
- group-owner as <em>mapredGroup</em> which has
- only <em>mapred</em> as its member(and not <em>users</em> which has
+ group-owner as <em>specialGroup</em> which has
+ <em>mapred</em> as its member(and not <em>users</em> which has
<em>X</em> also as its member besides <em>mapred</em>).
</p>
+
+ <p>
+ The LinuxTaskController requires that paths including and leading up
+ to the directories specified in
+ <em>mapreduce.cluster.local.dir</em> and <em>hadoop.log.dir</em> to
+ be set 755 permissions.
+ </p>
+ <section>
+ <title>task-controller.cfg</title>
<p>The executable requires a configuration file called
<em>taskcontroller.cfg</em> to be
present in the configuration directory passed to the ant target
@@ -1000,14 +1014,16 @@
permissions on the log files so that they can be written to by the user's
tasks and read by the TaskTracker for serving on the web UI.</td>
</tr>
+ <tr>
+ <td>mapreduce.tasktracker.group</td>
+ <td>Group to which the TaskTracker belongs. The group owner of the
+ taskcontroller binary should be this group. Should be same as
+ the value with which the TaskTracker is configured. This
+ configuration is required for validating the secure access of the
+ task-controller binary.</td>
+ </tr>
</table>
-
- <p>
- The LinuxTaskController requires that paths including and leading up to
- the directories specified in
- <em>mapreduce.cluster.local.dir</em> and <em>hadoop.log.dir</em> to be set 755
- permissions.
- </p>
+ </section>
</section>
</section>
Modified: hadoop/mapreduce/trunk/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/mapred-default.xml?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/mapred-default.xml (original)
+++ hadoop/mapreduce/trunk/src/java/mapred-default.xml Sun Jan 31 11:46:20 2010
@@ -970,6 +970,15 @@
</description>
</property>
+<property>
+ <name>mapreduce.tasktracker.group</name>
+ <value></value>
+ <description>Expert: Group to which TaskTracker belongs. If
+ LinuxTaskController is configured via mapreduce.tasktracker.taskcontroller,
+ the group owner of the task-controller binary should be same as this group.
+ </description>
+</property>
+
<!-- Node health script variables -->
<property>
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LinuxTaskController.java?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LinuxTaskController.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LinuxTaskController.java Sun Jan 31 11:46:20 2010
@@ -34,6 +34,7 @@
import org.apache.hadoop.mapred.CleanupQueue.PathDeletionContext;
import org.apache.hadoop.mapred.JvmManager.JvmEnv;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
/**
@@ -95,6 +96,29 @@
ENABLE_TASK_FOR_CLEANUP
}
+ @Override
+ public void setup() throws IOException {
+ super.setup();
+
+ // Check the permissions of the task-controller binary by running it plainly.
+ // If permissions are correct, it returns an error code 1, else it returns
+ // 24 or something else if some other bugs are also present.
+ String[] taskControllerCmd =
+ new String[] { getTaskControllerExecutablePath() };
+ ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd);
+ try {
+ shExec.execute();
+ } catch (ExitCodeException e) {
+ int exitCode = shExec.getExitCode();
+ if (exitCode != 1) {
+ LOG.warn("Exit code from checking binary permissions is : " + exitCode);
+ logOutput(shExec.getOutput());
+ throw new IOException("Task controller setup failed because of invalid"
+ + "permissions/ownership with exit code " + exitCode, e);
+ }
+ }
+ }
+
/**
* Launch a task JVM that will run as the owner of the job.
*
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskController.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskController.java?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskController.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskController.java Sun Jan 31 11:46:20 2010
@@ -75,7 +75,7 @@
* <li>Hadoop log directories</li>
* </ul>
*/
- public void setup() {
+ public void setup() throws IOException {
for (String localDir : this.mapredLocalDirs) {
// Set up the mapreduce.cluster.local.directories.
File mapredlocalDir = new File(localDir);
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java Sun Jan 31 11:46:20 2010
@@ -86,4 +86,6 @@
public static final String TT_USER_NAME = "mapreduce.tasktracker.user.name";
public static final String TT_KEYTAB_FILE =
"mapreduce.tasktracker.keytab.file";
+ public static final String TT_GROUP =
+ "mapreduce.tasktracker.group";
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java Sun Jan 31 11:46:20 2010
@@ -22,10 +22,10 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
-import java.security.PrivilegedExceptionAction;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -34,6 +34,7 @@
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
@@ -68,7 +69,22 @@
*
**/
public static class MyLinuxTaskController extends LinuxTaskController {
- String taskControllerExePath;
+ String taskControllerExePath = System.getProperty(TASKCONTROLLER_PATH)
+ + "/task-controller";
+
+ @Override
+ public void setup() throws IOException {
+ // get the current ugi and set the task controller group owner
+ Groups groups = new Groups(new Configuration());
+ String ttGroup = groups.getGroups(
+ UserGroupInformation.getCurrentUser().getUserName()).get(0);
+ getConf().set(TTConfig.TT_GROUP, ttGroup);
+
+ // write configuration file
+ configurationFile = createTaskControllerConf(System
+ .getProperty(TASKCONTROLLER_PATH), getConf());
+ super.setup();
+ }
@Override
protected String getTaskControllerExecutablePath() {
@@ -105,12 +121,17 @@
private JobConf clusterConf = null;
protected Path homeDirectory;
+ /** changing this to a larger number needs more work for creating
+ * taskcontroller.cfg.
+ * see {@link #startCluster()} and
+ * {@link #createTaskControllerConf(String, Configuration)}
+ */
private static final int NUMBER_OF_NODES = 1;
static final String TASKCONTROLLER_PATH = "taskcontroller-path";
static final String TASKCONTROLLER_UGI = "taskcontroller-ugi";
- private File configurationFile = null;
+ private static File configurationFile = null;
protected UserGroupInformation taskControllerUser;
@@ -128,19 +149,9 @@
new MiniMRCluster(NUMBER_OF_NODES, dfsCluster.getFileSystem().getUri()
.toString(), 4, null, null, conf);
- // Get the configured taskcontroller-path
- String path = System.getProperty(TASKCONTROLLER_PATH);
- configurationFile =
- createTaskControllerConf(path, mrCluster.getTaskTrackerRunner(0)
- .getLocalDirs());
- String execPath = path + "/task-controller";
- TaskTracker tracker = mrCluster.getTaskTrackerRunner(0).tt;
- // TypeCasting the parent to our TaskController instance as we
- // know that that would be instance which should be present in TT.
- ((MyLinuxTaskController) tracker.getTaskController())
- .setTaskControllerExe(execPath);
- String ugi = System.getProperty(TASKCONTROLLER_UGI);
clusterConf = mrCluster.createJobConf();
+
+ String ugi = System.getProperty(TASKCONTROLLER_UGI);
String[] splits = ugi.split(",");
taskControllerUser = UserGroupInformation.createUserForTesting(splits[0],
new String[]{splits[1]});
@@ -167,16 +178,21 @@
taskControllerUser.getGroupNames()[0]);
}
+ static File getTaskControllerConfFile(String path) {
+ File confDirectory = new File(path, "../conf");
+ return new File(confDirectory, "taskcontroller.cfg");
+ }
+
/**
* Create taskcontroller.cfg.
*
* @param path Path to the taskcontroller binary.
- * @param localDirs
+ * @param conf TaskTracker's configuration
* @return the created conf file
* @throws IOException
*/
- static File createTaskControllerConf(String path, String[] localDirs)
- throws IOException {
+ static File createTaskControllerConf(String path,
+ Configuration conf) throws IOException {
File confDirectory = new File(path, "../conf");
if (!confDirectory.exists()) {
confDirectory.mkdirs();
@@ -185,17 +201,13 @@
PrintWriter writer =
new PrintWriter(new FileOutputStream(configurationFile));
- StringBuffer sb = new StringBuffer();
- for (int i = 0; i < localDirs.length; i++) {
- sb.append(localDirs[i]);
- if ((i + 1) != localDirs.length) {
- sb.append(",");
- }
- }
- writer.println(String.format(MRConfig.LOCAL_DIR + "=%s", sb.toString()));
+ writer.println(String.format(MRConfig.LOCAL_DIR + "=%s", conf
+ .get(MRConfig.LOCAL_DIR)));
writer
.println(String.format("hadoop.log.dir=%s", TaskLog.getBaseLogDir()));
+ writer.println(String.format(TTConfig.TT_GROUP + "=%s",
+ conf.get(TTConfig.TT_GROUP)));
writer.flush();
writer.close();
@@ -215,7 +227,7 @@
return true;
}
- private static boolean isTaskExecPathPassed() {
+ static boolean isTaskExecPathPassed() {
String path = System.getProperty(TASKCONTROLLER_PATH);
if (path == null || path.isEmpty()
|| path.equals("${" + TASKCONTROLLER_PATH + "}")) {
Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestLinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestLinuxTaskController.java?rev=905008&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestLinuxTaskController.java (added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestLinuxTaskController.java Sun Jan 31 11:46:20 2010
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import junit.framework.TestCase;
+
+public class TestLinuxTaskController extends TestCase {
+ private static int INVALID_TASKCONTROLLER_PERMISSIONS = 24;
+ private static File testDir = new File(System.getProperty("test.build.data",
+ "/tmp"), TestLinuxTaskController.class.getName());
+ private static String taskControllerPath = System
+ .getProperty(ClusterWithLinuxTaskController.TASKCONTROLLER_PATH);
+
+ @Before
+ protected void setUp() throws Exception {
+ testDir.mkdirs();
+ }
+
+ @After
+ protected void tearDown() throws Exception {
+ FileUtil.fullyDelete(testDir);
+ }
+
+ public static class MyLinuxTaskController extends LinuxTaskController {
+ String taskControllerExePath = taskControllerPath + "/task-controller";
+
+ @Override
+ protected String getTaskControllerExecutablePath() {
+ return taskControllerExePath;
+ }
+ }
+
+ private void validateTaskControllerSetup(TaskController controller,
+ boolean shouldFail) throws IOException {
+ if (shouldFail) {
+ // task controller setup should fail validating permissions.
+ Throwable th = null;
+ try {
+ controller.setup();
+ } catch (IOException ie) {
+ th = ie;
+ }
+ assertNotNull("No exception during setup", th);
+ assertTrue("Exception message does not contain exit code"
+ + INVALID_TASKCONTROLLER_PERMISSIONS, th.getMessage().contains(
+ "with exit code " + INVALID_TASKCONTROLLER_PERMISSIONS));
+ } else {
+ controller.setup();
+ }
+
+ }
+
+ @Test
+ public void testTaskControllerGroup() throws Exception {
+ if (!ClusterWithLinuxTaskController.isTaskExecPathPassed()) {
+ return;
+ }
+ // cleanup configuration file.
+ ClusterWithLinuxTaskController
+ .getTaskControllerConfFile(taskControllerPath).delete();
+ Configuration conf = new Configuration();
+ // create local dirs and set in the conf.
+ File mapredLocal = new File(testDir, "mapred/local");
+ mapredLocal.mkdirs();
+ conf.set(MRConfig.LOCAL_DIR, mapredLocal.toString());
+
+ // setup task-controller without setting any group name
+ TaskController controller = new MyLinuxTaskController();
+ controller.setConf(conf);
+ validateTaskControllerSetup(controller, true);
+
+ // set an invalid group name for the task controller group
+ conf.set(TTConfig.TT_GROUP, "invalid");
+ // write the task-controller's conf file
+ ClusterWithLinuxTaskController.createTaskControllerConf(taskControllerPath,
+ conf);
+ validateTaskControllerSetup(controller, true);
+
+ // get the current ugi and set the task controller group owner in conf
+ Groups groups = new Groups(new Configuration());
+ String ttGroup = groups.getGroups(
+ UserGroupInformation.getCurrentUser().getUserName()).get(0);
+ conf.set(TTConfig.TT_GROUP, ttGroup);
+ // write the task-controller's conf file
+ ClusterWithLinuxTaskController.createTaskControllerConf(taskControllerPath,
+ conf);
+ validateTaskControllerSetup(controller, false);
+ }
+}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestLocalizationWithLinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestLocalizationWithLinuxTaskController.java?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestLocalizationWithLinuxTaskController.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestLocalizationWithLinuxTaskController.java Sun Jan 31 11:46:20 2010
@@ -62,9 +62,6 @@
taskController = new MyLinuxTaskController();
String path =
System.getProperty(ClusterWithLinuxTaskController.TASKCONTROLLER_PATH);
- configFile =
- ClusterWithLinuxTaskController.createTaskControllerConf(path,
- localDirs);
String execPath = path + "/task-controller";
((MyLinuxTaskController) taskController).setTaskControllerExe(execPath);
taskTrackerSpecialGroup = getFilePermissionAttrs(execPath)[2];
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTrackerDistributedCacheManagerWithLinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTrackerDistributedCacheManagerWithLinuxTaskController.java?rev=905008&r1=905007&r2=905008&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTrackerDistributedCacheManagerWithLinuxTaskController.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTrackerDistributedCacheManagerWithLinuxTaskController.java Sun Jan 31 11:46:20 2010
@@ -61,9 +61,6 @@
taskController = new MyLinuxTaskController();
String path =
System.getProperty(ClusterWithLinuxTaskController.TASKCONTROLLER_PATH);
- configFile =
- ClusterWithLinuxTaskController.createTaskControllerConf(path, conf
- .getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY));
String execPath = path + "/task-controller";
((MyLinuxTaskController)taskController).setTaskControllerExe(execPath);
taskController.setConf(conf);
@@ -79,8 +76,7 @@
String path =
System.getProperty(ClusterWithLinuxTaskController.TASKCONTROLLER_PATH);
configFile =
- ClusterWithLinuxTaskController.createTaskControllerConf(path, conf
- .getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY));
+ ClusterWithLinuxTaskController.createTaskControllerConf(path, conf);
}