You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2011/01/07 03:13:25 UTC
svn commit: r1056163 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/JobTracker.java
src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java
Author: todd
Date: Fri Jan 7 02:13:25 2011
New Revision: 1056163
URL: http://svn.apache.org/viewvc?rev=1056163&view=rev
Log:
JobTracker should not try to remove mapred.system.dir during startup. Contributed by Todd Lipcon
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1056163&r1=1056162&r2=1056163&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Jan 7 02:13:25 2011
@@ -463,6 +463,9 @@ Release 0.22.0 - Unreleased
MAPREDUCE-2234. If Localizer can't create task log directory, it should fail
on the spot. (todd)
+ MAPREDUCE-2219. JobTracker should not try to remove mapred.system.dir
+ during startup. (todd)
+
Release 0.21.1 - Unreleased
NEW FEATURES
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1056163&r1=1056162&r2=1056163&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Jan 7 02:13:25 2011
@@ -1583,22 +1583,33 @@ public class JobTracker implements MRCon
break; // if there is something to recover else clean the sys dir
}
}
- LOG.info("Cleaning up the system directory");
- fs.delete(systemDir, true);
- if (FileSystem.mkdirs(fs, systemDir,
- new FsPermission(SYSTEM_DIR_PERMISSION))) {
+ if (!fs.exists(systemDir)) {
+ LOG.info("Creating the system directory");
+ if (FileSystem.mkdirs(fs, systemDir,
+ new FsPermission(SYSTEM_DIR_PERMISSION))) {
+ // success
+ break;
+ } else {
+ LOG.error("Mkdirs failed to create " + systemDir);
+ }
+ } else {
+ // It exists, just set permissions and clean the contents up
+ LOG.info("Cleaning up the system directory");
+ fs.setPermission(systemDir, new FsPermission(SYSTEM_DIR_PERMISSION));
+ deleteContents(fs, systemDir);
break;
}
- LOG.error("Mkdirs failed to create " + systemDir);
} catch (AccessControlException ace) {
- LOG.warn("Failed to operate on " + JTConfig.JT_SYSTEM_DIR + "(" + systemDir
+ LOG.warn("Failed to operate on " + JTConfig.JT_SYSTEM_DIR + "("
+ + systemDir.makeQualified(fs)
+ ") because of permissions.");
- LOG.warn("Manually delete the " + JTConfig.JT_SYSTEM_DIR + "(" + systemDir
- + ") and then start the JobTracker.");
+ LOG.warn("This directory should exist and be owned by the user '" +
+ UserGroupInformation.getCurrentUser() + "'");
LOG.warn("Bailing out ... ");
throw ace;
} catch (IOException ie) {
- LOG.info("problem cleaning system directory: " + systemDir, ie);
+ LOG.info("problem cleaning system directory: " +
+ systemDir.makeQualified(fs), ie);
}
Thread.sleep(FS_ACCESS_RETRY_PERIOD);
}
@@ -1634,6 +1645,18 @@ public class JobTracker implements MRCon
completedJobStatusStore = new CompletedJobStatusStore(conf, aclsManager);
}
+ /**
+ * Recursively delete the contents of a directory without deleting the
+ * directory itself.
+ */
+ private void deleteContents(FileSystem fs, Path dir) throws IOException {
+ for (FileStatus stat : fs.listStatus(dir)) {
+ if (!fs.delete(stat.getPath(), true)) {
+ throw new IOException("Unable to delete " + stat.getPath());
+ }
+ }
+ }
+
private static SimpleDateFormat getDateFormat() {
return new SimpleDateFormat("yyyyMMddHHmm");
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java?rev=1056163&r1=1056162&r2=1056163&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java Fri Jan 7 02:13:25 2011
@@ -44,8 +44,10 @@ public class TestMapredSystemDir extends
// mapred ugi
private static final UserGroupInformation MR_UGI =
TestMiniMRWithDFSWithDistinctUsers.createUGI("mr", false);
+ private static final FsPermission SYSTEM_DIR_PARENT_PERMISSION =
+ FsPermission.createImmutable((short) 0755); // rwxr-xr-x
private static final FsPermission SYSTEM_DIR_PERMISSION =
- FsPermission.createImmutable((short) 0733); // rwx-wx-wx
+ FsPermission.createImmutable((short) 0700); // rwx------
public void testGarbledMapredSystemDir() throws Exception {
Configuration conf = new Configuration();
@@ -60,17 +62,21 @@ public class TestMapredSystemDir extends
}
});
- // create Configs.SYSTEM_DIR's parent (the parent has to be given
- // permissions since the JT internally tries to delete the leaf of
- // the directory structure
- Path mapredSysDir =
- new Path(conf.get(JTConfig.JT_SYSTEM_DIR)).getParent();
+
+ // create Configs.SYSTEM_DIR's parent with restrictive permissions.
+ // So long as the JT has access to the system dir itself it should
+ // be able to start.
+ Path mapredSysDir = new Path(conf.get(JTConfig.JT_SYSTEM_DIR));
+ Path parentDir = mapredSysDir.getParent();
+ fs.mkdirs(parentDir);
+ fs.setPermission(parentDir,
+ new FsPermission(SYSTEM_DIR_PARENT_PERMISSION));
fs.mkdirs(mapredSysDir);
fs.setPermission(mapredSysDir, new FsPermission(SYSTEM_DIR_PERMISSION));
fs.setOwner(mapredSysDir, "mr", "mrgroup");
// start mr (i.e jobtracker)
- Configuration mrConf = new Configuration();
+ Configuration mrConf = new Configuration(conf);
mr = new MiniMRCluster(0, 0, 0, dfs.getFileSystem().getUri().toString(),
1, null, null, MR_UGI, new JobConf(mrConf));
JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
@@ -105,4 +111,4 @@ public class TestMapredSystemDir extends
if (mr != null) { mr.shutdown();}
}
}
-}
\ No newline at end of file
+}