You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:27:50 UTC
svn commit: r1077547 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
mapred/org/apache/hadoop/mapred/ test/org/apache/hadoop/mapred/
Author: omalley
Date: Fri Mar 4 04:27:49 2011
New Revision: 1077547
URL: http://svn.apache.org/viewvc?rev=1077547&view=rev
Log:
commit a1864dff889b9c2d34cf21d2c6208008185318d1
Author: Arun C Murthy <ac...@apache.org>
Date: Thu Jul 15 19:39:20 2010 -0700
MAPREDUCE-323. Fixed broken testcases in TestWebUIAuthorization and TestJobHistory. Contributed by Dick King.
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JSPUtil.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistory.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestWebUIAuthorization.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JSPUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JSPUtil.java?rev=1077547&r1=1077546&r2=1077547&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JSPUtil.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JSPUtil.java Fri Mar 4 04:27:49 2011
@@ -437,13 +437,7 @@ class JSPUtil {
}
static Path getJobConfFilePath(Path logFile) {
- String[] jobDetails = logFile.getName().split("_");
- String jobId = getJobID(logFile.getName());
- String jobUniqueString =
- jobDetails[0] + "_" + jobDetails[1] + "_" + jobId;
- Path logDir = logFile.getParent();
- Path jobFilePath = new Path(logDir, jobUniqueString + "_conf.xml");
- return jobFilePath;
+ return JobHistory.confPathFromLogFilePath(logFile);
}
/**
@@ -539,18 +533,15 @@ class JSPUtil {
}
static String getJobID(String historyFileName) {
- String[] jobDetails = historyFileName.split("_");
- return jobDetails[2] + "_" + jobDetails[3] + "_" + jobDetails[4];
+ return JobHistory.jobIdNameFromLogFileName(historyFileName);
}
static String getUserName(String historyFileName) {
- String[] jobDetails = historyFileName.split("_");
- return jobDetails[5];
+ return JobHistory.userNameFromLogFileName(historyFileName);
}
static String getJobName(String historyFileName) {
- String[] jobDetails = historyFileName.split("_");
- return jobDetails[6];
+ return JobHistory.jobNameFromLogFileName(historyFileName);
}
/**
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=1077547&r1=1077546&r2=1077547&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java Fri Mar 4 04:27:49 2011
@@ -351,7 +351,9 @@ public class JobHistory {
dateString = String.format
("%04d/%02d/%02d",
timestamp.get(Calendar.YEAR),
- timestamp.get(DEBUG_MODE ? Calendar.HOUR : Calendar.MONTH),
+ // months are 0-based in Calendar, but people will expect January
+ // to be month #1.
+ timestamp.get(DEBUG_MODE ? Calendar.HOUR : Calendar.MONTH) + 1,
timestamp.get(DEBUG_MODE ? Calendar.MINUTE : Calendar.DAY_OF_MONTH));
dateString = dateString.intern();
@@ -946,6 +948,29 @@ public class JobHistory {
throw e;
}
+
+ static Path confPathFromLogFilePath(Path logFile) {
+ String jobId = jobIdNameFromLogFileName(logFile.getName());
+
+ Path logDir = logFile.getParent();
+
+ return new Path(logDir, jobId + CONF_FILE_NAME_SUFFIX);
+ }
+
+ static String jobIdNameFromLogFileName(String logFileName) {
+ String[] jobDetails = logFileName.split("_");
+ return jobDetails[0] + "_" + jobDetails[1] + "_" + jobDetails[2];
+ }
+
+ static String userNameFromLogFileName(String logFileName) {
+ String[] jobDetails = logFileName.split("_");
+ return jobDetails[3];
+ }
+
+ static String jobNameFromLogFileName(String logFileName) {
+ String[] jobDetails = logFileName.split("_");
+ return jobDetails[4];
+ }
/**
* Helper class for logging or reading back events related to job start, finish or failure.
@@ -1007,6 +1032,7 @@ public class JobHistory {
return System.getProperty("hadoop.log.dir") + File.separator +
jobId + CONF_FILE_NAME_SUFFIX;
}
+
/**
* Helper function to encode the URL of the path of the job-history
@@ -1472,7 +1498,7 @@ public class JobHistory {
throws IOException {
FileSystem fs = null;
String userLogDir = null;
- String jobUniqueString = JOBTRACKER_UNIQUE_STRING + jobId;
+ String jobUniqueString = jobId.toString();
// Get the username and job name to be used in the actual log filename;
// sanity check them too
@@ -1492,7 +1518,7 @@ public class JobHistory {
//TODO this is a hack :(
// jobtracker-hostname_jobtracker-identifier_
String jtUniqueString = parts[0] + "_" + parts[1] + "_";
- jobUniqueString = jtUniqueString + jobId.toString();
+ jobUniqueString = jobId.toString();
}
} else {
logFileName =
@@ -2356,8 +2382,12 @@ public class JobHistory {
result.set(Calendar.YEAR, Integer.parseInt(year));
+ // months are 0-based in Calendar, but people will expect January
+ // to be month #1 . Therefore the number is bumped before we make the
+ // directory name and must be debumped to seek the time.
result.set(DEBUG_MODE ? Calendar.HOUR : Calendar.MONTH,
- Integer.parseInt(seg2));
+ Integer.parseInt(seg2) - 1);
+
result.set(DEBUG_MODE ? Calendar.MINUTE : Calendar.DAY_OF_MONTH,
Integer.parseInt(seg3));
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistory.java?rev=1077547&r1=1077546&r2=1077547&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistory.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistory.java Fri Mar 4 04:27:49 2011
@@ -434,18 +434,7 @@ public class TestJobHistory extends Test
* @param running whether the job is running or completed
*/
private static Path getPathForConf(Path path) {
- //TODO this is all a hack :(
- // jobtracker-hostname_jobtracker-identifier_
- String parts[] = path.getName().split("_");
- Path parent = path.getParent();
- Path ancestor = parent;
- for (int i = 0; i < 4; ++i) { // serial #, 3 laysers of date
- ancestor = ancestor.getParent();
- }
- String jobtrackerID = ancestor.getName();
- String id = parts[0] + "_" + parts[1] + "_" + parts[2];
- String jobUniqueString = jobtrackerID + id;
- return new Path(parent, jobUniqueString + "_conf.xml");
+ return JobHistory.confPathFromLogFilePath(path);
}
/**
@@ -1034,13 +1023,20 @@ public class TestJobHistory extends Test
}
// Returns the output path where user history log file is written to with
// default configuration setting for hadoop.job.history.user.location
- private static Path getLogLocationInOutputPath(String logFileName,
- JobConf conf) {
+ private static Path getLogLocationInOutputPath
+ (String logFileName, JobConf conf) {
JobConf jobConf = new JobConf(true);//default JobConf
FileOutputFormat.setOutputPath(jobConf,
FileOutputFormat.getOutputPath(conf));
- return JobHistory.JobInfo.getJobHistoryLogLocationForUser(
- logFileName, jobConf);
+
+ Path result = JobHistory.JobInfo.getJobHistoryLogLocationForUser
+ (logFileName, jobConf);
+ return result;
+ }
+
+ static private String coreLogLocation(String subdirLogLocation) {
+ return subdirLogLocation.substring
+ (subdirLogLocation.lastIndexOf(Path.SEPARATOR_CHAR) + 1);
}
/**
@@ -1056,12 +1052,15 @@ public class TestJobHistory extends Test
// User history log file location
Path logFile = JobHistory.JobInfo.getJobHistoryLogLocationForUser(
- logFileName, conf);
+ coreLogLocation(logFileName), conf);
+
if(logFile == null) {
// get the output path where history file is written to when
// hadoop.job.history.user.location is not set
- logFile = getLogLocationInOutputPath(logFileName, conf);
- }
+
+ logFile = getLogLocationInOutputPath(coreLogLocation(logFileName), conf);
+ }
+
FileSystem fileSys = null;
fileSys = logFile.getFileSystem(conf);
@@ -1101,9 +1100,6 @@ public class TestJobHistory extends Test
// hadoop.job.history.user.location as
// (1)null(default case), (2)"none", and (3)some user specified dir.
public void testJobHistoryUserLogLocation() throws IOException {
- // Disabled
- if (true) return;
-
MiniMRCluster mr = null;
try {
mr = new MiniMRCluster(2, "file:///", 3);
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestWebUIAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestWebUIAuthorization.java?rev=1077547&r1=1077546&r2=1077547&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestWebUIAuthorization.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestWebUIAuthorization.java Fri Mar 4 04:27:49 2011
@@ -293,10 +293,7 @@ public class TestWebUIAuthorization exte
MyGroupsProvider.mapping.put(mrAdminUser, Arrays.asList("group8"));
}
- public void testAuthorizationForJobHistoryPages() throws Exception {
- // Disabled
- if (true) return;
-
+ public void testAuthorizationForJobHistoryPages() throws Exception {
setupGroupsProvider();
Properties props = new Properties();
props.setProperty("hadoop.http.filter.initializers",