You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ma...@apache.org on 2011/05/27 22:13:17 UTC
svn commit: r1128440 - in /hadoop/mapreduce/branches/MR-279: ./
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/
...
Author: mahadev
Date: Fri May 27 20:13:17 2011
New Revision: 1128440
URL: http://svn.apache.org/viewvc?rev=1128440&view=rev
Log:
Bugfix for using user staging directory for history files (siddharth seth via mahadev)
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHConfig.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YARNApplicationConstants.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Fri May 27 20:13:17 2011
@@ -3,6 +3,8 @@ Hadoop MapReduce Change Log
Trunk (unreleased changes)
MAPREDUCE-279
+
+ Bugfix for using user staging directory for history files (siddharth seth via mahadev)
MAPREDUCE-2536. Backporting changes to MR-279.
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java Fri May 27 20:13:17 2011
@@ -44,7 +44,6 @@ import org.apache.hadoop.mapreduce.v2.jo
import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
-import org.apache.hadoop.security.SecurityInfo;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -102,20 +101,27 @@ public class JobHistoryEventHandler exte
this.conf = conf;
- String logDir = JobHistoryUtils.getConfiguredHistoryLogDirPrefix(conf);
- String userLogDir = JobHistoryUtils.getHistoryLogDirForUser(conf);
- String doneDirPrefix = JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
- String userDoneDirPrefix = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
+ String logDir = null;
+ String doneDirPrefix = null;
+ String userDoneDirPrefix = null;
+ try {
+ logDir = JobHistoryUtils.getConfiguredHistoryLogDirPrefix(conf);
+ doneDirPrefix = JobHistoryUtils
+ .getConfiguredHistoryIntermediateDoneDirPrefix(conf);
+ userDoneDirPrefix = JobHistoryUtils
+ .getHistoryIntermediateDoneDirForUser(conf);
+ } catch (IOException e) {
+ LOG.error("Failed while getting the configured log directories", e);
+ throw new YarnException(e);
+ }
//Check for the existance of the log dir. Maybe create it.
- Path path = null;
try {
- path = FileSystem.get(conf).makeQualified(new Path(logDir));
- logDirFS = FileSystem.get(path.toUri(), conf);
- LOG.info("Maybe creating staging history logDir: [" + path + "]");
- mkdir(logDirFS, path, new FsPermission(JobHistoryUtils.HISTORY_STAGING_DIR_PERMISSIONS));
+ logDirPath = FileSystem.get(conf).makeQualified(new Path(logDir));
+ logDirFS = FileSystem.get(logDirPath.toUri(), conf);
+ mkdir(logDirFS, logDirPath, new FsPermission(JobHistoryUtils.HISTORY_STAGING_DIR_PERMISSIONS));
} catch (IOException e) {
- LOG.error("Failed while checking for/ceating history staging path: [" + path + "]", e);
+ LOG.error("Failed while checking for/ceating history staging path: [" + logDirPath + "]", e);
throw new YarnException(e);
}
@@ -141,15 +147,6 @@ public class JobHistoryEventHandler exte
LOG.error("Failed checking for the existance of history intermediate done directory: [" + doneDirPath + "]");
throw new YarnException(e);
}
-
- //Check/create staging directory.
- try {
- logDirPath = FileSystem.get(conf).makeQualified(new Path(userLogDir));
- mkdir(logDirFS, logDirPath, new FsPermission(JobHistoryUtils.HISTORY_STAGING_USER_DIR_PERMISSIONS));
- } catch (IOException e) {
- LOG.error("Error creating user staging history directory: [" + logDirPath + "]", e);
- throw new YarnException(e);
- }
//Check/create user directory under intermediate done dir.
try {
@@ -267,9 +264,8 @@ public class JobHistoryEventHandler exte
Configuration conf = getConfig();
long submitTime = (oldFi == null ? context.getClock().getTime() : oldFi.getJobIndexInfo().getSubmitTime());
-
- // String user = conf.get(MRJobConfig.USER_NAME, System.getProperty("user.name"));
+ //TODO Ideally this should be written out to the job dir (.staging/jobid/files - RecoveryService will need to be patched)
Path logFile = JobHistoryUtils.getStagingJobHistoryFile(logDirPath, jobId, startCount);
String user = conf.get(MRJobConfig.USER_NAME);
if (user == null) {
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java Fri May 27 20:13:17 2011
@@ -151,7 +151,7 @@ public class RecoveryService extends Com
private void parse() throws IOException {
// TODO: parse history file based on startCount
String jobName = TypeConverter.fromYarn(appID).toString();
- String jobhistoryDir = JobHistoryUtils.getHistoryLogDirForUser(getConfig());
+ String jobhistoryDir = JobHistoryUtils.getConfiguredHistoryLogDirPrefix(getConfig());
FSDataInputStream in = null;
Path historyFile = null;
Path histDirPath = FileContext.getFileContext(getConfig()).makeQualified(
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java Fri May 27 20:13:17 2011
@@ -129,7 +129,7 @@ public class MRApp extends MRAppMaster {
public Job submit(Configuration conf) throws Exception {
String user = conf.get(MRJobConfig.USER_NAME, "mapred");
conf.set(MRJobConfig.USER_NAME, user);
- conf.set(YARNApplicationConstants.APPS_HISTORY_STAGING_DIR_KEY, testAbsPath.toString());
+ conf.set(YARNApplicationConstants.APPS_STAGING_DIR_KEY, testAbsPath.toString());
conf.setBoolean(JHConfig.CREATE_HISTORY_INTERMEDIATE_BASE_DIR_KEY, true);
init(conf);
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHConfig.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHConfig.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHConfig.java Fri May 27 20:13:17 2011
@@ -11,10 +11,6 @@ public class JHConfig {
public static final String DEFAULT_HS_BIND_ADDRESS = "0.0.0.0:10020";
- /** Staging Dir for AppMaster **/
- public static final String HISTORY_STAGING_DIR_KEY =
- "yarn.historyfile.stagingDir";
-
/** Done Dir for for AppMaster **/
public static final String HISTORY_INTERMEDIATE_DONE_DIR_KEY =
"yarn.historyfile.intermediateDoneDir";
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java Fri May 27 20:13:17 2011
@@ -27,8 +27,6 @@ import java.util.concurrent.atomic.Atomi
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
@@ -36,10 +34,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
+import org.apache.hadoop.yarn.util.SecurityUtil;
public class JobHistoryUtils {
@@ -151,12 +150,10 @@ public class JobHistoryUtils {
* @param conf
* @return A string representation of the prefix.
*/
- public static String getConfiguredHistoryLogDirPrefix(Configuration conf) {
- String defaultLogDir = conf.get(
- //TODO Change this to staging directory
- YARNApplicationConstants.APPS_HISTORY_STAGING_DIR_KEY, "/tmp") + "/history/staging";
- String logDir = conf.get(JHConfig.HISTORY_STAGING_DIR_KEY,
- defaultLogDir);
+ public static String getConfiguredHistoryLogDirPrefix(Configuration conf) throws IOException {
+ String user = UserGroupInformation.getCurrentUser().getShortUserName();
+ Path path = SecurityUtil.getStagingAreaDir(conf, user);
+ String logDir = path.toString();
return logDir;
}
@@ -165,12 +162,12 @@ public class JobHistoryUtils {
* @param conf
* @return A string representation of the prefix.
*/
- public static String getConfiguredHistoryIntermediateDoneDirPrefix(Configuration conf) {
- String defaultDoneDir = conf.get(
- YARNApplicationConstants.APPS_HISTORY_STAGING_DIR_KEY, "/tmp") + "/history/done_intermediate";
+ public static String getConfiguredHistoryIntermediateDoneDirPrefix(Configuration conf) throws IOException {
String doneDirPrefix =
- conf.get(JHConfig.HISTORY_INTERMEDIATE_DONE_DIR_KEY,
- defaultDoneDir);
+ conf.get(JHConfig.HISTORY_INTERMEDIATE_DONE_DIR_KEY);
+ if (doneDirPrefix == null) {
+ doneDirPrefix = conf.get(YARNApplicationConstants.APPS_STAGING_DIR_KEY) + "/history/done_intermediate";
+ }
return doneDirPrefix;
}
@@ -179,33 +176,23 @@ public class JobHistoryUtils {
* @param conf
* @return
*/
- public static String getConfiguredHistoryServerDoneDirPrefix(Configuration conf) {
- String defaultDoneDir = conf.get(
- YARNApplicationConstants.APPS_HISTORY_STAGING_DIR_KEY, "/tmp") + "/history/done";
+ public static String getConfiguredHistoryServerDoneDirPrefix(Configuration conf) throws IOException {
String doneDirPrefix =
- conf.get(JHConfig.HISTORY_DONE_DIR_KEY,
- defaultDoneDir);
- return getCurrentDoneDir(doneDirPrefix);
+ conf.get(JHConfig.HISTORY_DONE_DIR_KEY);
+ if (doneDirPrefix == null) {
+ doneDirPrefix = conf.get(YARNApplicationConstants.APPS_STAGING_DIR_KEY) + "/history/done";
+ }
+ return doneDirPrefix;
}
/**
- * Gets the user directory for In progress history files.
- * @param conf
- * @return
- */
- public static String getHistoryLogDirForUser(Configuration conf) {
- return getConfiguredHistoryLogDirPrefix(conf) + File.separator
- + conf.get(MRJobConfig.USER_NAME);
- }
-
- /**
* Gets the user directory for intermediate done history files.
* @param conf
* @return
*/
- public static String getHistoryIntermediateDoneDirForUser(Configuration conf) {
+ public static String getHistoryIntermediateDoneDirForUser(Configuration conf) throws IOException {
return getConfiguredHistoryIntermediateDoneDirPrefix(conf) + File.separator
- + conf.get(MRJobConfig.USER_NAME);
+ + UserGroupInformation.getCurrentUser().getShortUserName();
}
public static boolean shouldCreateNonUserDirectory(Configuration conf) {
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java Fri May 27 20:13:17 2011
@@ -179,9 +179,12 @@ public class JobHistory extends Abstract
serialNumberFormat = ("%0"
+ (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");
-
- String doneDirPrefix = JobHistoryUtils
- .getConfiguredHistoryServerDoneDirPrefix(conf);
+ String doneDirPrefix = null;
+ try {
+ doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
+ } catch (IOException e) {
+ throw new YarnException(e);
+ }
try {
doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(
new Path(doneDirPrefix));
@@ -191,8 +194,13 @@ public class JobHistory extends Abstract
throw new YarnException("Error creating done directory: [" + doneDirPrefixPath + "]", e);
}
- String intermediateDoneDirPrefix = JobHistoryUtils
- .getConfiguredHistoryIntermediateDoneDirPrefix(conf);
+ String intermediateDoneDirPrefix = null;
+ try {
+ intermediateDoneDirPrefix = JobHistoryUtils
+ .getConfiguredHistoryIntermediateDoneDirPrefix(conf);
+ } catch (IOException e) {
+ throw new YarnException(e);
+ }
try {
intermediateDoneDirPath = FileContext.getFileContext(conf)
.makeQualified(new Path(intermediateDoneDirPrefix));
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Fri May 27 20:13:17 2011
@@ -30,6 +30,7 @@ import org.apache.hadoop.mapreduce.Clien
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
@@ -71,9 +72,6 @@ public class MiniMRYarnCluster extends M
conf.set(YARNApplicationConstants.APPS_STAGING_DIR_KEY, new File(
getTestWorkDir(),
"apps_staging_dir/${user.name}/").getAbsolutePath());
- conf.set(YARNApplicationConstants.APPS_HISTORY_STAGING_DIR_KEY, new File(
- getTestWorkDir(), "history_staging_dir/${user.name}/")
- .getAbsolutePath());
conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
// which shuffle doesn't happen
//configure the shuffle service in NM
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YARNApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YARNApplicationConstants.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YARNApplicationConstants.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YARNApplicationConstants.java Fri May 27 20:13:17 2011
@@ -38,7 +38,7 @@ public class YARNApplicationConstants {
public static final String APPS_STAGING_DIR_KEY = "yarn.apps.stagingDir";
- public static final String APPS_HISTORY_STAGING_DIR_KEY = "yarn.apps.history.stagingDir";
+// public static final String APPS_HISTORY_STAGING_DIR_KEY = "yarn.apps.history.stagingDir";
public static final String YARN_MAPREDUCE_APP_JAR_PATH =
"$YARN_HOME/modules/" + HADOOP_MAPREDUCE_CLIENT_APP_JAR_NAME;