You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by at...@apache.org on 2011/09/14 00:49:38 UTC
svn commit: r1170378 [3/12] - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
dev-support/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-clie...
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java Tue Sep 13 22:49:27 2011
@@ -279,7 +279,8 @@ public class CompletedJob implements org
@Override
public boolean isUber() {
- throw new YarnException("Not yet implemented!");
+ LOG.warn("isUber is not yet implemented");
+ return false;
}
@Override
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java Tue Sep 13 22:49:27 2011
@@ -29,6 +29,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -82,12 +83,23 @@ public class CompletedTaskAttempt implem
@Override
public ContainerId getAssignedContainerID() {
- //TODO ContainerId needs to be part of some historyEvent to be able to render the log directory.
- ContainerId containerId = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ContainerId.class);
+ //TODO ContainerId needs to be part of some historyEvent to be able to
+ //render the log directory.
+ ContainerId containerId =
+ RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
+ ContainerId.class);
containerId.setId(-1);
- containerId.setAppId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class));
- containerId.getAppId().setId(-1);
- containerId.getAppId().setClusterTimestamp(-1);
+ ApplicationAttemptId applicationAttemptId =
+ RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
+ ApplicationAttemptId.class);
+ applicationAttemptId.setAttemptId(-1);
+ ApplicationId applicationId =
+ RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
+ ApplicationId.class);
+ applicationId.setClusterTimestamp(-1);
+ applicationId.setId(-1);
+ applicationAttemptId.setApplicationId(applicationId);
+ containerId.setApplicationAttemptId(applicationAttemptId);
return containerId;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java Tue Sep 13 22:49:27 2011
@@ -62,8 +62,8 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.hs.webapp.HSWebApp;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
+import org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebApp;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.security.client.ClientHSSecurityInfo;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityInfo;
@@ -107,8 +107,8 @@ public class HistoryClientService extend
YarnConfiguration.YARN_SECURITY_INFO,
ClientHSSecurityInfo.class, SecurityInfo.class);
initializeWebApp(getConfig());
- String serviceAddr = conf.get(JHConfig.HS_BIND_ADDRESS,
- JHConfig.DEFAULT_HS_BIND_ADDRESS);
+ String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
InetSocketAddress address = NetUtils.createSocketAddr(serviceAddr);
InetAddress hostNameResolved = null;
try {
@@ -120,8 +120,8 @@ public class HistoryClientService extend
server =
rpc.getServer(MRClientProtocol.class, protocolHandler, address,
conf, null,
- conf.getInt(JHConfig.HS_CLIENT_THREADS,
- JHConfig.DEFAULT_HS_CLIENT_THREADS));
+ conf.getInt(JHAdminConfig.MR_HISTORY_CLIENT_THREAD_COUNT,
+ JHAdminConfig.DEFAULT_MR_HISTORY_CLIENT_THREAD_COUNT));
server.start();
this.bindAddress =
NetUtils.createSocketAddr(hostNameResolved.getHostAddress()
@@ -132,9 +132,9 @@ public class HistoryClientService extend
}
private void initializeWebApp(Configuration conf) {
- webApp = new HSWebApp(history);
- String bindAddress = conf.get(JHConfig.HS_WEBAPP_BIND_ADDRESS,
- JHConfig.DEFAULT_HS_WEBAPP_BIND_ADDRESS);
+ webApp = new HsWebApp(history);
+ String bindAddress = conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS);
WebApps.$for("yarn", this).at(bindAddress).start(webApp);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java Tue Sep 13 22:49:27 2011
@@ -57,7 +57,7 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.yarn.Clock;
@@ -184,7 +184,7 @@ public class JobHistory extends Abstract
this.appAttemptID = RecordFactoryProvider.getRecordFactory(conf)
.newRecordInstance(ApplicationAttemptId.class);
- debugMode = conf.getBoolean(JHConfig.HISTORY_DEBUG_MODE_KEY, false);
+ debugMode = conf.getBoolean(JHAdminConfig.MR_HISTORY_DEBUG_MODE, false);
serialNumberLowDigits = debugMode ? 1 : 3;
serialNumberFormat = ("%0"
+ (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");
@@ -216,15 +216,15 @@ public class JobHistory extends Abstract
- jobListCacheSize = conf.getInt(JHConfig.HISTORY_SERVER_JOBLIST_CACHE_SIZE_KEY, DEFAULT_JOBLIST_CACHE_SIZE);
- loadedJobCacheSize = conf.getInt(JHConfig.HISTORY_SERVER_LOADED_JOB_CACHE_SIZE_KEY, DEFAULT_LOADEDJOB_CACHE_SIZE);
- dateStringCacheSize = conf.getInt(JHConfig.HISTORY_SERVER_DATESTRING_CACHE_SIZE_KEY, DEFAULT_DATESTRING_CACHE_SIZE);
+ jobListCacheSize = conf.getInt(JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE, DEFAULT_JOBLIST_CACHE_SIZE);
+ loadedJobCacheSize = conf.getInt(JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE, DEFAULT_LOADEDJOB_CACHE_SIZE);
+ dateStringCacheSize = conf.getInt(JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE, DEFAULT_DATESTRING_CACHE_SIZE);
moveThreadInterval =
- conf.getLong(JHConfig.HISTORY_SERVER_MOVE_THREAD_INTERVAL,
+ conf.getLong(JHAdminConfig.MR_HISTORY_MOVE_INTERVAL_MS,
DEFAULT_MOVE_THREAD_INTERVAL);
- numMoveThreads = conf.getInt(JHConfig.HISTORY_SERVER_NUM_MOVE_THREADS, DEFAULT_MOVE_THREAD_COUNT);
+ numMoveThreads = conf.getInt(JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT, DEFAULT_MOVE_THREAD_COUNT);
try {
- initExisting();
+ initExisting();
} catch (IOException e) {
throw new YarnException("Failed to intialize existing directories", e);
}
@@ -260,12 +260,12 @@ public class JobHistory extends Abstract
moveIntermediateToDoneThread.start();
//Start historyCleaner
- boolean startCleanerService = conf.getBoolean(JHConfig.RUN_HISTORY_CLEANER_KEY, true);
+ boolean startCleanerService = conf.getBoolean(JHAdminConfig.MR_HISTORY_CLEANER_ENABLE, true);
if (startCleanerService) {
- long maxAgeOfHistoryFiles = conf.getLong(JHConfig.HISTORY_MAXAGE,
+ long maxAgeOfHistoryFiles = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
DEFAULT_HISTORY_MAX_AGE);
cleanerScheduledExecutor = new ScheduledThreadPoolExecutor(1);
- long runInterval = conf.getLong(JHConfig.HISTORY_CLEANER_RUN_INTERVAL,
+ long runInterval = conf.getLong(JHAdminConfig.MR_HISTORY_CLEANER_INTERVAL_MS,
DEFAULT_RUN_INTERVAL);
cleanerScheduledExecutor
.scheduleAtFixedRate(new HistoryCleaner(maxAgeOfHistoryFiles),
@@ -319,6 +319,7 @@ public class JobHistory extends Abstract
*/
@SuppressWarnings("unchecked")
private void initExisting() throws IOException {
+ LOG.info("Initializing Existing Jobs...");
List<FileStatus> timestampedDirList = findTimestampedDirectories();
Collections.sort(timestampedDirList);
for (FileStatus fs : timestampedDirList) {
@@ -350,6 +351,9 @@ public class JobHistory extends Abstract
}
private void addDirectoryToSerialNumberIndex(Path serialDirPath) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Adding "+serialDirPath+" to serial index");
+ }
String serialPart = serialDirPath.getName();
String timestampPart = JobHistoryUtils.getTimestampPartFromPath(serialDirPath.toString());
if (timestampPart == null) {
@@ -374,9 +378,15 @@ public class JobHistory extends Abstract
}
private void addDirectoryToJobListCache(Path path) throws IOException {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Adding "+path+" to job list cache.");
+ }
List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path,
doneDirFc);
for (FileStatus fs : historyFileList) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Adding in history for "+fs.getPath());
+ }
JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
.getName());
String confFileName = JobHistoryUtils
@@ -423,6 +433,9 @@ public class JobHistory extends Abstract
* Adds an entry to the job list cache. Maintains the size.
*/
private void addToJobListCache(JobId jobId, MetaInfo metaInfo) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Adding "+jobId+" to job list cache with "+metaInfo.getJobIndexInfo());
+ }
jobListCache.put(jobId, metaInfo);
if (jobListCache.size() > jobListCacheSize) {
jobListCache.remove(jobListCache.firstKey());
@@ -432,7 +445,10 @@ public class JobHistory extends Abstract
/**
* Adds an entry to the loaded job cache. Maintains the size.
*/
- private void addToLoadedJobCache(Job job) {
+ private void addToLoadedJobCache(Job job) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Adding "+job.getID()+" to loaded job cache");
+ }
loadedJobCache.put(job.getID(), job);
if (loadedJobCache.size() > loadedJobCacheSize ) {
loadedJobCache.remove(loadedJobCache.firstKey());
@@ -967,6 +983,9 @@ public class JobHistory extends Abstract
@Override
public synchronized Job getJob(JobId jobId) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Looking for Job "+jobId);
+ }
Job job = null;
try {
job = findJob(jobId);
@@ -979,7 +998,9 @@ public class JobHistory extends Abstract
@Override
public Map<JobId, Job> getAllJobs(ApplicationId appID) {
- LOG.info("Called getAllJobs(AppId): " + appID);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Called getAllJobs(AppId): " + appID);
+ }
// currently there is 1 to 1 mapping between app and job id
org.apache.hadoop.mapreduce.JobID oldJobID = TypeConverter.fromYarn(appID);
Map<JobId, Job> jobs = new HashMap<JobId, Job>();
@@ -1002,12 +1023,9 @@ public class JobHistory extends Abstract
* This does involve a DFS oepration of scanning the intermediate directory.
*/
public Map<JobId, Job> getAllJobs() {
+ LOG.debug("Called getAllJobs()");
return getAllJobsInternal();
- }
-
-
-
-
+ }
static class MetaInfo {
private Path historyFile;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Tue Sep 13 22:49:27 2011
@@ -24,7 +24,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnException;
@@ -68,8 +68,8 @@ public class JobHistoryServer extends Co
}
protected void doSecureLogin(Configuration conf) throws IOException {
- SecurityUtil.login(conf, JHConfig.HS_KEYTAB_KEY,
- JHConfig.HS_SERVER_PRINCIPAL_KEY);
+ SecurityUtil.login(conf, JHAdminConfig.MR_HISTORY_KEYTAB,
+ JHAdminConfig.MR_HISTORY_PRINCIPAL);
}
public static void main(String[] args) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java Tue Sep 13 22:49:27 2011
@@ -46,6 +46,8 @@ public class PartialJob implements org.a
this.jobIndexInfo = jobIndexInfo;
this.jobId = jobId;
jobReport = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobReport.class);
+ jobReport.setStartTime(jobIndexInfo.getSubmitTime());
+ jobReport.setFinishTime(jobIndexInfo.getFinishTime());
}
@Override
@@ -142,7 +144,7 @@ public class PartialJob implements org.a
@Override
public String getUserName() {
- return null;
+ return jobIndexInfo.getUser();
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsController.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsController.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsController.java Tue Sep 13 22:49:27 2011
@@ -21,41 +21,123 @@ package org.apache.hadoop.mapreduce.v2.h
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.v2.app.webapp.App;
import org.apache.hadoop.mapreduce.v2.app.webapp.AppController;
+import org.apache.hadoop.yarn.webapp.View;
import com.google.inject.Inject;
+/**
+ * This class renders the various pages that the History Server WebApp supports
+ */
public class HsController extends AppController {
@Inject HsController(App app, Configuration conf, RequestContext ctx) {
super(app, conf, ctx, "History");
}
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.webapp.AppController#index()
+ */
@Override
public void index() {
- // TODO Auto-generated method stub
setTitle("JobHistory");
}
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.webapp.AppController#jobPage()
+ */
+ @Override
+ protected Class<? extends View> jobPage() {
+ return HsJobPage.class;
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.webapp.AppController#countersPage()
+ */
+ @Override
+ protected Class<? extends View> countersPage() {
+ return HsCountersPage.class;
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.webapp.AppController#tasksPage()
+ */
+ @Override
+ protected Class<? extends View> tasksPage() {
+ return HsTasksPage.class;
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.webapp.AppController#taskPage()
+ */
+ @Override
+ protected Class<? extends View> taskPage() {
+ return HsTaskPage.class;
+ }
// Need all of these methods here also as Guice doesn't look into parent
// classes.
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.webapp.AppController#job()
+ */
+ @Override
public void job() {
super.job();
}
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.webapp.AppController#jobCounters()
+ */
+ @Override
public void jobCounters() {
super.jobCounters();
}
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.webapp.AppController#tasks()
+ */
+ @Override
public void tasks() {
super.tasks();
}
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.webapp.AppController#task()
+ */
+ @Override
public void task() {
super.task();
}
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.webapp.AppController#attempts()
+ */
@Override
public void attempts() {
super.attempts();
}
+
+ /**
+ * @return the page about the current server.
+ */
+ protected Class<? extends View> aboutPage() {
+ return HsAboutPage.class;
+ }
+
+ /**
+ * Render a page about the current server.
+ */
+ public void about() {
+ render(aboutPage());
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java Tue Sep 13 22:49:27 2011
@@ -18,24 +18,42 @@
package org.apache.hadoop.mapreduce.v2.hs.webapp;
-import org.apache.hadoop.mapreduce.v2.app.webapp.JobsBlock;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.THEMESWITCHER_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.postInitID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+/**
+ * A view that should be used as the base class for all history server pages.
+ */
public class HsView extends TwoColumnLayout {
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.yarn.webapp.view.TwoColumnLayout#preHead(org.apache.hadoop.yarn.webapp.hamlet.Hamlet.HTML)
+ */
@Override protected void preHead(Page.HTML<_> html) {
commonPreHead(html);
set(DATATABLES_ID, "jobs");
set(initID(DATATABLES, "jobs"), jobsTableInit());
+ set(postInitID(DATATABLES, "jobs"), jobsPostTableInit());
setTableStyles(html, "jobs");
}
+ /**
+ * The prehead that should be common to all subclasses.
+ * @param html used to render.
+ */
protected void commonPreHead(Page.HTML<_> html) {
- //html.meta_http("refresh", "10");
set(ACCORDION_ID, "nav");
- set(initID(ACCORDION, "nav"), "{autoHeight:false, active:1}");
+ set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
set(THEMESWITCHER_ID, "themeswitcher");
}
@@ -43,22 +61,63 @@ public class HsView extends TwoColumnLay
* (non-Javadoc)
* @see org.apache.hadoop.yarn.webapp.view.TwoColumnLayout#nav()
*/
-
@Override
protected Class<? extends SubView> nav() {
- return org.apache.hadoop.mapreduce.v2.app.webapp.NavBlock.class;
+ return HsNavBlock.class;
}
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.yarn.webapp.view.TwoColumnLayout#content()
+ */
@Override
protected Class<? extends SubView> content() {
- return JobsBlock.class;
+ return HsJobsBlock.class;
}
-
+
+ //TODO We need a way to move all of the javascript/CSS that is for a subview
+ // into that subview.
+ /**
+ * @return The end of a javascript map that is the jquery datatable
+ * configuration for the jobs table. the Jobs table is assumed to be
+ * rendered by the class returned from {@link #content()}
+ */
private String jobsTableInit() {
return tableInit().
- append(",aoColumns:[{sType:'title-numeric'},").
- append("null,null,{sType:'title-numeric', bSearchable:false},null,").
- append("null,{sType:'title-numeric',bSearchable:false}, null, null]}").
+ append(",aoColumnDefs:[").
+ append("{'sType':'numeric', 'bSearchable': false, 'aTargets': [ 6 ] }").
+ append(",{'sType':'numeric', 'bSearchable': false, 'aTargets': [ 7 ] }").
+ append(",{'sType':'numeric', 'bSearchable': false, 'aTargets': [ 8 ] }").
+ append(",{'sType':'numeric', 'bSearchable': false, 'aTargets': [ 9 ] }").
+ append("]}").
toString();
}
+
+ /**
+ * @return javascript to add into the jquery block after the table has
+ * been initialized. This code adds in per field filtering.
+ */
+ private String jobsPostTableInit() {
+ return "var asInitVals = new Array();\n" +
+ "$('tfoot input').keyup( function () \n{"+
+ " jobsDataTable.fnFilter( this.value, $('tfoot input').index(this) );\n"+
+ "} );\n"+
+ "$('tfoot input').each( function (i) {\n"+
+ " asInitVals[i] = this.value;\n"+
+ "} );\n"+
+ "$('tfoot input').focus( function () {\n"+
+ " if ( this.className == 'search_init' )\n"+
+ " {\n"+
+ " this.className = '';\n"+
+ " this.value = '';\n"+
+ " }\n"+
+ "} );\n"+
+ "$('tfoot input').blur( function (i) {\n"+
+ " if ( this.value == '' )\n"+
+ " {\n"+
+ " this.className = 'search_init';\n"+
+ " this.value = asInitVals[$('tfoot input').index(this)];\n"+
+ " }\n"+
+ "} );\n";
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java Tue Sep 13 22:49:27 2011
@@ -1,27 +1,126 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.mapreduce.v2.hs.webapp;
-import org.apache.hadoop.yarn.webapp.WebApps;
+import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.APP_ID;
+import static org.junit.Assert.assertEquals;
+
+import java.util.Map;
+
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.MockJobs;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.junit.Test;
+
+import com.google.inject.Injector;
public class TestHSWebApp {
- public static void main(String[] args) {
- WebApps.$for("yarn").at(19888).start().joinThread();
+
+ static class TestAppContext implements AppContext {
+ final ApplicationAttemptId appAttemptID;
+ final ApplicationId appID;
+ final String user = MockJobs.newUserName();
+ final Map<JobId, Job> jobs;
+ final long startTime = System.currentTimeMillis();
+
+ TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
+ appID = MockJobs.newAppID(appid);
+ appAttemptID = MockJobs.newAppAttemptID(appID, 0);
+ jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
+ }
+
+ TestAppContext() {
+ this(0, 1, 1, 1);
+ }
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptID;
+ }
+
+ @Override
+ public ApplicationId getApplicationID() {
+ return appID;
+ }
+
+ @Override
+ public CharSequence getUser() {
+ return user;
+ }
+
+ @Override
+ public Job getJob(JobId jobID) {
+ return jobs.get(jobID);
+ }
+
+ @Override
+ public Map<JobId, Job> getAllJobs() {
+ return jobs; // OK
+ }
+
+ @Override
+ public EventHandler getEventHandler() {
+ return null;
+ }
+
+ @Override
+ public Clock getClock() {
+ return null;
+ }
+
+ @Override
+ public String getApplicationName() {
+ return "TestApp";
+ }
+
+ @Override
+ public long getStartTime() {
+ return startTime;
+ }
+ }
+
+ @Test public void testAppControllerIndex() {
+ TestAppContext ctx = new TestAppContext();
+ Injector injector = WebAppTests.createMockInjector(AppContext.class, ctx);
+ HsController controller = injector.getInstance(HsController.class);
+ controller.index();
+ assertEquals(ctx.appID.toString(), controller.get(APP_ID,""));
+ }
+
+ @Test public void testJobView() {
+ WebAppTests.testPage(HsJobPage.class, AppContext.class, new TestAppContext());
+ }
+
+ @Test public void testTasksView() {
+ WebAppTests.testPage(HsTasksPage.class, AppContext.class,
+ new TestAppContext());
+ }
+
+ @Test public void testTaskView() {
+ WebAppTests.testPage(HsTaskPage.class, AppContext.class,
+ new TestAppContext());
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java Tue Sep 13 22:49:27 2011
@@ -27,7 +27,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityInfo;
import org.apache.hadoop.yarn.YarnException;
@@ -72,8 +72,8 @@ public class ClientCache {
private MRClientProtocol instantiateHistoryProxy()
throws IOException {
- String serviceAddr = conf.get(JHConfig.HS_BIND_ADDRESS,
- JHConfig.DEFAULT_HS_BIND_ADDRESS);
+ String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
LOG.info("Connecting to HistoryServer at: " + serviceAddr);
Configuration myConf = new Configuration(conf);
//TODO This should ideally be using it's own class (instead of ClientRMSecurityInfo)
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java Tue Sep 13 22:49:27 2011
@@ -19,6 +19,7 @@
package org.apache.hadoop.mapred;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.security.PrivilegedAction;
import java.util.HashMap;
@@ -149,8 +150,7 @@ class ClientServiceDelegate {
LOG.info("Connecting to " + serviceAddr);
instantiateAMProxy(serviceAddr);
return realProxy;
- } catch (Exception e) {
- //possibly
+ } catch (IOException e) {
//possibly the AM has crashed
//there may be some time before AM is restarted
//keep retrying by getting the address from RM
@@ -159,8 +159,13 @@ class ClientServiceDelegate {
try {
Thread.sleep(2000);
} catch (InterruptedException e1) {
+ LOG.warn("getProxy() call interruped", e1);
+ throw new YarnException(e1);
}
application = rm.getApplicationReport(appId);
+ } catch (InterruptedException e) {
+ LOG.warn("getProxy() call interruped", e);
+ throw new YarnException(e);
}
}
@@ -193,7 +198,7 @@ class ClientServiceDelegate {
//succeeded.
if (application.getState() == ApplicationState.SUCCEEDED) {
LOG.info("Application state is completed. " +
- "Redirecting to job history server " + serviceAddr);
+ "Redirecting to job history server");
realProxy = historyServerProxy;
}
return realProxy;
@@ -234,8 +239,14 @@ class ClientServiceDelegate {
LOG.warn("Exception thrown by remote end.");
LOG.warn(RPCUtil.toString(yre));
throw yre;
+ } catch (InvocationTargetException e) {
+ //TODO Finite # of errors before giving up?
+ LOG.info("Failed to contact AM/History for job " + jobId
+ + " Will retry..", e.getTargetException());
+ forceRefresh = true;
} catch (Exception e) {
- LOG.info("Failed to contact AM for job " + jobId + " Will retry..");
+ LOG.info("Failed to contact AM/History for job " + jobId
+ + " Will retry..", e);
LOG.debug("Failing to contact application master", e);
forceRefresh = true;
}
@@ -302,10 +313,13 @@ class ClientServiceDelegate {
return TypeConverter.fromYarn(report, jobFile, "");
}
- org.apache.hadoop.mapreduce.TaskReport[] getTaskReports(JobID jobID, TaskType taskType)
+ org.apache.hadoop.mapreduce.TaskReport[] getTaskReports(JobID oldJobID, TaskType taskType)
throws YarnRemoteException, YarnRemoteException {
- org.apache.hadoop.mapreduce.v2.api.records.JobId nJobID = TypeConverter.toYarn(jobID);
+ org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
+ TypeConverter.toYarn(oldJobID);
GetTaskReportsRequest request = recordFactory.newRecordInstance(GetTaskReportsRequest.class);
+ request.setJobId(jobId);
+ request.setTaskType(TypeConverter.toYarn(taskType));
List<org.apache.hadoop.mapreduce.v2.api.records.TaskReport> taskReports =
((GetTaskReportsResponse) invoke("getTaskReports", GetTaskReportsRequest.class,
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Tue Sep 13 22:49:27 2011
@@ -26,7 +26,6 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@ -59,7 +58,6 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationMaster;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -86,8 +84,8 @@ public class ResourceMgrDelegate {
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress rmAddress =
NetUtils.createSocketAddr(conf.get(
- YarnConfiguration.APPSMANAGER_ADDRESS,
- YarnConfiguration.DEFAULT_APPSMANAGER_BIND_ADDRESS));
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS));
LOG.info("Connecting to ResourceManager at " + rmAddress);
Configuration appsManagerServerConf = new Configuration(this.conf);
appsManagerServerConf.setClass(
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Tue Sep 13 22:49:27 2011
@@ -39,7 +39,7 @@ import org.apache.hadoop.fs.UnsupportedF
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.ClusterMetrics;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobContext;
@@ -55,11 +55,9 @@ import org.apache.hadoop.mapreduce.TaskR
import org.apache.hadoop.mapreduce.TaskTrackerInfo;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.mapreduce.v2.ClientConstants;
import org.apache.hadoop.mapreduce.v2.MRConstants;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
@@ -82,7 +80,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -93,10 +90,6 @@ public class YARNRunner implements Clien
private static final Log LOG = LogFactory.getLog(YARNRunner.class);
- public static final String YARN_AM_VMEM_MB =
- "yarn.am.mapreduce.resource.mb";
- private static final int DEFAULT_YARN_AM_VMEM_MB = 2048;
-
private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
private ResourceMgrDelegate resMgrDelegate;
private ClientCache clientCache;
@@ -108,10 +101,20 @@ public class YARNRunner implements Clien
* yarn
* @param conf the configuration object for the client
*/
- public YARNRunner(Configuration conf) {
- this.conf = new YarnConfiguration(conf);
+ public YARNRunner(YarnConfiguration conf) {
+ this(conf, new ResourceMgrDelegate(conf));
+ }
+
+ /**
+ * Similar to {@link #YARNRunner(YarnConfiguration)} but allowing injecting
+ * {@link ResourceMgrDelegate}. Enables mocking and testing.
+ * @param conf the configuration object for the client
+ * @param resMgrDelegate the resourcemanager client handle.
+ */
+ public YARNRunner(YarnConfiguration conf, ResourceMgrDelegate resMgrDelegate) {
+ this.conf = conf;
try {
- this.resMgrDelegate = new ResourceMgrDelegate(this.conf);
+ this.resMgrDelegate = resMgrDelegate;
this.clientCache = new ClientCache(this.conf,
resMgrDelegate);
this.defaultFileContext = FileContext.getFileContext(this.conf);
@@ -119,7 +122,7 @@ public class YARNRunner implements Clien
throw new RuntimeException("Error in instantiating YarnClient", ufe);
}
}
-
+
@Override
public void cancelDelegationToken(Token<DelegationTokenIdentifier> arg0)
throws IOException, InterruptedException {
@@ -245,9 +248,11 @@ public class YARNRunner implements Clien
ApplicationReport appMaster = resMgrDelegate
.getApplicationReport(applicationId);
+ String diagnostics = (appMaster == null ? "application report is null" : appMaster.getDiagnostics());
if (appMaster == null || appMaster.getState() == ApplicationState.FAILED
|| appMaster.getState() == ApplicationState.KILLED) {
- throw RPCUtil.getRemoteException("failed to run job");
+ throw new IOException("Failed to run job : " +
+ diagnostics);
}
return clientCache.getClient(jobId).getJobStatus(jobId);
}
@@ -265,7 +270,7 @@ public class YARNRunner implements Clien
return rsrc;
}
- private ApplicationSubmissionContext createApplicationSubmissionContext(
+ public ApplicationSubmissionContext createApplicationSubmissionContext(
Configuration jobConf,
String jobSubmitDir, Credentials ts) throws IOException {
ApplicationSubmissionContext appContext =
@@ -273,7 +278,8 @@ public class YARNRunner implements Clien
ApplicationId applicationId = resMgrDelegate.getApplicationId();
appContext.setApplicationId(applicationId);
Resource capability = recordFactory.newRecordInstance(Resource.class);
- capability.setMemory(conf.getInt(YARN_AM_VMEM_MB, DEFAULT_YARN_AM_VMEM_MB));
+ capability.setMemory(conf.getInt(MRJobConfig.MR_AM_VMEM_MB,
+ MRJobConfig.DEFAULT_MR_AM_VMEM_MB));
LOG.info("AppMaster capability = " + capability);
appContext.setMasterCapability(capability);
@@ -334,15 +340,14 @@ public class YARNRunner implements Clien
Vector<CharSequence> vargs = new Vector<CharSequence>(8);
vargs.add(javaHome + "/bin/java");
vargs.add("-Dhadoop.root.logger="
- + conf.get(ClientConstants.MR_APPMASTER_LOG_OPTS,
- ClientConstants.DEFAULT_MR_APPMASTER_LOG_OPTS) + ",console");
+ + conf.get(MRJobConfig.MR_AM_LOG_OPTS,
+ MRJobConfig.DEFAULT_MR_AM_LOG_OPTS) + ",console");
- vargs.add(conf.get(ClientConstants.MR_APPMASTER_COMMAND_OPTS,
- ClientConstants.DEFAULT_MR_APPMASTER_COMMAND_OPTS));
+ vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
+ MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS));
// Add { job jar, MR app jar } to classpath.
Map<String, String> environment = new HashMap<String, String>();
-// appContext.environment = new HashMap<CharSequence, CharSequence>();
MRApps.setInitialClasspath(environment);
MRApps.addToClassPath(environment, MRConstants.JOB_JAR);
MRApps.addToClassPath(environment,
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java Tue Sep 13 22:49:27 2011
@@ -25,13 +25,14 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
public class YarnClientProtocolProvider extends ClientProtocolProvider {
@Override
public ClientProtocol create(Configuration conf) throws IOException {
if ("yarn".equals(conf.get(MRConfig.FRAMEWORK_NAME))) {
- return new YARNRunner(conf);
+ return new YARNRunner(new YarnConfiguration(conf));
}
return null;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java Tue Sep 13 22:49:27 2011
@@ -21,7 +21,6 @@ package org.apache.hadoop.mapred;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
-import java.util.ArrayList;
import java.util.Iterator;
import junit.framework.Assert;
@@ -64,8 +63,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.YarnException;
@@ -122,8 +120,8 @@ public class TestClientRedirect {
Configuration conf = new YarnConfiguration();
conf.set(MRConfig.FRAMEWORK_NAME, "yarn");
- conf.set(YarnConfiguration.APPSMANAGER_ADDRESS, RMADDRESS);
- conf.set(JHConfig.HS_BIND_ADDRESS, HSHOSTADDRESS);
+ conf.set(YarnConfiguration.RM_ADDRESS, RMADDRESS);
+ conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, HSHOSTADDRESS);
RMService rmService = new RMService("test");
rmService.init(conf);
rmService.start();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Tue Sep 13 22:49:27 2011
@@ -29,11 +29,10 @@ import org.apache.hadoop.mapreduce.MRCon
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.service.Service;
@@ -62,19 +61,21 @@ public class MiniMRYarnCluster extends M
public void init(Configuration conf) {
conf.set(MRConfig.FRAMEWORK_NAME, "yarn");
conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
- conf.set(MRConstants.APPS_STAGING_DIR_KEY, new File(getTestWorkDir(),
+ conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(),
"apps_staging_dir/${user.name}/").getAbsolutePath());
conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
// which shuffle doesn't happen
//configure the shuffle service in NM
- conf.setStrings(AuxServices.AUX_SERVICES,
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
- conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT,
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
Service.class);
+
// Non-standard shuffle port
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 8083);
- conf.setClass(NMConfig.NM_CONTAINER_EXECUTOR_CLASS,
+
+ conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
DefaultContainerExecutor.class, ContainerExecutor.class);
// TestMRJobs is for testing non-uberized operation only; see TestUberAM
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java Tue Sep 13 22:49:27 2011
@@ -68,9 +68,6 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.YarnServerConfig;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
-import org.apache.hadoop.yarn.server.resourcemanager.RMConfig;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -335,11 +332,11 @@ public class TestMRJobs {
mrCluster.getConfig().set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
- mrCluster.getConfig().set(RMConfig.RM_KEYTAB, "/etc/krb5.keytab");
- mrCluster.getConfig().set(NMConfig.NM_KEYTAB, "/etc/krb5.keytab");
- mrCluster.getConfig().set(YarnConfiguration.RM_SERVER_PRINCIPAL_KEY,
+ mrCluster.getConfig().set(YarnConfiguration.RM_KEYTAB, "/etc/krb5.keytab");
+ mrCluster.getConfig().set(YarnConfiguration.NM_KEYTAB, "/etc/krb5.keytab");
+ mrCluster.getConfig().set(YarnConfiguration.RM_PRINCIPAL,
"rm/sightbusy-lx@LOCALHOST");
- mrCluster.getConfig().set(YarnServerConfig.NM_SERVER_PRINCIPAL_KEY,
+ mrCluster.getConfig().set(YarnConfiguration.NM_PRINCIPAL,
"nm/sightbusy-lx@LOCALHOST");
UserGroupInformation.setConfiguration(mrCluster.getConfig());
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Tue Sep 13 22:49:27 2011
@@ -67,7 +67,7 @@ import org.apache.hadoop.metrics2.lib.Mu
import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.service.AbstractService;
@@ -263,10 +263,6 @@ public class ShuffleHandler extends Abst
}
}
- Shuffle createShuffle() {
- return new Shuffle(getConfig());
- }
-
class HttpPipelineFactory implements ChannelPipelineFactory {
final Shuffle SHUFFLE;
@@ -295,11 +291,13 @@ public class ShuffleHandler extends Abst
private final Configuration conf;
private final IndexCache indexCache;
private final LocalDirAllocator lDirAlloc =
- new LocalDirAllocator(NMConfig.NM_LOCAL_DIR);
+ new LocalDirAllocator(YarnConfiguration.NM_LOCAL_DIRS);
+ private final int port;
public Shuffle(Configuration conf) {
this.conf = conf;
indexCache = new IndexCache(new JobConf(conf));
+ this.port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
}
private List<String> splitMaps(List<String> mapq) {
@@ -362,7 +360,7 @@ public class ShuffleHandler extends Abst
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
try {
verifyRequest(jobId, ctx, request, response,
- new URL("http", "", port, reqUri));
+ new URL("http", "", this.port, reqUri));
} catch (IOException e) {
LOG.warn("Shuffle failure ", e);
sendError(ctx, e.getMessage(), UNAUTHORIZED);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml Tue Sep 13 22:49:27 2011
@@ -66,6 +66,11 @@
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
<Match>
+ <Class name="~org\.apache\.hadoop\.yarn\.server\.resourcemanager\.RMAppManager.*" />
+ <Method name="handle" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
+ <Match>
<Class name="~org\.apache\.hadoop\.yarn\.server\.resourcemanager\.scheduler\.capacity\.CapacityScheduler.*" />
<Method name="handle" />
<Bug pattern="BC_UNCONFIRMED_CAST" />
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java Tue Sep 13 22:49:27 2011
@@ -48,20 +48,20 @@ public class StartContainerResponsePBImp
viaProto = true;
}
- public StartContainerResponseProto getProto() {
+ public synchronized StartContainerResponseProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
- private void mergeLocalToBuilder() {
+ private synchronized void mergeLocalToBuilder() {
if (this.serviceResponse != null) {
addServiceResponseToProto();
}
}
- private void mergeLocalToProto() {
+ private synchronized void mergeLocalToProto() {
if (viaProto) {
maybeInitBuilder();
}
@@ -70,7 +70,7 @@ public class StartContainerResponsePBImp
viaProto = true;
}
- private void maybeInitBuilder() {
+ private synchronized void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = StartContainerResponseProto.newBuilder(proto);
}
@@ -79,17 +79,17 @@ public class StartContainerResponsePBImp
@Override
- public Map<String, ByteBuffer> getAllServiceResponse() {
+ public synchronized Map<String, ByteBuffer> getAllServiceResponse() {
initServiceResponse();
return this.serviceResponse;
}
@Override
- public ByteBuffer getServiceResponse(String key) {
+ public synchronized ByteBuffer getServiceResponse(String key) {
initServiceResponse();
return this.serviceResponse.get(key);
}
- private void initServiceResponse() {
+ private synchronized void initServiceResponse() {
if (this.serviceResponse != null) {
return;
}
@@ -103,14 +103,14 @@ public class StartContainerResponsePBImp
}
@Override
- public void addAllServiceResponse(final Map<String, ByteBuffer> serviceResponse) {
+ public synchronized void addAllServiceResponse(final Map<String, ByteBuffer> serviceResponse) {
if (serviceResponse == null)
return;
initServiceResponse();
this.serviceResponse.putAll(serviceResponse);
}
- private void addServiceResponseToProto() {
+ private synchronized void addServiceResponseToProto() {
maybeInitBuilder();
builder.clearServiceResponse();
if (serviceResponse == null)
@@ -118,24 +118,24 @@ public class StartContainerResponsePBImp
Iterable<StringBytesMapProto> iterable = new Iterable<StringBytesMapProto>() {
@Override
- public Iterator<StringBytesMapProto> iterator() {
+ public synchronized Iterator<StringBytesMapProto> iterator() {
return new Iterator<StringBytesMapProto>() {
Iterator<String> keyIter = serviceResponse.keySet().iterator();
@Override
- public void remove() {
+ public synchronized void remove() {
throw new UnsupportedOperationException();
}
@Override
- public StringBytesMapProto next() {
+ public synchronized StringBytesMapProto next() {
String key = keyIter.next();
return StringBytesMapProto.newBuilder().setKey(key).setValue(convertToProtoFormat(serviceResponse.get(key))).build();
}
@Override
- public boolean hasNext() {
+ public synchronized boolean hasNext() {
return keyIter.hasNext();
}
};
@@ -144,17 +144,17 @@ public class StartContainerResponsePBImp
builder.addAllServiceResponse(iterable);
}
@Override
- public void setServiceResponse(String key, ByteBuffer val) {
+ public synchronized void setServiceResponse(String key, ByteBuffer val) {
initServiceResponse();
this.serviceResponse.put(key, val);
}
@Override
- public void removeServiceResponse(String key) {
+ public synchronized void removeServiceResponse(String key) {
initServiceResponse();
this.serviceResponse.remove(key);
}
@Override
- public void clearServiceResponse() {
+ public synchronized void clearServiceResponse() {
initServiceResponse();
this.serviceResponse.clear();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java Tue Sep 13 22:49:27 2011
@@ -86,32 +86,17 @@ public interface AMResponse {
*/
@Public
@Stable
- public List<Container> getNewContainerList();
+ public List<Container> getAllocatedContainers();
- @Private
- @Unstable
- public Container getNewContainer(int index);
-
- @Private
- @Unstable
- public int getNewContainerCount();
-
- @Private
- @Unstable
- public void addAllNewContainers(List<Container> containers);
-
- @Private
- @Unstable
- public void addNewContainer(Container container);
-
- @Private
- @Unstable
- public void removeNewContainer(int index);
+ /**
+ * Set the list of <em>newly allocated</em> <code>Container</code> by the
+ * <code>ResourceManager</code>.
+ * @param containers list of <em>newly allocated</em> <code>Container</code>
+ */
+ @Public
+ @Stable
+ public void setAllocatedContainers(List<Container> containers);
- @Private
- @Unstable
- public void clearNewContainers();
-
/**
* Get the <em>available headroom</em> for resources in the cluster for the
* application.
@@ -127,35 +112,14 @@ public interface AMResponse {
public void setAvailableResources(Resource limit);
/**
- * Get the list of <em>completed containers</em>.
- * @return the list of <em>completed containers</em>
+ * Get the list of <em>completed containers' statuses</em>.
+ * @return the list of <em>completed containers' statuses</em>
*/
@Public
@Stable
- public List<Container> getFinishedContainerList();
-
- @Private
- @Unstable
- public Container getFinishedContainer(int index);
-
- @Private
- @Unstable
- public int getFinishedContainerCount();
-
-
- @Private
- @Unstable
- public void addAllFinishedContainers(List<Container> containers);
-
- @Private
- @Unstable
- public void addFinishedContainer(Container container);
-
- @Private
- @Unstable
- public void removeFinishedContainer(int index);
+ public List<ContainerStatus> getCompletedContainersStatuses();
@Private
@Unstable
- public void clearFinishedContainers();
+ public void setCompletedContainersStatuses(List<ContainerStatus> containers);
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java Tue Sep 13 22:49:27 2011
@@ -18,11 +18,105 @@
package org.apache.hadoop.yarn.api.records;
-public interface ApplicationAttemptId extends Comparable<ApplicationAttemptId>{
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <p><code>ApplicationAttemptId</code> denotes the particular <em>attempt</em>
+ * of an <code>ApplicationMaster</code> for a given {@link ApplicationId}.</p>
+ *
+ * <p>Multiple attempts might be needed to run an application to completion due
+ * to temporal failures of the <code>ApplicationMaster</code> such as hardware
+ * failures, connectivity issues etc. on the node on which it was scheduled.</p>
+ */
+@Public
+@Stable
+public abstract class ApplicationAttemptId implements
+ Comparable<ApplicationAttemptId> {
+
+ /**
+ * Get the <code>ApplicationId</code> of the <code>ApplicationAttempId</code>.
+ * @return <code>ApplicationId</code> of the <code>ApplicationAttempId</code>
+ */
+ @Public
+ @Stable
public abstract ApplicationId getApplicationId();
- public abstract int getAttemptId();
+ @Private
+ @Unstable
public abstract void setApplicationId(ApplicationId appID);
+
+ /**
+ * Get the <code>attempt id</code> of the <code>Application</code>.
+ * @return <code>attempt id</code> of the <code>Application</code>
+ */
+ public abstract int getAttemptId();
+
+ @Private
+ @Unstable
public abstract void setAttemptId(int attemptId);
+
+
+
+ protected static final NumberFormat idFormat = NumberFormat.getInstance();
+ static {
+ idFormat.setGroupingUsed(false);
+ idFormat.setMinimumIntegerDigits(4);
+ }
+
+ protected static final NumberFormat counterFormat = NumberFormat
+ .getInstance();
+ static {
+ counterFormat.setGroupingUsed(false);
+ counterFormat.setMinimumIntegerDigits(6);
+ }
+
+ @Override
+ public int hashCode() {
+ // Generated by eclipse.
+ final int prime = 31;
+ int result = 1;
+ ApplicationId appId = getApplicationId();
+ result = prime * result + ((appId == null) ? 0 : appId.hashCode());
+ result = prime * result + getAttemptId();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ ApplicationAttemptId otherAttemptId = (ApplicationAttemptId) other;
+ if (this.getApplicationId().equals(otherAttemptId.getApplicationId())) {
+ return this.getAttemptId() == otherAttemptId.getAttemptId();
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(ApplicationAttemptId other) {
+ int compareAppIds = this.getApplicationId().compareTo(
+ other.getApplicationId());
+ if (compareAppIds == 0) {
+ return this.getAttemptId() - other.getAttemptId();
+ } else {
+ return compareAppIds;
+ }
+ }
+ @Override
+ public String toString() {
+ String id =
+ (this.getApplicationId() != null) ? this.getApplicationId()
+ .getClusterTimestamp()
+ + "_"
+ + idFormat.format(this.getApplicationId().getId()) : "none";
+ return "appattempt_" + id + "_" + counterFormat.format(getAttemptId());
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java Tue Sep 13 22:49:27 2011
@@ -18,13 +18,87 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-public interface ApplicationId extends Comparable<ApplicationId> {
+/**
+ * <p><code>ApplicationId</code> represents the <em>globally unique</em>
+ * identifier for an application.</p>
+ *
+ * <p>The globally unique nature of the identifier is achieved by using the
+ * <em>cluster timestamp</em> i.e. start-time of the
+ * <code>ResourceManager</code> along with a monotonically increasing counter
+ * for the application.</p>
+ */
+@Public
+@Stable
+public abstract class ApplicationId implements Comparable<ApplicationId> {
+
+ /**
+ * Get the short integer identifier of the <code>ApplicationId</code>
+ * which is unique for all applications started by a particular instance
+ * of the <code>ResourceManager</code>.
+ * @return short integer identifier of the <code>ApplicationId</code>
+ */
+ @Public
+ @Stable
public abstract int getId();
- public abstract long getClusterTimestamp();
+ @Private
+ @Unstable
public abstract void setId(int id);
+
+ /**
+ * Get the <em>start time</em> of the <code>ResourceManager</code> which is
+ * used to generate globally unique <code>ApplicationId</code>.
+ * @return <em>start time</em> of the <code>ResourceManager</code>
+ */
+ public abstract long getClusterTimestamp();
+
+ @Private
+ @Unstable
public abstract void setClusterTimestamp(long clusterTimestamp);
+
+ @Override
+ public int compareTo(ApplicationId other) {
+ if (this.getClusterTimestamp() - other.getClusterTimestamp() == 0) {
+ return this.getId() - other.getId();
+ } else {
+ return this.getClusterTimestamp() > other.getClusterTimestamp() ? 1 :
+ this.getClusterTimestamp() < other.getClusterTimestamp() ? -1 : 0;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "application_" + this.getClusterTimestamp() + "_" + this.getId();
+ }
+
+ @Override
+ public int hashCode() {
+ // Generated by eclipse.
+ final int prime = 31;
+ int result = 1;
+ long clusterTimestamp = getClusterTimestamp();
+ result = prime * result
+ + (int) (clusterTimestamp ^ (clusterTimestamp >>> 32));
+ result = prime * result + getId();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ ApplicationId otherAppId = (ApplicationId)other;
+ if (this.getClusterTimestamp() == otherAppId.getClusterTimestamp() &&
+ this.getId() == otherAppId.getId()) {
+ return true;
+ }
+ }
+ return false;
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java Tue Sep 13 22:49:27 2011
@@ -18,26 +18,43 @@
package org.apache.hadoop.yarn.api.records;
-//TODO: Split separate object for register, deregister and in-RM use.
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <em>For internal use only...</em>
+ */
+@Private
+@Unstable
public interface ApplicationMaster {
ApplicationId getApplicationId();
- String getHost();
- int getRpcPort();
- String getTrackingUrl();
- ApplicationStatus getStatus();
- ApplicationState getState();
- String getClientToken();
- int getAMFailCount();
- int getContainerCount();
- String getDiagnostics();
void setApplicationId(ApplicationId appId);
+
+ String getHost();
void setHost(String host);
+
+ int getRpcPort();
void setRpcPort(int rpcPort);
+
+ String getTrackingUrl();
void setTrackingUrl(String url);
+
+ ApplicationStatus getStatus();
void setStatus(ApplicationStatus status);
+
+ ApplicationState getState();
void setState(ApplicationState state);
+
+ String getClientToken();
void setClientToken(String clientToken);
+
+ int getAMFailCount();
void setAMFailCount(int amFailCount);
+
+ int getContainerCount();
void setContainerCount(int containerCount);
+
+ String getDiagnostics();
void setDiagnostics(String diagnostics);
}
+
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationState.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationState.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationState.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationState.java Tue Sep 13 22:49:27 2011
@@ -18,6 +18,30 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+
+/**
+ * Ennumeration of various states of an <code>Application</code>.
+ */
+@Public
+@Stable
public enum ApplicationState {
- NEW, SUBMITTED, RUNNING, SUCCEEDED, FAILED, KILLED
+ /** Application which was just created. */
+ NEW,
+
+ /** Application which has been submitted. */
+ SUBMITTED,
+
+ /** Application which is currently running. */
+ RUNNING,
+
+ /** Application which completed successfully. */
+ SUCCEEDED,
+
+ /** Application which failed. */
+ FAILED,
+
+ /** Application which was terminated by a user or admin. */
+ KILLED
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationStatus.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationStatus.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationStatus.java Tue Sep 13 22:49:27 2011
@@ -18,12 +18,21 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <em>For internal use only...</em>
+ */
+@Private
+@Unstable
public interface ApplicationStatus {
ApplicationAttemptId getApplicationAttemptId();
- int getResponseId();
- float getProgress();
-
void setApplicationAttemptId(ApplicationAttemptId applicationAttemptId);
+
+ int getResponseId();
void setResponseId(int id);
+
+ float getProgress();
void setProgress(float progress);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java Tue Sep 13 22:49:27 2011
@@ -18,21 +18,133 @@
package org.apache.hadoop.yarn.api.records;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.api.ContainerManager;
+/**
+ * <p><code>Container</code> represents an allocated resource in the cluster.
+ * </p>
+ *
+ * <p>The <code>ResourceManager</code> is the sole authority to allocate any
+ * <code>Container</code> to applications. The allocated <code>Container</code>
+ * is always on a single node and has a unique {@link ContainerId}. It has
+ * a specific amount of {@link Resource} allocated.</p>
+ *
+ * <p>It includes details such as:
+ * <ul>
+ * <li>{@link ContainerId} for the container, which is globally unique.</li>
+ * <li>
+ * {@link NodeId} of the node on which identifies the node on which it
+ * is allocated.
+ * </li>
+ * <li>HTTP uri of the node.</li>
+ * <li>{@link Resource} allocated to the container.</li>
+ * <li>{@link ContainerState} of the container.</li>
+ * <li>
+ * {@link ContainerToken} of the container, used to securely verify
+ * authenticity of the allocation.
+ * </li>
+ * <li>{@link ContainerStatus} of the container.</li>
+ * </ul>
+ * </p>
+ *
+ * <p>Typically, an <code>ApplicationMaster</code> receives the
+ * <code>Container</code> from the <code>ResourceManager</code> during
+ * resource-negotiation and then talks to the <code>NodManager</code> to
+ * start/stop containers.</p>
+ *
+ * @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ * @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)
+ * @see ContainerManager#stopContainer(org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest)
+ */
+@Public
+@Stable
public interface Container extends Comparable<Container> {
+ /**
+ * Get the globally unique identifier for the container.
+ * @return globally unique identifier for the container
+ */
+ @Public
+ @Stable
ContainerId getId();
- NodeId getNodeId();
- String getNodeHttpAddress();
- Resource getResource();
- ContainerState getState();
- ContainerToken getContainerToken();
- ContainerStatus getContainerStatus();
+ @Private
+ @Unstable
void setId(ContainerId id);
+
+ /**
+ * Get the identifier of the node on which the container is allocated.
+ * @return identifier of the node on which the container is allocated
+ */
+ @Public
+ @Stable
+ NodeId getNodeId();
+
+ @Private
+ @Unstable
void setNodeId(NodeId nodeId);
+
+ /**
+ * Get the http uri of the node on which the container is allocated.
+ * @return http uri of the node on which the container is allocated
+ */
+ @Public
+ @Stable
+ String getNodeHttpAddress();
+
+ @Private
+ @Unstable
void setNodeHttpAddress(String nodeHttpAddress);
+
+ /**
+ * Get the <code>Resource</code> allocated to the container.
+ * @return <code>Resource</code> allocated to the container
+ */
+ @Public
+ @Stable
+ Resource getResource();
+
+ @Private
+ @Unstable
void setResource(Resource resource);
+
+ /**
+ * Get the current <code>ContainerState</code> of the container.
+ * @return current <code>ContainerState</code> of the container
+ */
+ @Public
+ @Stable
+ ContainerState getState();
+
+ @Private
+ @Unstable
void setState(ContainerState state);
+
+ /**
+ * Get the <code>ContainerToken</code> for the container.
+ * @return <code>ContainerToken</code> for the container
+ */
+ @Public
+ @Stable
+ ContainerToken getContainerToken();
+
+ @Private
+ @Unstable
void setContainerToken(ContainerToken containerToken);
+
+ /**
+ * Get the <code>ContainerStatus</code> of the container.
+ * @return <code>ContainerStatus</code> of the container
+ */
+ @Public
+ @Stable
+ ContainerStatus getContainerStatus();
+
+ @Private
+ @Unstable
void setContainerStatus(ContainerStatus containerStatus);
}