You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ar...@apache.org on 2013/09/26 05:41:13 UTC
svn commit: r1526366 - in
/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src...
Author: arp
Date: Thu Sep 26 03:41:09 2013
New Revision: 1526366
URL: http://svn.apache.org/r1526366
Log:
Merging r1525759 through r1526365 from trunk to branch HDFS-2832
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/INSTALL
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobClient.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1525759-1526365
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt Thu Sep 26 03:41:09 2013
@@ -193,6 +193,18 @@ Release 2.2.0 - UNRELEASED
BUG FIXES
+Release 2.1.2 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
MAPREDUCE-5504. mapred queue -info inconsistent with types (Kousuke Saruta
via tgraves)
@@ -204,6 +216,11 @@ Release 2.2.0 - UNRELEASED
needs to set up its own certificates etc and not depend on clusters'.
(Omkar Vinit Joshi via vinodkv)
+ MAPREDUCE-5505. Clients should be notified job finished only after job
+ successfully unregistered (Zhijie Shen via bikas)
+
+ MAPREDUCE-5503. Fixed a test issue in TestMRJobClient. (Jian He via vinodkv)
+
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES
@@ -221,6 +238,10 @@ Release 2.1.1-beta - 2013-09-23
MAPREDUCE-5379. Include token tracking ids in jobconf. (kkambatl via tucu)
+ MAPREDUCE-5523. Added separate configuration properties for https for JHS
+ without which even when https is enabled, it starts on http port itself.
+ (Omkar Vinit Joshi via vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-5446. TestJobHistoryEvents and TestJobHistoryParsing have race
Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1525759-1526365
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/INSTALL
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/INSTALL?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/INSTALL (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/INSTALL Thu Sep 26 03:41:09 2013
@@ -39,11 +39,11 @@ export YARN_CONF_DIR=$HADOOP_CONF_DIR
Step 7) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager.
<property>
<name>yarn.nodemanager.aux-services</name>
- <value>mapreduce.shuffle</value>
+ <value>mapreduce_shuffle</value>
</property>
<property>
- <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
+ <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1525759-1526365
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java Thu Sep 26 03:41:09 2013
@@ -63,4 +63,7 @@ public interface AppContext {
ClientToAMTokenSecretManager getClientToAMTokenSecretManager();
boolean isLastAMRetry();
+
+ boolean safeToReportTerminationToUser();
+
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Thu Sep 26 03:41:09 2013
@@ -31,6 +31,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
@@ -209,6 +210,10 @@ public class MRAppMaster extends Composi
private long recoveredJobStartTime = 0;
+ @VisibleForTesting
+ protected AtomicBoolean safeToReportTerminationToUser =
+ new AtomicBoolean(false);
+
public MRAppMaster(ApplicationAttemptId applicationAttemptId,
ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
long appSubmitTime, int maxAppAttempts) {
@@ -554,8 +559,10 @@ public class MRAppMaster extends Composi
LOG.info("Calling stop for all the services");
MRAppMaster.this.stop();
- // TODO: Stop ClientService last, since only ClientService should wait for
- // some time so clients can know the final states. Will be removed once RM come on.
+ // Except ClientService, other services are already stopped, it is safe to
+ // let clients know the final states. ClientService should wait for some
+ // time so clients have enough time to know the final states.
+ safeToReportTerminationToUser.set(true);
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
@@ -964,6 +971,11 @@ public class MRAppMaster extends Composi
public boolean isLastAMRetry(){
return isLastAMRetry;
}
+
+ @Override
+ public boolean safeToReportTerminationToUser() {
+ return safeToReportTerminationToUser.get();
+ }
}
@SuppressWarnings("unchecked")
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Thu Sep 26 03:41:09 2013
@@ -641,6 +641,8 @@ public class JobImpl implements org.apac
private ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
private ScheduledFuture failWaitTriggerScheduledFuture;
+ private JobState lastNonFinalState = JobState.NEW;
+
public JobImpl(JobId jobId, ApplicationAttemptId applicationAttemptId,
Configuration conf, EventHandler eventHandler,
TaskAttemptListener taskAttemptListener,
@@ -928,7 +930,14 @@ public class JobImpl implements org.apac
public JobState getState() {
readLock.lock();
try {
- return getExternalState(getInternalState());
+ JobState state = getExternalState(getInternalState());
+ if (!appContext.safeToReportTerminationToUser()
+ && (state == JobState.SUCCEEDED || state == JobState.FAILED
+ || state == JobState.KILLED || state == JobState.ERROR)) {
+ return lastNonFinalState;
+ } else {
+ return state;
+ }
} finally {
readLock.unlock();
}
@@ -972,6 +981,7 @@ public class JobImpl implements org.apac
if (oldState != getInternalState()) {
LOG.info(jobId + "Job Transitioned from " + oldState + " to "
+ getInternalState());
+ rememberLastNonFinalState(oldState);
}
}
@@ -980,6 +990,15 @@ public class JobImpl implements org.apac
}
}
+ private void rememberLastNonFinalState(JobStateInternal stateInternal) {
+ JobState state = getExternalState(stateInternal);
+ // if state is not the final state, set lastNonFinalState
+ if (state != JobState.SUCCEEDED && state != JobState.FAILED
+ && state != JobState.KILLED && state != JobState.ERROR) {
+ lastNonFinalState = state;
+ }
+ }
+
@Private
public JobStateInternal getInternalState() {
readLock.lock();
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java Thu Sep 26 03:41:09 2013
@@ -37,11 +37,11 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AppInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.StringHelper;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.Controller;
import org.apache.hadoop.yarn.webapp.View;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.base.Joiner;
import com.google.inject.Inject;
@@ -62,7 +62,7 @@ public class AppController extends Contr
set(APP_ID, app.context.getApplicationID().toString());
set(RM_WEB,
JOINER.join(WebAppUtil.getSchemePrefix(),
- YarnConfiguration.getRMWebAppHostAndPort(conf)));
+ WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf)));
}
@Inject
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java Thu Sep 26 03:41:09 2013
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.mapreduce.v2.app.webapp;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+
public class WebAppUtil {
private static boolean isSSLEnabledInYARN;
@@ -36,4 +40,21 @@ public class WebAppUtil {
return "http://";
}
}
-}
+
+ public static void setJHSWebAppURLWithoutScheme(Configuration conf,
+ String hostAddress) {
+ if (HttpConfig.isSecure()) {
+ conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS, hostAddress);
+ } else {
+ conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, hostAddress);
+ }
+ }
+
+ public static String getJHSWebAppURLWithoutScheme(Configuration conf) {
+ if (HttpConfig.isSecure()) {
+ return conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS);
+ } else {
+ return conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS);
+ }
+ }
+}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java Thu Sep 26 03:41:09 2013
@@ -136,10 +136,21 @@ public class MRApp extends MRAppMaster {
}
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
+ boolean cleanOnStart, Clock clock, boolean shutdown) {
+ this(maps, reduces, autoComplete, testName, cleanOnStart, 1, clock,
+ shutdown);
+ }
+
+ public MRApp(int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart) {
this(maps, reduces, autoComplete, testName, cleanOnStart, 1);
}
-
+
+ public MRApp(int maps, int reduces, boolean autoComplete, String testName,
+ boolean cleanOnStart, boolean shutdown) {
+ this(maps, reduces, autoComplete, testName, cleanOnStart, 1, shutdown);
+ }
+
@Override
protected void initJobCredentialsAndUGI(Configuration conf) {
// Fake a shuffle secret that normally is provided by the job client.
@@ -170,22 +181,42 @@ public class MRApp extends MRAppMaster {
}
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
+ boolean cleanOnStart, int startCount, boolean shutdown) {
+ this(maps, reduces, autoComplete, testName, cleanOnStart, startCount,
+ new SystemClock(), shutdown);
+ }
+
+ public MRApp(int maps, int reduces, boolean autoComplete, String testName,
+ boolean cleanOnStart, int startCount, Clock clock, boolean shutdown) {
+ this(getApplicationAttemptId(applicationId, startCount), getContainerId(
+ applicationId, startCount), maps, reduces, autoComplete, testName,
+ cleanOnStart, startCount, clock, shutdown);
+ }
+
+ public MRApp(int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, int startCount, Clock clock) {
this(getApplicationAttemptId(applicationId, startCount), getContainerId(
applicationId, startCount), maps, reduces, autoComplete, testName,
- cleanOnStart, startCount, clock);
+ cleanOnStart, startCount, clock, true);
+ }
+
+ public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
+ int maps, int reduces, boolean autoComplete, String testName,
+ boolean cleanOnStart, int startCount, boolean shutdown) {
+ this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName,
+ cleanOnStart, startCount, new SystemClock(), shutdown);
}
public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, int startCount) {
this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName,
- cleanOnStart, startCount, new SystemClock());
+ cleanOnStart, startCount, new SystemClock(), true);
}
public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
int maps, int reduces, boolean autoComplete, String testName,
- boolean cleanOnStart, int startCount, Clock clock) {
+ boolean cleanOnStart, int startCount, Clock clock, boolean shutdown) {
super(appAttemptId, amContainerId, NM_HOST, NM_PORT, NM_HTTP_PORT, clock, System
.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
this.testWorkDir = new File("target", testName);
@@ -204,6 +235,9 @@ public class MRApp extends MRAppMaster {
this.maps = maps;
this.reduces = reduces;
this.autoComplete = autoComplete;
+ // If safeToReportTerminationToUser is set to true, we can verify whether
+ // the job can reaches the final state when MRAppMaster shuts down.
+ this.safeToReportTerminationToUser.set(shutdown);
}
@Override
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java Thu Sep 26 03:41:09 2013
@@ -135,4 +135,11 @@ public class MockAppContext implements A
public boolean isLastAMRetry() {
return false;
}
+
+ @Override
+ public boolean safeToReportTerminationToUser() {
+ // bogus - Not Required
+ return true;
+ }
+
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java Thu Sep 26 03:41:09 2013
@@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobStartEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
@@ -374,6 +375,19 @@ public class TestMRApp {
app.waitForState(job, JobState.ERROR);
}
+ @SuppressWarnings("resource")
+ @Test
+ public void testJobSuccess() throws Exception {
+ MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true, false);
+ JobImpl job = (JobImpl) app.submit(new Configuration());
+ app.waitForInternalState(job, JobStateInternal.SUCCEEDED);
+ // AM is not unregistered
+ Assert.assertEquals(JobState.RUNNING, job.getState());
+ // imitate that AM is unregistered
+ app.safeToReportTerminationToUser.set(true);
+ app.waitForState(job, JobState.SUCCEEDED);
+ }
+
@Test
public void testJobRebootNotLastRetry() throws Exception {
MRApp app = new MRApp(1, 0, false, this.getClass().getName(), true);
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java Thu Sep 26 03:41:09 2013
@@ -867,5 +867,12 @@ public class TestRuntimeEstimators {
public boolean isLastAMRetry() {
return false;
}
+
+ @Override
+ public boolean safeToReportTerminationToUser() {
+ // bogus - Not Required
+ return true;
+ }
+
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java Thu Sep 26 03:41:09 2013
@@ -275,6 +275,7 @@ public class TestJobImpl {
AppContext mockContext = mock(AppContext.class);
when(mockContext.isLastAMRetry()).thenReturn(true);
+ when(mockContext.safeToReportTerminationToUser()).thenReturn(false);
JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, mockContext);
completeJobTasks(job);
assertJobState(job, JobStateInternal.COMMITTING);
@@ -282,7 +283,9 @@ public class TestJobImpl {
syncBarrier.await();
job.handle(new JobEvent(job.getID(), JobEventType.JOB_AM_REBOOT));
assertJobState(job, JobStateInternal.REBOOT);
- // return the external state as FAILED since this is last retry.
+ // return the external state as ERROR since this is last retry.
+ Assert.assertEquals(JobState.RUNNING, job.getState());
+ when(mockContext.safeToReportTerminationToUser()).thenReturn(true);
Assert.assertEquals(JobState.ERROR, job.getState());
dispatcher.stop();
@@ -590,12 +593,14 @@ public class TestJobImpl {
final JobDiagnosticsUpdateEvent diagUpdateEvent =
new JobDiagnosticsUpdateEvent(jobId, diagMsg);
MRAppMetrics mrAppMetrics = MRAppMetrics.create();
+ AppContext mockContext = mock(AppContext.class);
+ when(mockContext.safeToReportTerminationToUser()).thenReturn(true);
JobImpl job = new JobImpl(jobId, Records
.newRecord(ApplicationAttemptId.class), new Configuration(),
mock(EventHandler.class),
null, mock(JobTokenSecretManager.class), null,
new SystemClock(), null,
- mrAppMetrics, null, true, null, 0, null, null, null, null);
+ mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
job.handle(diagUpdateEvent);
String diagnostics = job.getReport().getDiagnostics();
Assert.assertNotNull(diagnostics);
@@ -606,7 +611,7 @@ public class TestJobImpl {
mock(EventHandler.class),
null, mock(JobTokenSecretManager.class), null,
new SystemClock(), null,
- mrAppMetrics, null, true, null, 0, null, null, null, null);
+ mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
job.handle(diagUpdateEvent);
diagnostics = job.getReport().getDiagnostics();
@@ -699,7 +704,9 @@ public class TestJobImpl {
commitHandler.init(conf);
commitHandler.start();
- JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
+ AppContext mockContext = mock(AppContext.class);
+ when(mockContext.safeToReportTerminationToUser()).thenReturn(false);
+ JobImpl job = createStubbedJob(conf, dispatcher, 2, mockContext);
JobId jobId = job.getID();
job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
assertJobState(job, JobStateInternal.INITED);
@@ -707,12 +714,15 @@ public class TestJobImpl {
assertJobState(job, JobStateInternal.FAILED);
job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_COMPLETED));
- Assert.assertEquals(JobState.FAILED, job.getState());
+ assertJobState(job, JobStateInternal.FAILED);
job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_COMPLETED));
- Assert.assertEquals(JobState.FAILED, job.getState());
+ assertJobState(job, JobStateInternal.FAILED);
job.handle(new JobEvent(jobId, JobEventType.JOB_MAP_TASK_RESCHEDULED));
- Assert.assertEquals(JobState.FAILED, job.getState());
+ assertJobState(job, JobStateInternal.FAILED);
job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE));
+ assertJobState(job, JobStateInternal.FAILED);
+ Assert.assertEquals(JobState.RUNNING, job.getState());
+ when(mockContext.safeToReportTerminationToUser()).thenReturn(true);
Assert.assertEquals(JobState.FAILED, job.getState());
dispatcher.stop();
@@ -750,6 +760,10 @@ public class TestJobImpl {
Dispatcher dispatcher, int numSplits, AppContext appContext) {
JobID jobID = JobID.forName("job_1234567890000_0001");
JobId jobId = TypeConverter.toYarn(jobID);
+ if (appContext == null) {
+ appContext = mock(AppContext.class);
+ when(appContext.safeToReportTerminationToUser()).thenReturn(true);
+ }
StubbedJob job = new StubbedJob(jobId,
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0),
conf,dispatcher.getEventHandler(), true, "somebody", numSplits, appContext);
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java Thu Sep 26 03:41:09 2013
@@ -18,8 +18,15 @@
package org.apache.hadoop.mapreduce.v2.jobhistory;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.net.NetUtils;
/**
* Stores Job History configuration keys that can be set by administrators of
@@ -130,6 +137,13 @@ public class JHAdminConfig {
public static final String DEFAULT_MR_HISTORY_WEBAPP_ADDRESS =
"0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_PORT;
+ /**The https address the history server webapp is on.*/
+ public static final String MR_HISTORY_WEBAPP_HTTPS_ADDRESS =
+ MR_HISTORY_PREFIX + "webapp.https.address";
+ public static final int DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT = 19890;
+ public static final String DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS =
+ "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT;
+
/**The kerberos principal to be used for spnego filter for history server*/
public static final String MR_WEBAPP_SPNEGO_USER_NAME_KEY =
MR_HISTORY_PREFIX + "webapp.spnego-principal";
@@ -160,4 +174,36 @@ public class JHAdminConfig {
*/
public static boolean DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS = false;
+ public static String getResolvedMRHistoryWebAppURLWithoutScheme(
+ Configuration conf) {
+ InetSocketAddress address = null;
+ if (HttpConfig.isSecure()) {
+ address =
+ conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT);
+ } else {
+ address =
+ conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT); }
+ address = NetUtils.getConnectAddress(address);
+ StringBuffer sb = new StringBuffer();
+ InetAddress resolved = address.getAddress();
+ if (resolved == null || resolved.isAnyLocalAddress() ||
+ resolved.isLoopbackAddress()) {
+ String lh = address.getHostName();
+ try {
+ lh = InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ //Ignore and fallback.
+ }
+ sb.append(lh);
+ } else {
+ sb.append(address.getHostName());
+ }
+ sb.append(":").append(address.getPort());
+ return sb.toString();
+ }
+
}
Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1525759-1526365
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm Thu Sep 26 03:41:09 2013
@@ -83,14 +83,14 @@ Hadoop MapReduce Next Generation - Plugg
*--------------------------------------+---------------------+-----------------+
| <<Property>> | <<Default Value>> | <<Explanation>> |
*--------------------------------------+---------------------+-----------------+
-| <<<yarn.nodemanager.aux-services>>> | <<<...,mapreduce.shuffle>>> | The auxiliary service name |
+| <<<yarn.nodemanager.aux-services>>> | <<<...,mapreduce_shuffle>>> | The auxiliary service name |
*--------------------------------------+---------------------+-----------------+
-| <<<yarn.nodemanager.aux-services.mapreduce.shuffle.class>>> | <<<org.apache.hadoop.mapred.ShuffleHandler>>> | The auxiliary service class to use |
+| <<<yarn.nodemanager.aux-services.mapreduce_shuffle.class>>> | <<<org.apache.hadoop.mapred.ShuffleHandler>>> | The auxiliary service class to use |
*--------------------------------------+---------------------+-----------------+
<<IMPORTANT:>> If setting an auxiliary service in addition the default
- <<<mapreduce.shuffle>>> service, then a new service key should be added to the
+ <<<mapreduce_shuffle>>> service, then a new service key should be added to the
<<<yarn.nodemanager.aux-services>>> property, for example <<<mapred.shufflex>>>.
Then the property defining the corresponding class must be
- <<<yarn.nodemanager.aux-services.mapreduce.shufflex.class>>>.
+ <<<yarn.nodemanager.aux-services.mapreduce_shufflex.class>>>.
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java Thu Sep 26 03:41:09 2013
@@ -387,4 +387,11 @@ public class JobHistory extends Abstract
// bogus - Not Required
return false;
}
+
+ @Override
+ public boolean safeToReportTerminationToUser() {
+ // bogus - Not Required
+ return true;
+ }
+
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobClient.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobClient.java Thu Sep 26 03:41:09 2013
@@ -79,7 +79,7 @@ public class TestJobClient extends TestM
Configuration conf = createJobConf();
String jobId = runJob();
testGetCounter(jobId, conf);
- testJobList(jobId, conf);
+ testAllJobList(jobId, conf);
testChangingJobPriority(jobId, conf);
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java Thu Sep 26 03:41:09 2013
@@ -29,6 +29,8 @@ import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.PrintStream;
+import junit.framework.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -60,6 +62,22 @@ public class TestMRJobClient extends Clu
return job;
}
+ private Job runJobInBackGround(Configuration conf) throws Exception {
+ String input = "hello1\nhello2\nhello3\n";
+
+ Job job = MapReduceTestUtil.createJob(conf, getInputDir(), getOutputDir(),
+ 1, 1, input);
+ job.setJobName("mr");
+ job.setPriority(JobPriority.NORMAL);
+ job.submit();
+ int i = 0;
+ while (i++ < 200 && job.getJobID() == null) {
+ LOG.info("waiting for jobId...");
+ Thread.sleep(100);
+ }
+ return job;
+ }
+
public static int runTool(Configuration conf, Tool tool, String[] args,
OutputStream out) throws Exception {
PrintStream oldOut = System.out;
@@ -108,8 +126,10 @@ public class TestMRJobClient extends Clu
Job job = runJob(conf);
String jobId = job.getJobID().toString();
- // test jobs list
- testJobList(jobId, conf);
+ // test all jobs list
+ testAllJobList(jobId, conf);
+ // test only submitted jobs list
+ testSubmittedJobList(conf);
// test job counter
testGetCounter(jobId, conf);
// status
@@ -131,38 +151,37 @@ public class TestMRJobClient extends Clu
// submit job from file
testSubmit(conf);
// kill a task
- testKillTask(job, conf);
+ testKillTask(conf);
// fail a task
- testfailTask(job, conf);
+ testfailTask(conf);
// kill job
- testKillJob(jobId, conf);
-
+ testKillJob(conf);
}
/**
* test fail task
*/
- private void testfailTask(Job job, Configuration conf) throws Exception {
+ private void testfailTask(Configuration conf) throws Exception {
+ Job job = runJobInBackGround(conf);
CLI jc = createJobClient();
TaskID tid = new TaskID(job.getJobID(), TaskType.MAP, 0);
TaskAttemptID taid = new TaskAttemptID(tid, 1);
ByteArrayOutputStream out = new ByteArrayOutputStream();
- // TaskAttemptId is not set
+ // TaskAttemptId is not set
int exitCode = runTool(conf, jc, new String[] { "-fail-task" }, out);
assertEquals("Exit code", -1, exitCode);
- try {
- runTool(conf, jc, new String[] { "-fail-task", taid.toString() }, out);
- fail(" this task should field");
- } catch (IOException e) {
- // task completed !
- assertTrue(e.getMessage().contains("_0001_m_000000_1"));
- }
+ runTool(conf, jc, new String[] { "-fail-task", taid.toString() }, out);
+ String answer = new String(out.toByteArray(), "UTF-8");
+ Assert
+ .assertTrue(answer.contains("Killed task " + taid + " by failing it"));
}
+
/**
* test a kill task
*/
- private void testKillTask(Job job, Configuration conf) throws Exception {
+ private void testKillTask(Configuration conf) throws Exception {
+ Job job = runJobInBackGround(conf);
CLI jc = createJobClient();
TaskID tid = new TaskID(job.getJobID(), TaskType.MAP, 0);
TaskAttemptID taid = new TaskAttemptID(tid, 1);
@@ -171,20 +190,17 @@ public class TestMRJobClient extends Clu
int exitCode = runTool(conf, jc, new String[] { "-kill-task" }, out);
assertEquals("Exit code", -1, exitCode);
- try {
- runTool(conf, jc, new String[] { "-kill-task", taid.toString() }, out);
- fail(" this task should be killed");
- } catch (IOException e) {
- System.out.println(e);
- // task completed
- assertTrue(e.getMessage().contains("_0001_m_000000_1"));
- }
+ runTool(conf, jc, new String[] { "-kill-task", taid.toString() }, out);
+ String answer = new String(out.toByteArray(), "UTF-8");
+ Assert.assertTrue(answer.contains("Killed task " + taid));
}
/**
* test a kill job
*/
- private void testKillJob(String jobId, Configuration conf) throws Exception {
+ private void testKillJob(Configuration conf) throws Exception {
+ Job job = runJobInBackGround(conf);
+ String jobId = job.getJobID().toString();
CLI jc = createJobClient();
ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -435,7 +451,8 @@ public class TestMRJobClient extends Clu
/**
* print a job list
*/
- protected void testJobList(String jobId, Configuration conf) throws Exception {
+ protected void testAllJobList(String jobId, Configuration conf)
+ throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
// bad options
@@ -458,23 +475,31 @@ public class TestMRJobClient extends Clu
}
assertEquals(1, counter);
out.reset();
- // only submitted
- exitCode = runTool(conf, createJobClient(), new String[] { "-list" }, out);
- assertEquals("Exit code", 0, exitCode);
- br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(
- out.toByteArray())));
+ }
+
+ protected void testSubmittedJobList(Configuration conf) throws Exception {
+ Job job = runJobInBackGround(conf);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ String line;
+ int counter = 0;
+ // only submitted
+ int exitCode =
+ runTool(conf, createJobClient(), new String[] { "-list" }, out);
+ assertEquals("Exit code", 0, exitCode);
+ BufferedReader br =
+ new BufferedReader(new InputStreamReader(new ByteArrayInputStream(
+ out.toByteArray())));
counter = 0;
while ((line = br.readLine()) != null) {
LOG.info("line = " + line);
- if (line.contains(jobId)) {
+ if (line.contains(job.getJobID().toString())) {
counter++;
}
}
// all jobs submitted! no current
assertEquals(1, counter);
-
}
-
+
protected void verifyJobPriority(String jobId, String priority,
Configuration conf, CLI jc) throws Exception {
PipedInputStream pis = new PipedInputStream();
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Thu Sep 26 03:41:09 2013
@@ -28,10 +28,12 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.mapred.LocalContainerLauncher;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.v2.app.webapp.WebAppUtil;
import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
@@ -43,6 +45,8 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.hamcrest.core.IsEqual;
/**
* Configures and starts the MR-specific components in the YARN cluster.
@@ -155,8 +159,7 @@ public class MiniMRYarnCluster extends M
// pick free random ports.
getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
hostname + ":0");
- getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
- hostname + ":0");
+ WebAppUtil.setJHSWebAppURLWithoutScheme(getConfig(), hostname + ":0");
getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS,
hostname + ":0");
}
@@ -182,17 +185,17 @@ public class MiniMRYarnCluster extends M
//need to do this because historyServer.init creates a new Configuration
getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
- getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
- historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS));
+ WebAppUtil.setJHSWebAppURLWithoutScheme(getConfig(),
+ WebAppUtil.getJHSWebAppURLWithoutScheme(historyServer.getConfig()));
LOG.info("MiniMRYARN ResourceManager address: " +
getConfig().get(YarnConfiguration.RM_ADDRESS));
LOG.info("MiniMRYARN ResourceManager web address: " +
- getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS));
+ WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
LOG.info("MiniMRYARN HistoryServer address: " +
getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
LOG.info("MiniMRYARN HistoryServer web address: " +
- getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS));
+ JHAdminConfig.getResolvedMRHistoryWebAppURLWithoutScheme(getConfig()));
}
@Override
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Thu Sep 26 03:41:09 2013
@@ -146,7 +146,7 @@ public class ShuffleHandler extends Auxi
private ReadaheadPool readaheadPool = ReadaheadPool.getInstance();
public static final String MAPREDUCE_SHUFFLE_SERVICEID =
- "mapreduce.shuffle";
+ "mapreduce_shuffle";
private static final Map<String,String> userRsrc =
new ConcurrentHashMap<String,String>();