You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by wa...@apache.org on 2013/10/17 07:33:02 UTC
svn commit: r1532967 [4/4] - in
/hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project: ./ bin/ conf/
dev-support/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-...
Modified: hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java Thu Oct 17 05:32:42 2013
@@ -29,6 +29,8 @@ import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.PrintStream;
+import junit.framework.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -60,6 +62,22 @@ public class TestMRJobClient extends Clu
return job;
}
+ private Job runJobInBackGround(Configuration conf) throws Exception {
+ String input = "hello1\nhello2\nhello3\n";
+
+ Job job = MapReduceTestUtil.createJob(conf, getInputDir(), getOutputDir(),
+ 1, 1, input);
+ job.setJobName("mr");
+ job.setPriority(JobPriority.NORMAL);
+ job.submit();
+ int i = 0;
+ while (i++ < 200 && job.getJobID() == null) {
+ LOG.info("waiting for jobId...");
+ Thread.sleep(100);
+ }
+ return job;
+ }
+
public static int runTool(Configuration conf, Tool tool, String[] args,
OutputStream out) throws Exception {
PrintStream oldOut = System.out;
@@ -108,8 +126,10 @@ public class TestMRJobClient extends Clu
Job job = runJob(conf);
String jobId = job.getJobID().toString();
- // test jobs list
- testJobList(jobId, conf);
+ // test all jobs list
+ testAllJobList(jobId, conf);
+ // test only submitted jobs list
+ testSubmittedJobList(conf);
// test job counter
testGetCounter(jobId, conf);
// status
@@ -131,38 +151,37 @@ public class TestMRJobClient extends Clu
// submit job from file
testSubmit(conf);
// kill a task
- testKillTask(job, conf);
+ testKillTask(conf);
// fail a task
- testfailTask(job, conf);
+ testfailTask(conf);
// kill job
- testKillJob(jobId, conf);
-
+ testKillJob(conf);
}
/**
* test fail task
*/
- private void testfailTask(Job job, Configuration conf) throws Exception {
+ private void testfailTask(Configuration conf) throws Exception {
+ Job job = runJobInBackGround(conf);
CLI jc = createJobClient();
TaskID tid = new TaskID(job.getJobID(), TaskType.MAP, 0);
TaskAttemptID taid = new TaskAttemptID(tid, 1);
ByteArrayOutputStream out = new ByteArrayOutputStream();
- // TaskAttemptId is not set
+ // TaskAttemptId is not set
int exitCode = runTool(conf, jc, new String[] { "-fail-task" }, out);
assertEquals("Exit code", -1, exitCode);
- try {
- runTool(conf, jc, new String[] { "-fail-task", taid.toString() }, out);
- fail(" this task should field");
- } catch (IOException e) {
- // task completed !
- assertTrue(e.getMessage().contains("_0001_m_000000_1"));
- }
+ runTool(conf, jc, new String[] { "-fail-task", taid.toString() }, out);
+ String answer = new String(out.toByteArray(), "UTF-8");
+ Assert
+ .assertTrue(answer.contains("Killed task " + taid + " by failing it"));
}
+
/**
* test a kill task
*/
- private void testKillTask(Job job, Configuration conf) throws Exception {
+ private void testKillTask(Configuration conf) throws Exception {
+ Job job = runJobInBackGround(conf);
CLI jc = createJobClient();
TaskID tid = new TaskID(job.getJobID(), TaskType.MAP, 0);
TaskAttemptID taid = new TaskAttemptID(tid, 1);
@@ -171,20 +190,17 @@ public class TestMRJobClient extends Clu
int exitCode = runTool(conf, jc, new String[] { "-kill-task" }, out);
assertEquals("Exit code", -1, exitCode);
- try {
- runTool(conf, jc, new String[] { "-kill-task", taid.toString() }, out);
- fail(" this task should be killed");
- } catch (IOException e) {
- System.out.println(e);
- // task completed
- assertTrue(e.getMessage().contains("_0001_m_000000_1"));
- }
+ runTool(conf, jc, new String[] { "-kill-task", taid.toString() }, out);
+ String answer = new String(out.toByteArray(), "UTF-8");
+ Assert.assertTrue(answer.contains("Killed task " + taid));
}
/**
* test a kill job
*/
- private void testKillJob(String jobId, Configuration conf) throws Exception {
+ private void testKillJob(Configuration conf) throws Exception {
+ Job job = runJobInBackGround(conf);
+ String jobId = job.getJobID().toString();
CLI jc = createJobClient();
ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -435,7 +451,8 @@ public class TestMRJobClient extends Clu
/**
* print a job list
*/
- protected void testJobList(String jobId, Configuration conf) throws Exception {
+ protected void testAllJobList(String jobId, Configuration conf)
+ throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
// bad options
@@ -458,23 +475,31 @@ public class TestMRJobClient extends Clu
}
assertEquals(1, counter);
out.reset();
- // only submitted
- exitCode = runTool(conf, createJobClient(), new String[] { "-list" }, out);
- assertEquals("Exit code", 0, exitCode);
- br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(
- out.toByteArray())));
+ }
+
+ protected void testSubmittedJobList(Configuration conf) throws Exception {
+ Job job = runJobInBackGround(conf);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ String line;
+ int counter = 0;
+ // only submitted
+ int exitCode =
+ runTool(conf, createJobClient(), new String[] { "-list" }, out);
+ assertEquals("Exit code", 0, exitCode);
+ BufferedReader br =
+ new BufferedReader(new InputStreamReader(new ByteArrayInputStream(
+ out.toByteArray())));
counter = 0;
while ((line = br.readLine()) != null) {
LOG.info("line = " + line);
- if (line.contains(jobId)) {
+ if (line.contains(job.getJobID().toString())) {
counter++;
}
}
// all jobs submitted! no current
assertEquals(1, counter);
-
}
-
+
protected void verifyJobPriority(String jobId, String priority,
Configuration conf, CLI jc) throws Exception {
PipedInputStream pis = new PipedInputStream();
Modified: hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java Thu Oct 17 05:32:42 2013
@@ -39,6 +39,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService;
import org.apache.hadoop.mapreduce.v2.hs.JHSDelegationTokenSecretManager;
import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
@@ -87,10 +88,11 @@ public class TestJHSSecurity {
// no keytab based login
};
+ @Override
protected JHSDelegationTokenSecretManager createJHSSecretManager(
- Configuration conf) {
+ Configuration conf, HistoryServerStateStoreService store) {
return new JHSDelegationTokenSecretManager(initialInterval,
- maxLifetime, renewInterval, 3600000);
+ maxLifetime, renewInterval, 3600000, store);
}
};
// final JobHistoryServer jobHistoryServer = jhServer;
Modified: hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Thu Oct 17 05:32:42 2013
@@ -20,6 +20,9 @@ package org.apache.hadoop.mapreduce.v2;
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -28,6 +31,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.mapred.LocalContainerLauncher;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapreduce.MRConfig;
@@ -35,6 +39,8 @@ import org.apache.hadoop.mapreduce.MRJob
import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
+import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.util.JarFinder;
@@ -43,6 +49,7 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
/**
* Configures and starts the MR-specific components in the YARN cluster.
@@ -67,6 +74,38 @@ public class MiniMRYarnCluster extends M
addService(historyServerWrapper);
}
+ public static String getResolvedMRHistoryWebAppURLWithoutScheme(
+ Configuration conf, boolean isSSLEnabled) {
+ InetSocketAddress address = null;
+ if (isSSLEnabled) {
+ address =
+ conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT);
+ } else {
+ address =
+ conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT); }
+ address = NetUtils.getConnectAddress(address);
+ StringBuffer sb = new StringBuffer();
+ InetAddress resolved = address.getAddress();
+ if (resolved == null || resolved.isAnyLocalAddress() ||
+ resolved.isLoopbackAddress()) {
+ String lh = address.getHostName();
+ try {
+ lh = InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ //Ignore and fallback.
+ }
+ sb.append(lh);
+ } else {
+ sb.append(address.getHostName());
+ }
+ sb.append(":").append(address.getPort());
+ return sb.toString();
+ }
+
@Override
public void serviceInit(Configuration conf) throws Exception {
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
@@ -155,8 +194,8 @@ public class MiniMRYarnCluster extends M
// pick free random ports.
getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
hostname + ":0");
- getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
- hostname + ":0");
+ MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), hostname
+ + ":0");
getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS,
hostname + ":0");
}
@@ -182,17 +221,18 @@ public class MiniMRYarnCluster extends M
//need to do this because historyServer.init creates a new Configuration
getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
- getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
- historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS));
+ MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(),
+ MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig()));
LOG.info("MiniMRYARN ResourceManager address: " +
getConfig().get(YarnConfiguration.RM_ADDRESS));
LOG.info("MiniMRYARN ResourceManager web address: " +
- getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS));
+ WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
LOG.info("MiniMRYARN HistoryServer address: " +
getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
LOG.info("MiniMRYARN HistoryServer web address: " +
- getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS));
+ getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
+ HttpConfig.isSecure()));
}
@Override
Modified: hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java Thu Oct 17 05:32:42 2013
@@ -111,7 +111,7 @@ public class TestMRJobsWithHistoryServic
}
}
- @Test (timeout = 30000)
+ @Test (timeout = 90000)
public void testJobHistoryData() throws IOException, InterruptedException,
AvroRemoteException, ClassNotFoundException {
if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
Modified: hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java Thu Oct 17 05:32:42 2013
@@ -120,6 +120,7 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeHTTPAddress"));
Assert.assertNotNull(n.get("LastHealthUpdate"));
Assert.assertNotNull(n.get("HealthReport"));
+ Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNotNull(n.get("NumContainers"));
Assert.assertEquals(
n.get("NodeId") + ": Unexpected number of used containers",
@@ -156,6 +157,7 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeHTTPAddress"));
Assert.assertNotNull(n.get("LastHealthUpdate"));
Assert.assertNotNull(n.get("HealthReport"));
+ Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNull(n.get("NumContainers"));
Assert.assertNull(n.get("UsedMemoryMB"));
Assert.assertNull(n.get("AvailableMemoryMB"));
Modified: hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Thu Oct 17 05:32:42 2013
@@ -146,7 +146,7 @@ public class ShuffleHandler extends Auxi
private ReadaheadPool readaheadPool = ReadaheadPool.getInstance();
public static final String MAPREDUCE_SHUFFLE_SERVICEID =
- "mapreduce.shuffle";
+ "mapreduce_shuffle";
private static final Map<String,String> userRsrc =
new ConcurrentHashMap<String,String>();
Modified: hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java Thu Oct 17 05:32:42 2013
@@ -213,7 +213,7 @@ public class TeraSort extends Configured
splitPoints = readPartitions(fs, partFile, conf);
trie = buildTrie(splitPoints, 0, splitPoints.length, new Text(), 2);
} catch (IOException ie) {
- throw new IllegalArgumentException("can't read paritions file", ie);
+ throw new IllegalArgumentException("can't read partitions file", ie);
}
}