You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by su...@apache.org on 2013/01/11 20:40:35 UTC
svn commit: r1432246 [2/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project: ./ conf/
dev-support/ hadoop-mapreduce-client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
ha...
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java?rev=1432246&r1=1432245&r2=1432246&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java Fri Jan 11 19:40:23 2013
@@ -31,6 +31,11 @@ public interface MiniMRClientCluster {
public void start() throws IOException;
+ /**
+ * Stop and start back the cluster using the same configuration.
+ */
+ public void restart() throws IOException;
+
public void stop() throws IOException;
public Configuration getConfig() throws IOException;
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java?rev=1432246&r1=1432245&r2=1432246&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java Fri Jan 11 19:40:23 2013
@@ -67,6 +67,10 @@ public class MiniMRClientClusterFactory
MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller
.getName(), noOfNMs);
+ job.getConfiguration().set("minimrclientcluster.caller.name",
+ caller.getName());
+ job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number",
+ noOfNMs);
miniMRYarnCluster.init(job.getConfiguration());
miniMRYarnCluster.start();
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java?rev=1432246&r1=1432245&r2=1432246&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java Fri Jan 11 19:40:23 2013
@@ -18,8 +18,13 @@
package org.apache.hadoop.mapred;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.Service.STATE;
/**
* An adapter for MiniMRYarnCluster providing a MiniMRClientCluster interface.
@@ -29,6 +34,8 @@ public class MiniMRYarnClusterAdapter im
private MiniMRYarnCluster miniMRYarnCluster;
+ private static final Log LOG = LogFactory.getLog(MiniMRYarnClusterAdapter.class);
+
public MiniMRYarnClusterAdapter(MiniMRYarnCluster miniMRYarnCluster) {
this.miniMRYarnCluster = miniMRYarnCluster;
}
@@ -48,4 +55,22 @@ public class MiniMRYarnClusterAdapter im
miniMRYarnCluster.stop();
}
+ @Override
+ public void restart() {
+ if (!miniMRYarnCluster.getServiceState().equals(STATE.STARTED)){
+ LOG.warn("Cannot restart the mini cluster, start it first");
+ return;
+ }
+ Configuration oldConf = new Configuration(getConfig());
+ String callerName = oldConf.get("minimrclientcluster.caller.name",
+ this.getClass().getName());
+ int noOfNMs = oldConf.getInt("minimrclientcluster.nodemanagers.number", 1);
+ oldConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
+ oldConf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
+ stop();
+ miniMRYarnCluster = new MiniMRYarnCluster(callerName, noOfNMs);
+ miniMRYarnCluster.init(oldConf);
+ miniMRYarnCluster.start();
+ }
+
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java?rev=1432246&r1=1432245&r2=1432246&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java Fri Jan 11 19:40:23 2013
@@ -32,6 +32,8 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -92,6 +94,65 @@ public class TestMiniMRClientCluster {
}
@Test
+ public void testRestart() throws Exception {
+
+ String rmAddress1 = mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS);
+ String rmAdminAddress1 = mrCluster.getConfig().get(
+ YarnConfiguration.RM_ADMIN_ADDRESS);
+ String rmSchedAddress1 = mrCluster.getConfig().get(
+ YarnConfiguration.RM_SCHEDULER_ADDRESS);
+ String rmRstrackerAddress1 = mrCluster.getConfig().get(
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS);
+ String rmWebAppAddress1 = mrCluster.getConfig().get(
+ YarnConfiguration.RM_WEBAPP_ADDRESS);
+
+ String mrHistAddress1 = mrCluster.getConfig().get(
+ JHAdminConfig.MR_HISTORY_ADDRESS);
+ String mrHistWebAppAddress1 = mrCluster.getConfig().get(
+ JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS);
+
+ mrCluster.restart();
+
+ String rmAddress2 = mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS);
+ String rmAdminAddress2 = mrCluster.getConfig().get(
+ YarnConfiguration.RM_ADMIN_ADDRESS);
+ String rmSchedAddress2 = mrCluster.getConfig().get(
+ YarnConfiguration.RM_SCHEDULER_ADDRESS);
+ String rmRstrackerAddress2 = mrCluster.getConfig().get(
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS);
+ String rmWebAppAddress2 = mrCluster.getConfig().get(
+ YarnConfiguration.RM_WEBAPP_ADDRESS);
+
+ String mrHistAddress2 = mrCluster.getConfig().get(
+ JHAdminConfig.MR_HISTORY_ADDRESS);
+ String mrHistWebAppAddress2 = mrCluster.getConfig().get(
+ JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS);
+
+ assertEquals("Address before restart: " + rmAddress1
+ + " is different from new address: " + rmAddress2, rmAddress1,
+ rmAddress2);
+ assertEquals("Address before restart: " + rmAdminAddress1
+ + " is different from new address: " + rmAdminAddress2,
+ rmAdminAddress1, rmAdminAddress2);
+ assertEquals("Address before restart: " + rmSchedAddress1
+ + " is different from new address: " + rmSchedAddress2,
+ rmSchedAddress1, rmSchedAddress2);
+ assertEquals("Address before restart: " + rmRstrackerAddress1
+ + " is different from new address: " + rmRstrackerAddress2,
+ rmRstrackerAddress1, rmRstrackerAddress2);
+ assertEquals("Address before restart: " + rmWebAppAddress1
+ + " is different from new address: " + rmWebAppAddress2,
+ rmWebAppAddress1, rmWebAppAddress2);
+ assertEquals("Address before restart: " + mrHistAddress1
+ + " is different from new address: " + mrHistAddress2, mrHistAddress1,
+ mrHistAddress2);
+ assertEquals("Address before restart: " + mrHistWebAppAddress1
+ + " is different from new address: " + mrHistWebAppAddress2,
+ mrHistWebAppAddress1, mrHistWebAppAddress2);
+
+ }
+
+ @Test
public void testJob() throws Exception {
final Job job = createJob();
org.apache.hadoop.mapreduce.lib.input.FileInputFormat.setInputPaths(job,
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java?rev=1432246&r1=1432245&r2=1432246&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java Fri Jan 11 19:40:23 2013
@@ -45,7 +45,7 @@ public class TestJobOutputCommitter exte
private static String TEST_ROOT_DIR = new File(System.getProperty(
"test.build.data", "/tmp")
- + "/" + "test-job-cleanup").toString();
+ + "/" + "test-job-output-committer").toString();
private static final String CUSTOM_CLEANUP_FILE_NAME = "_custom_cleanup";
private static final String ABORT_KILLED_FILE_NAME = "_custom_abort_killed";
private static final String ABORT_FAILED_FILE_NAME = "_custom_abort_failed";
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java?rev=1432246&r1=1432245&r2=1432246&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java Fri Jan 11 19:40:23 2013
@@ -151,7 +151,7 @@ public class TestProcfsBasedProcessTree
ProcfsBasedProcessTree p = new ProcfsBasedProcessTree(pid,
ProcessTree.isSetsidAvailable,
ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL);
- p = p.getProcessTree(); // initialize
+ p.updateProcessTree(); // initialize
LOG.info("ProcessTree: " + p.toString());
File leaf = new File(lowestDescendant);
@@ -164,7 +164,7 @@ public class TestProcfsBasedProcessTree
}
}
- p = p.getProcessTree(); // reconstruct
+ p.updateProcessTree(); // reconstruct
LOG.info("ProcessTree: " + p.toString());
// Get the process-tree dump
@@ -203,7 +203,7 @@ public class TestProcfsBasedProcessTree
}
// ProcessTree is gone now. Any further calls should be sane.
- p = p.getProcessTree();
+ p.updateProcessTree();
assertFalse("ProcessTree must have been gone", p.isAlive());
assertTrue("Cumulative vmem for the gone-process is "
+ p.getCumulativeVmem() + " . It should be zero.", p
@@ -336,7 +336,7 @@ public class TestProcfsBasedProcessTree
new ProcfsBasedProcessTree("100", true, 100L,
procfsRootDir.getAbsolutePath());
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// verify cumulative memory
assertEquals("Cumulative virtual memory does not match", 600000L,
@@ -362,7 +362,7 @@ public class TestProcfsBasedProcessTree
writeStatFiles(procfsRootDir, pids, procInfos);
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// verify cumulative cpu time again
cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ?
@@ -409,7 +409,7 @@ public class TestProcfsBasedProcessTree
new ProcfsBasedProcessTree("100", true, 100L,
procfsRootDir.getAbsolutePath());
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// verify cumulative memory
assertEquals("Cumulative memory does not match",
@@ -425,7 +425,7 @@ public class TestProcfsBasedProcessTree
writeStatFiles(procfsRootDir, newPids, newProcInfos);
// check memory includes the new process.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
assertEquals("Cumulative vmem does not include new process",
1200000L, processTree.getCumulativeVmem());
long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
@@ -451,7 +451,7 @@ public class TestProcfsBasedProcessTree
writeStatFiles(procfsRootDir, newPids, newProcInfos);
// refresh process tree
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// processes older than 2 iterations should be same as before.
assertEquals("Cumulative vmem shouldn't have included new processes",
@@ -555,7 +555,7 @@ public class TestProcfsBasedProcessTree
new ProcfsBasedProcessTree("100", true, 100L, procfsRootDir
.getAbsolutePath());
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// Get the process-tree dump
String processTreeDump = processTree.getProcessTreeDump();
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml?rev=1432246&r1=1432245&r2=1432246&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml Fri Jan 11 19:40:23 2013
@@ -194,5 +194,6 @@
<module>hadoop-mapreduce-client-app</module>
<module>hadoop-mapreduce-client-jobclient</module>
<module>hadoop-mapreduce-client-hs</module>
+ <module>hadoop-mapreduce-client-hs-plugins</module>
</modules>
</project>