You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tu...@apache.org on 2012/12/18 14:25:22 UTC
svn commit: r1423426 - in /hadoop/common/branches/branch-1: CHANGES.txt
src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java
src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java
src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
Author: tucu
Date: Tue Dec 18 13:25:20 2012
New Revision: 1423426
URL: http://svn.apache.org/viewvc?rev=1423426&view=rev
Log:
MAPREDUCE-4703. Add the ability to start the MiniMRClientCluster using the configurations used before it is being stopped. (ahmed.radwan via tucu)
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1423426&r1=1423425&r2=1423426&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Dec 18 13:25:20 2012
@@ -139,6 +139,9 @@ Release 1.2.0 - unreleased
HADOOP-9098. Add missing license headers. (Arpit Agarwal via suresh)
+ MAPREDUCE-4703. Add the ability to start the MiniMRClientCluster using the
+ configurations used before it is being stopped. (ahmed.radwan via tucu)
+
OPTIMIZATIONS
HDFS-2533. Backport: Remove needless synchronization on some FSDataSet
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java?rev=1423426&r1=1423425&r2=1423426&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java Tue Dec 18 13:25:20 2012
@@ -31,6 +31,11 @@ public interface MiniMRClientCluster {
public void start() throws IOException;
+ /**
+ * Stop and start back the cluster using the same configuration.
+ */
+ public void restart() throws IOException;
+
public void stop() throws IOException;
public Configuration getConfig() throws IOException;
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java?rev=1423426&r1=1423425&r2=1423426&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java Tue Dec 18 13:25:20 2012
@@ -20,7 +20,11 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
/**
* An adapter for MiniMRCluster providing a MiniMRClientCluster interface. This
@@ -30,6 +34,8 @@ public class MiniMRClusterAdapter implem
private MiniMRCluster miniMRCluster;
+ private static final Log LOG = LogFactory.getLog(MiniMRClusterAdapter.class);
+
public MiniMRClusterAdapter(MiniMRCluster miniMRCluster) {
this.miniMRCluster = miniMRCluster;
}
@@ -50,4 +56,37 @@ public class MiniMRClusterAdapter implem
miniMRCluster.shutdown();
}
+ @Override
+ public void restart() throws IOException {
+ if (!miniMRCluster.getJobTrackerRunner().isActive()) {
+ LOG.warn("Cannot restart the mini cluster, start it first");
+ return;
+ }
+
+ int jobTrackerPort = miniMRCluster.getJobTrackerPort();
+ int taskTrackerPort = getConfig().getInt(
+ "mapred.task.tracker.report.address", 0);
+ int numtaskTrackers = miniMRCluster.getNumTaskTrackers();
+ String namenode = getConfig().get(FileSystem.FS_DEFAULT_NAME_KEY);
+
+ stop();
+
+ // keep retrying to start the cluster for a max. of 30 sec.
+ for (int i = 0; i < 30; i++) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ try {
+ miniMRCluster = new MiniMRCluster(jobTrackerPort, taskTrackerPort,
+ numtaskTrackers, namenode, 1);
+ break;
+ } catch (Exception e) {
+ LOG.info("Retrying to start the cluster");
+ }
+ }
+
+ }
+
}
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java?rev=1423426&r1=1423425&r2=1423426&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java Tue Dec 18 13:25:20 2012
@@ -99,6 +99,29 @@ public class TestMiniMRClientCluster {
validateCounters(job.getCounters(), 5, 25, 5, 5);
}
+ @Test
+ public void testRestart() throws Exception {
+
+ String jobTrackerAddr1 = mrCluster.getConfig().get("mapred.job.tracker");
+ String taskTrackerAddr1 = mrCluster.getConfig().get(
+ "mapred.task.tracker.report.address");
+
+ mrCluster.restart();
+
+ String jobTrackerAddr2 = mrCluster.getConfig().get("mapred.job.tracker");
+ String taskTrackerAddr2 = mrCluster.getConfig().get(
+ "mapred.task.tracker.report.address");
+
+ assertEquals("Address before restart: " + jobTrackerAddr1
+ + " is different from new address: " + jobTrackerAddr2,
+ jobTrackerAddr1, jobTrackerAddr2);
+
+ assertEquals("Address before restart: " + taskTrackerAddr1
+ + " is different from new address: " + taskTrackerAddr2,
+ taskTrackerAddr1, taskTrackerAddr2);
+
+ }
+
private void validateCounters(Counters counters, long mapInputRecords,
long mapOutputRecords, long reduceInputGroups, long reduceOutputRecords) {
assertEquals("MapInputRecords", mapInputRecords, counters.findCounter(