You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2011/04/08 06:08:35 UTC
svn commit: r1090104 - in /hadoop/mapreduce/trunk: ./
src/contrib/mumak/src/test/org/apache/hadoop/mapred/
src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapreduce/
src/java/org/apache/hadoop/mapreduce/protocol/ src/java/org/apache/hadoo...
Author: tomwhite
Date: Fri Apr 8 04:08:34 2011
New Revision: 1090104
URL: http://svn.apache.org/viewvc?rev=1090104&view=rev
Log:
MAPREDUCE-2337. Remove dependence of public MapReduce API on classes in server package.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/ClusterStatus.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/Cluster.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/State.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Apr 8 04:08:34 2011
@@ -317,6 +317,9 @@ Release 0.22.0 - Unreleased
MAPREDUCE-1159. Limit Job name in web UI to be 80 char long. (Harsh J
Chouraria via szetszwo)
+ MAPREDUCE-2337. Remove dependence of public MapReduce API on classes in
+ server package. (tomwhite)
+
OPTIMIZATIONS
MAPREDUCE-1354. Enhancements to JobTracker for better performance and
Modified: hadoop/mapreduce/trunk/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java Fri Apr 8 04:08:34 2011
@@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.TaskA
import org.apache.hadoop.mapreduce.TaskReport;
import org.apache.hadoop.mapreduce.TaskTrackerInfo;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.tools.rumen.TaskInfo;
import org.apache.hadoop.tools.rumen.MapTaskAttemptInfo;
@@ -384,6 +385,11 @@ public class MockSimulatorJobTracker imp
throws IOException, InterruptedException {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public JobTrackerStatus getJobTrackerStatus() {
+ throw new UnsupportedOperationException();
+ }
@Override
public QueueInfo getQueue(String queueName) throws IOException,
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/ClusterStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/ClusterStatus.java?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/ClusterStatus.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/ClusterStatus.java Fri Apr 8 04:08:34 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapreduce.ClusterMetrics;
import org.apache.hadoop.mapreduce.TaskTrackerInfo;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
/**
* Status information on the current state of the Map-Reduce cluster.
@@ -381,10 +382,21 @@ public class ClusterStatus implements Wr
* as {@link JobTracker.State}
*
* @return the current state of the <code>JobTracker</code>.
+ * @deprecated Use {@link #getJobTrackerStatus()} instead.
*/
+ @Deprecated
public JobTracker.State getJobTrackerState() {
return state;
}
+
+ /**
+ * Get the JobTracker's status.
+ *
+ * @return {@link JobTrackerStatus} of the JobTracker
+ */
+ public JobTrackerStatus getJobTrackerStatus() {
+ return JobTrackerStatus.valueOf(state.name());
+ }
/**
* Gets the list of blacklisted trackers along with reasons for blacklisting.
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Apr 8 04:08:34 2011
@@ -83,6 +83,7 @@ import org.apache.hadoop.mapreduce.MRCon
import org.apache.hadoop.mapreduce.QueueInfo;
import org.apache.hadoop.mapreduce.TaskTrackerInfo;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.security.token.DelegationTokenRenewal;
@@ -90,6 +91,7 @@ import org.apache.hadoop.mapreduce.secur
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.mapreduce.server.jobtracker.State;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
import org.apache.hadoop.mapreduce.util.ConfigUtil;
import org.apache.hadoop.mapreduce.util.MRAsyncDiskService;
@@ -3241,11 +3243,20 @@ public class JobTracker implements MRCon
getBlacklistedTrackerCount(), getExcludedNodes().size()) ;
}
+ /**
+ * @deprecated Use {@link #getJobTrackerStatus()} instead.
+ */
+ @Deprecated
public org.apache.hadoop.mapreduce.server.jobtracker.State
getJobTrackerState() {
return org.apache.hadoop.mapreduce.server.jobtracker.
State.valueOf(state.name());
}
+
+ @Override
+ public JobTrackerStatus getJobTrackerStatus() {
+ return JobTrackerStatus.valueOf(state.name());
+ }
public long getTaskTrackerExpiryInterval() {
return tasktrackerExpiryInterval;
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Fri Apr 8 04:08:34 2011
@@ -48,6 +48,7 @@ import org.apache.hadoop.mapreduce.Queue
import org.apache.hadoop.mapreduce.TaskCompletionEvent;
import org.apache.hadoop.mapreduce.TaskTrackerInfo;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.filecache.TaskDistributedCacheManager;
import org.apache.hadoop.mapreduce.filecache.TrackerDistributedCacheManager;
@@ -659,9 +660,17 @@ public class LocalJobRunner implements C
reduce_tasks, 0, 0, 1, 1, jobs.size(), 1, 0, 0);
}
+ /**
+ * @deprecated Use {@link #getJobTrackerStatus()} instead.
+ */
+ @Deprecated
public State getJobTrackerState() throws IOException, InterruptedException {
return State.RUNNING;
}
+
+ public JobTrackerStatus getJobTrackerStatus() {
+ return JobTrackerStatus.RUNNING;
+ }
public long getTaskTrackerExpiryInterval() throws IOException, InterruptedException {
return 0;
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/Cluster.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/Cluster.java?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/Cluster.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/Cluster.java Fri Apr 8 04:08:34 2011
@@ -52,6 +52,10 @@ import org.apache.hadoop.security.token.
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class Cluster {
+
+ @InterfaceStability.Evolving
+ public static enum JobTrackerStatus {INITIALIZING, RUNNING};
+
private ClientProtocol client;
private UserGroupInformation ugi;
private Configuration conf;
@@ -311,12 +315,26 @@ public class Cluster {
* @return {@link State} of the JobTracker
* @throws IOException
* @throws InterruptedException
+ * @deprecated Use {@link #getJobTrackerStatus()} instead.
*/
+ @Deprecated
public State getJobTrackerState() throws IOException, InterruptedException {
return client.getJobTrackerState();
}
/**
+ * Get the JobTracker's status.
+ *
+ * @return {@link JobTrackerStatus} of the JobTracker
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public JobTrackerStatus getJobTrackerStatus() throws IOException,
+ InterruptedException {
+ return client.getJobTrackerStatus();
+ }
+
+ /**
* Get the tasktracker expiry interval for the cluster
* @return the expiry interval in msec
*/
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java Fri Apr 8 04:08:34 2011
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenSelector;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.ClusterMetrics;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobID;
@@ -111,8 +112,10 @@ public interface ClientProtocol extends
* Version 34: Modified submitJob to use Credentials instead of TokenStorage.
* Version 35: Added the method getQueueAdmins(queueName) as part of
* MAPREDUCE-1664.
+ * Version 36: Added the method getJobTrackerStatus() as part of
+ * MAPREDUCE-2337.
*/
- public static final long versionID = 35L;
+ public static final long versionID = 36L;
/**
* Allocate a name for the job.
@@ -142,9 +145,21 @@ public interface ClientProtocol extends
* @return {@link State} of the JobTracker
* @throws IOException
* @throws InterruptedException
+ * @deprecated Use {@link #getJobTrackerStatus()} instead.
*/
+ @Deprecated
public State getJobTrackerState() throws IOException, InterruptedException;
-
+
+ /**
+ * Get the JobTracker's status.
+ *
+ * @return {@link JobTrackerStatus} of the JobTracker
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public JobTrackerStatus getJobTrackerStatus() throws IOException,
+ InterruptedException;
+
public long getTaskTrackerExpiryInterval() throws IOException,
InterruptedException;
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/State.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/State.java?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/State.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/State.java Fri Apr 8 04:08:34 2011
@@ -19,12 +19,15 @@ package org.apache.hadoop.mapreduce.serv
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
/**
- * Describes the state of JobTracker
+ * Describes the state of JobTracker
+ * @deprecated Use {@link JobTrackerStatus} instead.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
+@Deprecated
public enum State {
INITIALIZING, RUNNING;
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java Fri Apr 8 04:08:34 2011
@@ -29,6 +29,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
@@ -646,8 +647,8 @@ public class MiniMRCluster {
ClusterStatus status = null;
if (jobTracker.isUp()) {
status = jobTracker.getJobTracker().getClusterStatus(false);
- while (jobTracker.isActive() && status.getJobTrackerState()
- == JobTracker.State.INITIALIZING) {
+ while (jobTracker.isActive() && status.getJobTrackerStatus()
+ == JobTrackerStatus.INITIALIZING) {
try {
LOG.info("JobTracker still initializing. Waiting.");
Thread.sleep(1000);
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java Fri Apr 8 04:08:34 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.JobTracker.RecoveryManager;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.MRConfig;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.createQueuesConfigFile;
@@ -131,7 +132,7 @@ public class TestRecoveryManager extends
// check if the jobtracker came up or not
assertEquals("JobTracker crashed!",
- JobTracker.State.RUNNING, status.getJobTrackerState());
+ JobTrackerStatus.RUNNING, status.getJobTrackerStatus());
// assert the no of recovered jobs
assertEquals("No of recovered jobs not correct",
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java?rev=1090104&r1=1090103&r2=1090104&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java Fri Apr 8 04:08:34 2011
@@ -50,6 +50,7 @@ import org.apache.hadoop.io.SequenceFile
import org.apache.hadoop.mapred.SortValidator.RecordStatsChecker.NonSplitableSequenceFileInputFormat;
import org.apache.hadoop.mapred.lib.IdentityMapper;
import org.apache.hadoop.mapred.lib.IdentityReducer;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.util.StringUtils;
@@ -246,7 +247,7 @@ public class UtilsForTests {
while (true) {
try {
ClusterStatus status = jobClient.getClusterStatus();
- while (status.getJobTrackerState() != JobTracker.State.RUNNING) {
+ while (status.getJobTrackerStatus() != JobTrackerStatus.RUNNING) {
waitFor(100);
status = jobClient.getClusterStatus();
}