You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2008/10/23 23:06:44 UTC
svn commit: r707473 - in /hadoop/core/trunk: ./
src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/
src/mapred/org/apache/hadoop/mapred/
Author: omalley
Date: Thu Oct 23 14:06:43 2008
New Revision: 707473
URL: http://svn.apache.org/viewvc?rev=707473&view=rev
Log:
HADOOP-4276. Improve the hashing functions and deserialization of the
mapred ID classes. (omalley)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/CommitTaskAction.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/HeartbeatResponse.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ID.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JVMId.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobID.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobProfile.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobStatus.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/KillJobAction.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/KillTaskAction.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTaskStatus.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskAttemptID.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskCompletionEvent.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskID.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskReport.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskStatus.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Oct 23 14:06:43 2008
@@ -49,6 +49,9 @@
HADOOP-4431. Add versionning/tags to Chukwa Chunk.
(Jerome Boulon via Johan)
+ HADOOP-4276. Improve the hashing functions and deserialization of the
+ mapred ID classes. (omalley)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/core/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java (original)
+++ hadoop/core/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java Thu Oct 23 14:06:43 2008
@@ -665,7 +665,7 @@
t = obtainNewTask(taskTracker, j);
if (t != null) {
LOG.debug("Got task from job " +
- j.getJobID().toStringWOPrefix() + " in queue " + qsi.queueName);
+ j.getJobID() + " in queue " + qsi.queueName);
return t;
}
}
@@ -686,7 +686,7 @@
t = obtainNewTask(taskTracker, j);
if (t != null) {
LOG.debug("Getting task from job " +
- j.getJobID().toStringWOPrefix() + " in queue " + qsi.queueName);
+ j.getJobID() + " in queue " + qsi.queueName);
return t;
}
}
@@ -705,7 +705,7 @@
t = obtainNewTask(taskTracker, j);
if (t != null) {
LOG.debug("Getting task from job " +
- j.getJobID().toStringWOPrefix() + " in queue " + qsi.queueName);
+ j.getJobID() + " in queue " + qsi.queueName);
return t;
}
}
@@ -718,7 +718,7 @@
t = obtainNewTask(taskTracker, j);
if (t != null) {
LOG.debug("Getting task from job " +
- j.getJobID().toStringWOPrefix() + " in queue " + qsi.queueName);
+ j.getJobID() + " in queue " + qsi.queueName);
return t;
}
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/CommitTaskAction.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/CommitTaskAction.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/CommitTaskAction.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/CommitTaskAction.java Thu Oct 23 14:06:43 2008
@@ -32,6 +32,7 @@
public CommitTaskAction() {
super(ActionType.COMMIT_TASK);
+ taskId = new TaskAttemptID();
}
public CommitTaskAction(TaskAttemptID taskId) {
@@ -48,6 +49,6 @@
}
public void readFields(DataInput in) throws IOException {
- taskId = TaskAttemptID.read(in);
+ taskId.readFields(in);
}
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/HeartbeatResponse.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/HeartbeatResponse.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/HeartbeatResponse.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/HeartbeatResponse.java Thu Oct 23 14:06:43 2008
@@ -134,7 +134,8 @@
if (size != 0) {
lastKnownIndexMap = new HashMap<JobID, Integer>(size);
for (int i = 0; i < size; ++i) {
- JobID id = JobID.read(in);
+ JobID id = new JobID();
+ id.readFields(in);
int count = in.readInt();
lastKnownIndexMap.put(id, count);
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ID.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ID.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ID.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ID.java Thu Oct 23 14:06:43 2008
@@ -34,6 +34,7 @@
* @see TaskAttemptID
*/
public abstract class ID implements WritableComparable<ID> {
+ protected static final char SEPARATOR = '_';
protected int id;
/** constructs an ID object from the given int */
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JVMId.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JVMId.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JVMId.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JVMId.java Thu Oct 23 14:06:43 2008
@@ -27,7 +27,6 @@
boolean isMap;
JobID jobId;
private static final String JVM = "jvm";
- private static char UNDERSCORE = '_';
private static NumberFormat idFormat = NumberFormat.getInstance();
static {
idFormat.setGroupingUsed(false);
@@ -44,7 +43,9 @@
this(new JobID(jtIdentifier, jobId), isMap, id);
}
- private JVMId() { }
+ public JVMId() {
+ jobId = new JobID();
+ }
public boolean isMapJVM() {
return isMap;
@@ -73,35 +74,41 @@
if(jobComp == 0) {
if(this.isMap == that.isMap) {
return this.id - that.id;
+ } else {
+ return this.isMap ? -1 : 1;
}
- else return this.isMap ? -1 : 1;
+ } else {
+ return jobComp;
}
- else return jobComp;
}
@Override
public String toString() {
- StringBuilder builder = new StringBuilder();
- return builder.append(JVM).append(UNDERSCORE)
- .append(toStringWOPrefix()).toString();
+ return appendTo(new StringBuilder(JVM)).toString();
}
- StringBuilder toStringWOPrefix() {
- StringBuilder builder = new StringBuilder();
- builder.append(jobId.toStringWOPrefix())
- .append(isMap ? "_m_" : "_r_");
- return builder.append(idFormat.format(id));
+ /**
+ * Add the unique id to the given StringBuilder.
+ * @param builder the builder to append to
+ * @return the passed in builder.
+ */
+ protected StringBuilder appendTo(StringBuilder builder) {
+ return jobId.appendTo(builder).
+ append(SEPARATOR).
+ append(isMap ? 'm' : 'r').
+ append(SEPARATOR).
+ append(idFormat.format(id));
}
@Override
public int hashCode() {
- return toStringWOPrefix().toString().hashCode();
+ return jobId.hashCode() * 11 + id;
}
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
- this.jobId = JobID.read(in);
+ this.jobId.readFields(in);
this.isMap = in.readBoolean();
}
@@ -112,12 +119,6 @@
out.writeBoolean(isMap);
}
- public static JVMId read(DataInput in) throws IOException {
- JVMId jvmId = new JVMId();
- jvmId.readFields(in);
- return jvmId;
- }
-
/** Construct a JVMId object from given string
* @return constructed JVMId object or null if the given String is null
* @throws IllegalArgumentException if the given string is malformed
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobID.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobID.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobID.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobID.java Thu Oct 23 14:06:43 2008
@@ -45,9 +45,8 @@
* @see JobTracker#getStartTime()
*/
public class JobID extends ID {
- private static final String JOB = "job";
- private String jtIdentifier;
- private static char UNDERSCORE = '_';
+ protected static final String JOB = "job";
+ private Text jtIdentifier = new Text();
private static NumberFormat idFormat = NumberFormat.getInstance();
static {
@@ -62,13 +61,13 @@
*/
public JobID(String jtIdentifier, int id) {
super(id);
- this.jtIdentifier = jtIdentifier;
+ this.jtIdentifier.set(jtIdentifier);
}
- private JobID() { }
+ public JobID() { }
public String getJtIdentifier() {
- return jtIdentifier;
+ return jtIdentifier.toString();
}
@Override
@@ -93,42 +92,47 @@
@Override
public String toString() {
- StringBuilder builder = new StringBuilder();
- return builder.append(JOB).append(UNDERSCORE)
- .append(toStringWOPrefix()).toString();
+ return appendTo(new StringBuilder(JOB)).toString();
}
-
- /** Returns the string representation w/o prefix */
- StringBuilder toStringWOPrefix() {
- StringBuilder builder = new StringBuilder();
- builder.append(jtIdentifier).append(UNDERSCORE)
- .append(idFormat.format(id)).toString();
+
+ /**
+ * Add the stuff after the "job" prefix to the given builder. This is useful,
+ * because the sub-ids use this substring at the start of their string.
+ * @param builder the builder to append to
+ * @return the builder that was passed in
+ */
+ protected StringBuilder appendTo(StringBuilder builder) {
+ builder.append(SEPARATOR);
+ builder.append(jtIdentifier);
+ builder.append(SEPARATOR);
+ builder.append(idFormat.format(id));
return builder;
}
-
+
@Override
public int hashCode() {
- return toStringWOPrefix().toString().hashCode();
+ return jtIdentifier.hashCode() + id;
}
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
- this.jtIdentifier = Text.readString(in);
+ jtIdentifier.readFields(in);
}
@Override
public void write(DataOutput out) throws IOException {
super.write(out);
- Text.writeString(out, jtIdentifier);
+ jtIdentifier.write(out);
}
+ @Deprecated
public static JobID read(DataInput in) throws IOException {
JobID jobId = new JobID();
jobId.readFields(in);
return jobId;
}
-
+
/** Construct a JobId object from given string
* @return constructed JobId object or null if the given String is null
* @throws IllegalArgumentException if the given string is malformed
@@ -137,7 +141,7 @@
if(str == null)
return null;
try {
- String[] parts = str.split("_");
+ String[] parts = str.split(Character.toString(SEPARATOR));
if(parts.length == 3) {
if(parts[0].equals(JOB)) {
return new JobID(parts[1], Integer.parseInt(parts[2]));
@@ -163,16 +167,23 @@
* @param jobId job number, or null
* @return a regex pattern matching JobIDs
*/
+ @Deprecated
public static String getJobIDsPattern(String jtIdentifier, Integer jobId) {
- StringBuilder builder = new StringBuilder(JOB).append(UNDERSCORE);
+ StringBuilder builder = new StringBuilder(JOB).append(SEPARATOR);
builder.append(getJobIDsPatternWOPrefix(jtIdentifier, jobId));
return builder.toString();
}
- static StringBuilder getJobIDsPatternWOPrefix(String jtIdentifier
- , Integer jobId) {
- StringBuilder builder = new StringBuilder()
- .append(jtIdentifier != null ? jtIdentifier : "[^_]*").append(UNDERSCORE)
+ @Deprecated
+ static StringBuilder getJobIDsPatternWOPrefix(String jtIdentifier,
+ Integer jobId) {
+ StringBuilder builder = new StringBuilder();
+ if (jtIdentifier != null) {
+ builder.append(jtIdentifier);
+ } else {
+ builder.append("[^").append(SEPARATOR).append("]*");
+ }
+ builder.append(SEPARATOR)
.append(jobId != null ? idFormat.format(jobId) : "[0-9]*");
return builder;
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobProfile.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobProfile.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobProfile.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobProfile.java Thu Oct 23 14:06:43 2008
@@ -43,7 +43,7 @@
}
String user;
- JobID jobid;
+ final JobID jobid;
String jobFile;
String url;
String name;
@@ -53,6 +53,7 @@
* Construct an empty {@link JobProfile}.
*/
public JobProfile() {
+ jobid = new JobID();
}
/**
@@ -166,8 +167,9 @@
Text.writeString(out, name);
Text.writeString(out, queueName);
}
+
public void readFields(DataInput in) throws IOException {
- this.jobid = JobID.read(in);
+ jobid.readFields(in);
this.jobFile = Text.readString(in);
this.url = Text.readString(in);
this.user = Text.readString(in);
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobStatus.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobStatus.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobStatus.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobStatus.java Thu Oct 23 14:06:43 2008
@@ -48,7 +48,7 @@
public static final int PREP = 4;
public static final int KILLED = 5;
- private JobID jobid;
+ private final JobID jobid;
private float mapProgress;
private float reduceProgress;
private float cleanupProgress;
@@ -62,6 +62,7 @@
/**
*/
public JobStatus() {
+ jobid = new JobID();
}
/**
@@ -287,7 +288,7 @@
}
public synchronized void readFields(DataInput in) throws IOException {
- this.jobid = JobID.read(in);
+ jobid.readFields(in);
this.setupProgress = in.readFloat();
this.mapProgress = in.readFloat();
this.reduceProgress = in.readFloat();
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/KillJobAction.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/KillJobAction.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/KillJobAction.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/KillJobAction.java Thu Oct 23 14:06:43 2008
@@ -30,10 +30,11 @@
*
*/
class KillJobAction extends TaskTrackerAction {
- JobID jobId;
+ final JobID jobId;
public KillJobAction() {
super(ActionType.KILL_JOB);
+ jobId = new JobID();
}
public KillJobAction(JobID jobId) {
@@ -52,7 +53,7 @@
@Override
public void readFields(DataInput in) throws IOException {
- jobId = JobID.read(in);
+ jobId.readFields(in);
}
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/KillTaskAction.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/KillTaskAction.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/KillTaskAction.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/KillTaskAction.java Thu Oct 23 14:06:43 2008
@@ -29,10 +29,11 @@
*
*/
class KillTaskAction extends TaskTrackerAction {
- TaskAttemptID taskId;
+ final TaskAttemptID taskId;
public KillTaskAction() {
super(ActionType.KILL_TASK);
+ taskId = new TaskAttemptID();
}
public KillTaskAction(TaskAttemptID taskId) {
@@ -51,6 +52,6 @@
@Override
public void readFields(DataInput in) throws IOException {
- taskId = TaskAttemptID.read(in);
+ taskId.readFields(in);
}
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTaskStatus.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTaskStatus.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTaskStatus.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTaskStatus.java Thu Oct 23 14:06:43 2008
@@ -131,7 +131,9 @@
int noFailedFetchTasks = in.readInt();
failedFetchTasks = new ArrayList<TaskAttemptID>(noFailedFetchTasks);
for (int i=0; i < noFailedFetchTasks; ++i) {
- failedFetchTasks.add(TaskAttemptID.read(in));
+ TaskAttemptID id = new TaskAttemptID();
+ id.readFields(in);
+ failedFetchTasks.add(id);
}
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java Thu Oct 23 14:06:43 2008
@@ -104,7 +104,7 @@
////////////////////////////////////////////
private String jobFile; // job configuration file
- private TaskAttemptID taskId; // unique, includes job id
+ private final TaskAttemptID taskId; // unique, includes job id
private int partition; // id within job
TaskStatus taskStatus; // current status of the task
protected boolean cleanupJob = false;
@@ -135,6 +135,7 @@
public Task() {
taskStatus = TaskStatus.createTaskStatus(isMapTask());
+ taskId = new TaskAttemptID();
}
public Task(String jobFile, TaskAttemptID taskId, int partition) {
@@ -262,7 +263,7 @@
}
public void readFields(DataInput in) throws IOException {
jobFile = Text.readString(in);
- taskId = TaskAttemptID.read(in);
+ taskId.readFields(in);
partition = in.readInt();
taskStatus.readFields(in);
this.mapOutputFile.setJobId(taskId.getJobID());
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskAttemptID.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskAttemptID.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskAttemptID.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskAttemptID.java Thu Oct 23 14:06:43 2008
@@ -44,8 +44,7 @@
*/
public class TaskAttemptID extends ID {
private static final String ATTEMPT = "attempt";
- private TaskID taskId;
- private static final char UNDERSCORE = '_';
+ private final TaskID taskId;
/**
* Constructs a TaskAttemptID object from given {@link TaskID}.
@@ -73,7 +72,9 @@
this(new TaskID(jtIdentifier, jobId, isMap, taskId), id);
}
- private TaskAttemptID() { }
+ public TaskAttemptID() {
+ taskId = new TaskID();
+ }
/** Returns the {@link JobID} object that this task attempt belongs to */
public JobID getJobID() {
@@ -114,26 +115,27 @@
}
@Override
public String toString() {
- StringBuilder builder = new StringBuilder();
- return builder.append(ATTEMPT).append(UNDERSCORE)
- .append(toStringWOPrefix()).toString();
+ return appendTo(new StringBuilder(ATTEMPT)).toString();
}
- StringBuilder toStringWOPrefix() {
- StringBuilder builder = new StringBuilder();
- return builder.append(taskId.toStringWOPrefix())
- .append(UNDERSCORE).append(id);
+ /**
+ * Add the unique string to the StringBuilder
+ * @param builder the builder to append ot
+ * @return the builder that was passed in.
+ */
+ protected StringBuilder appendTo(StringBuilder builder) {
+ return taskId.appendTo(builder).append(SEPARATOR).append(id);
}
@Override
public int hashCode() {
- return toStringWOPrefix().toString().hashCode();
+ return taskId.hashCode() * 5 + id;
}
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
- this.taskId = TaskID.read(in);
+ taskId.readFields(in);
}
@Override
@@ -142,6 +144,7 @@
taskId.write(out);
}
+ @Deprecated
public static TaskAttemptID read(DataInput in) throws IOException {
TaskAttemptID taskId = new TaskAttemptID();
taskId.readFields(in);
@@ -152,11 +155,12 @@
* @return constructed TaskAttemptID object or null if the given String is null
* @throws IllegalArgumentException if the given string is malformed
*/
- public static TaskAttemptID forName(String str) throws IllegalArgumentException {
+ public static TaskAttemptID forName(String str
+ ) throws IllegalArgumentException {
if(str == null)
return null;
try {
- String[] parts = str.split("_");
+ String[] parts = str.split(Character.toString(SEPARATOR));
if(parts.length == 6) {
if(parts[0].equals(ATTEMPT)) {
boolean isMap = false;
@@ -167,7 +171,8 @@
isMap, Integer.parseInt(parts[4]), Integer.parseInt(parts[5]));
}
}
- }catch (Exception ex) {//fall below
+ } catch (Exception ex) {
+ //fall below
}
throw new IllegalArgumentException("TaskAttemptId string : " + str
+ " is not properly formed");
@@ -191,20 +196,22 @@
* @param attemptId the task attempt number, or null
* @return a regex pattern matching TaskAttemptIDs
*/
+ @Deprecated
public static String getTaskAttemptIDsPattern(String jtIdentifier,
Integer jobId, Boolean isMap, Integer taskId, Integer attemptId) {
- StringBuilder builder = new StringBuilder(ATTEMPT).append(UNDERSCORE);
+ StringBuilder builder = new StringBuilder(ATTEMPT).append(SEPARATOR);
builder.append(getTaskAttemptIDsPatternWOPrefix(jtIdentifier, jobId,
isMap, taskId, attemptId));
return builder.toString();
}
+ @Deprecated
static StringBuilder getTaskAttemptIDsPatternWOPrefix(String jtIdentifier
, Integer jobId, Boolean isMap, Integer taskId, Integer attemptId) {
StringBuilder builder = new StringBuilder();
builder.append(TaskID.getTaskIDsPatternWOPrefix(jtIdentifier
, jobId, isMap, taskId))
- .append(UNDERSCORE)
+ .append(SEPARATOR)
.append(attemptId != null ? attemptId : "[0-9]*");
return builder;
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskCompletionEvent.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskCompletionEvent.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskCompletionEvent.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskCompletionEvent.java Thu Oct 23 14:06:43 2008
@@ -45,7 +45,10 @@
* Default constructor for Writable.
*
*/
- public TaskCompletionEvent(){}
+ public TaskCompletionEvent(){
+ taskId = new TaskAttemptID();
+ }
+
/**
* Constructor. eventId should be created externally and incremented
* per event for each job.
@@ -220,12 +223,12 @@
}
public void readFields(DataInput in) throws IOException {
- this.taskId = TaskAttemptID.read(in);
- this.idWithinJob = WritableUtils.readVInt(in);
- this.isMap = in.readBoolean();
- this.status = WritableUtils.readEnum(in, Status.class);
- this.taskTrackerHttp = WritableUtils.readString(in);
- this.taskRunTime = WritableUtils.readVInt(in);
- this.eventId = WritableUtils.readVInt(in);
+ taskId.readFields(in);
+ idWithinJob = WritableUtils.readVInt(in);
+ isMap = in.readBoolean();
+ status = WritableUtils.readEnum(in, Status.class);
+ taskTrackerHttp = WritableUtils.readString(in);
+ taskRunTime = WritableUtils.readVInt(in);
+ eventId = WritableUtils.readVInt(in);
}
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskID.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskID.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskID.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskID.java Thu Oct 23 14:06:43 2008
@@ -47,8 +47,7 @@
*/
public class TaskID extends ID {
private static final String TASK = "task";
- private static char UNDERSCORE = '_';
- private static NumberFormat idFormat = NumberFormat.getInstance();
+ private static final NumberFormat idFormat = NumberFormat.getInstance();
static {
idFormat.setGroupingUsed(false);
idFormat.setMinimumIntegerDigits(6);
@@ -83,7 +82,9 @@
this(new JobID(jtIdentifier, jobId), isMap, id);
}
- private TaskID() { }
+ public TaskID() {
+ jobId = new JobID();
+ }
/** Returns the {@link JobID} object that this tip belongs to */
public JobID getJobID() {
@@ -110,39 +111,45 @@
public int compareTo(ID o) {
TaskID that = (TaskID)o;
int jobComp = this.jobId.compareTo(that.jobId);
- if(jobComp == 0) {
- if(this.isMap == that.isMap) {
+ if (jobComp == 0) {
+ if (this.isMap == that.isMap) {
return this.id - that.id;
+ } else {
+ return this.isMap ? -1 : 1;
}
- else return this.isMap ? -1 : 1;
+ } else {
+ return jobComp;
}
- else return jobComp;
}
@Override
public String toString() {
- StringBuilder builder = new StringBuilder();
- return builder.append(TASK).append(UNDERSCORE)
- .append(toStringWOPrefix()).toString();
+ return appendTo(new StringBuilder(TASK)).toString();
}
- StringBuilder toStringWOPrefix() {
- StringBuilder builder = new StringBuilder();
- builder.append(jobId.toStringWOPrefix())
- .append(isMap ? "_m_" : "_r_");
- return builder.append(idFormat.format(id));
+ /**
+ * Add the unique string to the given builder.
+ * @param builder the builder to append to
+ * @return the builder that was passed in
+ */
+ protected StringBuilder appendTo(StringBuilder builder) {
+ return jobId.appendTo(builder).
+ append(SEPARATOR).
+ append(isMap ? 'm' : 'r').
+ append(SEPARATOR).
+ append(idFormat.format(id));
}
@Override
public int hashCode() {
- return toStringWOPrefix().toString().hashCode();
+ return jobId.hashCode() * 524287 + id;
}
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
- this.jobId = JobID.read(in);
- this.isMap = in.readBoolean();
+ jobId.readFields(in);
+ isMap = in.readBoolean();
}
@Override
@@ -151,7 +158,8 @@
jobId.write(out);
out.writeBoolean(isMap);
}
-
+
+ @Deprecated
public static TaskID read(DataInput in) throws IOException {
TaskID tipId = new TaskID();
tipId.readFields(in);
@@ -167,7 +175,7 @@
if(str == null)
return null;
try {
- String[] parts = str.split("_");
+ String[] parts = str.split(Character.toString(SEPARATOR));
if(parts.length == 5) {
if(parts[0].equals(TASK)) {
boolean isMap = false;
@@ -200,19 +208,21 @@
* @param taskId taskId number, or null
* @return a regex pattern matching TaskIDs
*/
+ @Deprecated
public static String getTaskIDsPattern(String jtIdentifier, Integer jobId
, Boolean isMap, Integer taskId) {
- StringBuilder builder = new StringBuilder(TASK).append(UNDERSCORE)
+ StringBuilder builder = new StringBuilder(TASK).append(SEPARATOR)
.append(getTaskIDsPatternWOPrefix(jtIdentifier, jobId, isMap, taskId));
return builder.toString();
}
+ @Deprecated
static StringBuilder getTaskIDsPatternWOPrefix(String jtIdentifier
, Integer jobId, Boolean isMap, Integer taskId) {
StringBuilder builder = new StringBuilder();
builder.append(JobID.getJobIDsPatternWOPrefix(jtIdentifier, jobId))
- .append(UNDERSCORE)
- .append(isMap != null ? (isMap ? "m" : "r") : "(m|r)").append(UNDERSCORE)
+ .append(SEPARATOR)
+ .append(isMap != null ? (isMap ? "m" : "r") : "(m|r)").append(SEPARATOR)
.append(taskId != null ? idFormat.format(taskId) : "[0-9]*");
return builder;
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskReport.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskReport.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskReport.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskReport.java Thu Oct 23 14:06:43 2008
@@ -28,7 +28,7 @@
/** A report on the state of a task. */
public class TaskReport implements Writable {
- private TaskID taskid;
+ private final TaskID taskid;
private float progress;
private String state;
private String[] diagnostics;
@@ -36,7 +36,9 @@
private long finishTime;
private Counters counters;
- public TaskReport() {}
+ public TaskReport() {
+ taskid = new TaskID();
+ }
TaskReport(TaskID taskid, float progress, String state,
String[] diagnostics, long startTime, long finishTime,
@@ -133,11 +135,11 @@
}
public void readFields(DataInput in) throws IOException {
- this.taskid = TaskID.read(in);
- this.progress = in.readFloat();
- this.state = Text.readString(in);
- this.startTime = in.readLong();
- this.finishTime = in.readLong();
+ taskid.readFields(in);
+ progress = in.readFloat();
+ state = Text.readString(in);
+ startTime = in.readLong();
+ finishTime = in.readLong();
diagnostics = WritableUtils.readStringArray(in);
counters = new Counters();
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskStatus.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskStatus.java?rev=707473&r1=707472&r2=707473&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskStatus.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskStatus.java Thu Oct 23 14:06:43 2008
@@ -43,7 +43,7 @@
public static enum State {RUNNING, SUCCEEDED, FAILED, UNASSIGNED, KILLED,
COMMIT_PENDING}
- private TaskAttemptID taskid;
+ private final TaskAttemptID taskid;
private float progress;
private volatile State runState;
private String diagnosticInfo;
@@ -59,7 +59,9 @@
private boolean includeCounters;
private SortedRanges.Range nextRecordRange = new SortedRanges.Range();
- public TaskStatus() {}
+ public TaskStatus() {
+ taskid = new TaskAttemptID();
+ }
public TaskStatus(TaskAttemptID taskid, float progress,
State runState, String diagnosticInfo,
@@ -340,7 +342,7 @@
}
public void readFields(DataInput in) throws IOException {
- this.taskid = TaskAttemptID.read(in);
+ this.taskid.readFields(in);
this.progress = in.readFloat();
this.runState = WritableUtils.readEnum(in, State.class);
this.diagnosticInfo = Text.readString(in);