You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by at...@apache.org on 2011/12/28 19:37:24 UTC
svn commit: r1225264 - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src...
Author: atm
Date: Wed Dec 28 18:37:22 2011
New Revision: 1225264
URL: http://svn.apache.org/viewvc?rev=1225264&view=rev
Log:
Merge trunk into HA branch.
Modified:
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/.gitignore (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/c++/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/data_join/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/index/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/hadoop-mapreduce-project:1152502-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project:1152502-1225257
/hadoop/core/branches/branch-0.19/mapred:713112
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/.gitignore
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/.gitignore:1161333-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/.gitignore:1161333-1225257
/hadoop/core/branches/branch-0.19/mapred/.gitignore:713112
/hadoop/core/trunk/.gitignore:784664-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt Wed Dec 28 18:37:22 2011
@@ -49,6 +49,8 @@ Trunk (unreleased changes)
Move the support for multiple protocols to lower layer so that Writable,
PB and Avro can all use it (Sanjay)
+ MAPREDUCE-2944. Improve checking of input for JobClient.displayTasks() (XieXianshan via harsh)
+
BUG FIXES
MAPREDUCE-3349. Log rack-name in JobHistory for unsuccessful tasks.
(Devaraj K and Amar Kamat via amarrk)
@@ -175,6 +177,12 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar
Vavilapalli via sseth)
+ MAPREDUCE-3399. Modifying ContainerLocalizer to send a heartbeat to NM
+ immediately after downloading a resource instead of always waiting for a
+ second. (Siddarth Seth via vinodkv)
+
+ MAPREDUCE-3568. Optimized Job's progress calculations in MR AM. (vinodkv)
+
BUG FIXES
MAPREDUCE-3221. Reenabled the previously ignored test in TestSubmitJob
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:1161333-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:1161333-1225257
/hadoop/core/branches/branch-0.19/mapred/CHANGES.txt:713112
/hadoop/mapreduce/branches/HDFS-641/CHANGES.txt:817878-835964
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/conf:1152502-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/conf:1152502-1225257
/hadoop/core/branches/branch-0.19/mapred/conf:713112
/hadoop/core/trunk/conf:784664-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Wed Dec 28 18:37:22 2011
@@ -228,7 +228,7 @@ public class MRAppMaster extends Composi
+ recoveryEnabled + " recoverySupportedByCommitter: "
+ recoverySupportedByCommitter + " ApplicationAttemptID: "
+ appAttemptID.getAttemptId());
- dispatcher = new AsyncDispatcher();
+ dispatcher = createDispatcher();
addIfService(dispatcher);
}
@@ -291,6 +291,10 @@ public class MRAppMaster extends Composi
super.init(conf);
} // end of init()
+ protected Dispatcher createDispatcher() {
+ return new AsyncDispatcher();
+ }
+
private OutputCommitter createOutputCommitter(Configuration conf) {
OutputCommitter committer = null;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java Wed Dec 28 18:37:22 2011
@@ -53,6 +53,7 @@ public interface Job {
int getTotalReduces();
int getCompletedMaps();
int getCompletedReduces();
+ float getProgress();
boolean isUber();
String getUserName();
String getQueueName();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Wed Dec 28 18:37:22 2011
@@ -128,6 +128,10 @@ public class JobImpl implements org.apac
private final String username;
private final OutputCommitter committer;
private final Map<JobACL, AccessControlList> jobACLs;
+ private float setupWeight = 0.05f;
+ private float cleanupWeight = 0.05f;
+ private float mapWeight = 0.0f;
+ private float reduceWeight = 0.0f;
private final Set<TaskId> completedTasksFromPreviousRun;
private final List<AMInfo> amInfos;
private final Lock readLock;
@@ -147,7 +151,7 @@ public class JobImpl implements org.apac
private final long appSubmitTime;
private boolean lazyTasksCopyNeeded = false;
- private volatile Map<TaskId, Task> tasks = new LinkedHashMap<TaskId, Task>();
+ volatile Map<TaskId, Task> tasks = new LinkedHashMap<TaskId, Task>();
private Counters jobCounters = newCounters();
// FIXME:
//
@@ -353,6 +357,8 @@ public class JobImpl implements org.apac
private long startTime;
private long finishTime;
private float setupProgress;
+ private float mapProgress;
+ private float reduceProgress;
private float cleanupProgress;
private boolean isUber = false;
@@ -587,30 +593,51 @@ public class JobImpl implements org.apac
cleanupProgress, remoteJobConfFile.toString(), amInfos, isUber);
}
+ computeProgress();
return MRBuilderUtils.newJobReport(jobId, jobName, username, state,
appSubmitTime, startTime, finishTime, setupProgress,
- computeProgress(mapTasks), computeProgress(reduceTasks),
+ this.mapProgress, this.reduceProgress,
cleanupProgress, remoteJobConfFile.toString(), amInfos, isUber);
} finally {
readLock.unlock();
}
}
- private float computeProgress(Set<TaskId> taskIds) {
- readLock.lock();
+ @Override
+ public float getProgress() {
+ this.readLock.lock();
+ try {
+ computeProgress();
+ return (this.setupProgress * this.setupWeight + this.cleanupProgress
+ * this.cleanupWeight + this.mapProgress * this.mapWeight + this.reduceProgress
+ * this.reduceWeight);
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ private void computeProgress() {
+ this.readLock.lock();
try {
- float progress = 0;
- for (TaskId taskId : taskIds) {
- Task task = tasks.get(taskId);
- progress += task.getProgress();
- }
- int taskIdsSize = taskIds.size();
- if (taskIdsSize != 0) {
- progress = progress/taskIdsSize;
+ float mapProgress = 0f;
+ float reduceProgress = 0f;
+ for (Task task : this.tasks.values()) {
+ if (task.getType() == TaskType.MAP) {
+ mapProgress += task.getProgress();
+ } else {
+ reduceProgress += task.getProgress();
+ }
+ }
+ if (this.numMapTasks != 0) {
+ mapProgress = mapProgress / this.numMapTasks;
}
- return progress;
+ if (this.numReduceTasks != 0) {
+ reduceProgress = reduceProgress / this.numReduceTasks;
+ }
+ this.mapProgress = mapProgress;
+ this.reduceProgress = reduceProgress;
} finally {
- readLock.unlock();
+ this.readLock.unlock();
}
}
@@ -731,7 +758,7 @@ public class JobImpl implements org.apac
static JobState checkJobCompleteSuccess(JobImpl job) {
// check for Job success
- if (job.completedTaskCount == job.getTasks().size()) {
+ if (job.completedTaskCount == job.tasks.size()) {
try {
// Commit job & do cleanup
job.getCommitter().commitJob(job.getJobContext());
@@ -970,6 +997,12 @@ public class JobImpl implements org.apac
if (job.numMapTasks == 0 && job.numReduceTasks == 0) {
job.addDiagnostic("No of maps and reduces are 0 " + job.jobId);
+ } else if (job.numMapTasks == 0) {
+ job.reduceWeight = 0.9f;
+ } else if (job.numReduceTasks == 0) {
+ job.mapWeight = 0.9f;
+ } else {
+ job.mapWeight = job.reduceWeight = 0.45f;
}
checkTaskLimits();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java Wed Dec 28 18:37:22 2011
@@ -376,7 +376,7 @@ public abstract class TaskImpl implement
try {
TaskAttempt bestAttempt = selectBestAttempt();
if (bestAttempt == null) {
- return 0;
+ return 0f;
}
return bestAttempt.getProgress();
} finally {
@@ -457,9 +457,10 @@ public abstract class TaskImpl implement
result = at; //The first time around
}
// calculate the best progress
- if (at.getProgress() > progress) {
+ float attemptProgress = at.getProgress();
+ if (attemptProgress > progress) {
result = at;
- progress = at.getProgress();
+ progress = attemptProgress;
}
}
return result;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java Wed Dec 28 18:37:22 2011
@@ -128,25 +128,7 @@ public abstract class RMCommunicator ext
protected float getApplicationProgress() {
// For now just a single job. In future when we have a DAG, we need an
// aggregate progress.
- JobReport report = this.job.getReport();
- float setupWeight = 0.05f;
- float cleanupWeight = 0.05f;
- float mapWeight = 0.0f;
- float reduceWeight = 0.0f;
- int numMaps = this.job.getTotalMaps();
- int numReduces = this.job.getTotalReduces();
- if (numMaps == 0 && numReduces == 0) {
- } else if (numMaps == 0) {
- reduceWeight = 0.9f;
- } else if (numReduces == 0) {
- mapWeight = 0.9f;
- } else {
- mapWeight = reduceWeight = 0.45f;
- }
- return (report.getSetupProgress() * setupWeight
- + report.getCleanupProgress() * cleanupWeight
- + report.getMapProgress() * mapWeight + report.getReduceProgress()
- * reduceWeight);
+ return this.job.getProgress();
}
protected void register() {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java Wed Dec 28 18:37:22 2011
@@ -161,7 +161,7 @@ public abstract class RMContainerRequest
" finishedContainers=" +
response.getCompletedContainersStatuses().size() +
" resourcelimit=" + availableResources +
- "knownNMs=" + clusterNmCount);
+ " knownNMs=" + clusterNmCount);
ask.clear();
release.clear();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java Wed Dec 28 18:37:22 2011
@@ -115,7 +115,8 @@ public class MRApp extends MRAppMaster {
applicationId.setId(0);
}
- public MRApp(int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart) {
+ public MRApp(int maps, int reduces, boolean autoComplete, String testName,
+ boolean cleanOnStart) {
this(maps, reduces, autoComplete, testName, cleanOnStart, 1);
}
@@ -141,10 +142,17 @@ public class MRApp extends MRAppMaster {
return containerId;
}
- public MRApp(int maps, int reduces, boolean autoComplete, String testName,
+ public MRApp(int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, int startCount) {
- super(getApplicationAttemptId(applicationId, startCount), getContainerId(
- applicationId, startCount), NM_HOST, NM_PORT, NM_HTTP_PORT, System
+ this(getApplicationAttemptId(applicationId, startCount), getContainerId(
+ applicationId, startCount), maps, reduces, autoComplete, testName,
+ cleanOnStart, startCount);
+ }
+
+ public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
+ int maps, int reduces, boolean autoComplete, String testName,
+ boolean cleanOnStart, int startCount) {
+ super(appAttemptId, amContainerId, NM_HOST, NM_PORT, NM_HTTP_PORT, System
.currentTimeMillis());
this.testWorkDir = new File("target", testName);
testAbsPath = new Path(testWorkDir.getAbsolutePath());
@@ -205,9 +213,9 @@ public class MRApp extends MRAppMaster {
TaskReport report = task.getReport();
while (!finalState.equals(report.getTaskState()) &&
timeoutSecs++ < 20) {
- System.out.println("Task State is : " + report.getTaskState() +
- " Waiting for state : " + finalState +
- " progress : " + report.getProgress());
+ System.out.println("Task State for " + task.getID() + " is : "
+ + report.getTaskState() + " Waiting for state : " + finalState
+ + " progress : " + report.getProgress());
report = task.getReport();
Thread.sleep(500);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java Wed Dec 28 18:37:22 2011
@@ -426,6 +426,11 @@ public class MockJobs extends MockApps {
}
@Override
+ public float getProgress() {
+ return 0;
+ }
+
+ @Override
public Counters getCounters() {
return counters;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java Wed Dec 28 18:37:22 2011
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -36,15 +37,20 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerFailedEvent;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
import org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator;
@@ -78,6 +84,7 @@ import org.apache.hadoop.yarn.util.Build
import org.junit.After;
import org.junit.Test;
+@SuppressWarnings("unchecked")
public class TestRMContainerAllocator {
static final Log LOG = LogFactory
@@ -338,98 +345,155 @@ public class TestRMContainerAllocator {
}
}
- private static class FakeJob extends JobImpl {
-
- public FakeJob(ApplicationAttemptId appAttemptID, Configuration conf,
- int numMaps, int numReduces) {
- super(MRBuilderUtils.newJobId(appAttemptID.getApplicationId(), 0),
- appAttemptID, conf, null, null, null, null, null, null, null, null,
- true, null, System.currentTimeMillis(), null);
- this.jobId = getID();
- this.numMaps = numMaps;
- this.numReduces = numReduces;
- }
-
- private float setupProgress;
- private float mapProgress;
- private float reduceProgress;
- private float cleanupProgress;
- private final int numMaps;
- private final int numReduces;
- private JobId jobId;
-
- void setProgress(float setupProgress, float mapProgress,
- float reduceProgress, float cleanupProgress) {
- this.setupProgress = setupProgress;
- this.mapProgress = mapProgress;
- this.reduceProgress = reduceProgress;
- this.cleanupProgress = cleanupProgress;
- }
-
- @Override
- public int getTotalMaps() { return this.numMaps; }
- @Override
- public int getTotalReduces() { return this.numReduces;}
-
- @Override
- public JobReport getReport() {
- return MRBuilderUtils.newJobReport(this.jobId, "job", "user",
- JobState.RUNNING, 0, 0, 0, this.setupProgress, this.mapProgress,
- this.reduceProgress, this.cleanupProgress, "jobfile", null, false);
- }
- }
-
@Test
public void testReportedAppProgress() throws Exception {
LOG.info("Running testReportedAppProgress");
Configuration conf = new Configuration();
- MyResourceManager rm = new MyResourceManager(conf);
+ final MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
+ DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext()
.getDispatcher();
// Submit the application
- RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ RMApp rmApp = rm.submitApp(1024);
+ rmDispatcher.await();
- MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 21504);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rmDispatcher.await();
- ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
+ final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rmDispatcher.await();
- FakeJob job = new FakeJob(appAttemptId, conf, 2, 2);
- MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
- appAttemptId, job);
+ MRApp mrApp = new MRApp(appAttemptId, BuilderUtils.newContainerId(
+ appAttemptId, 0), 10, 10, false, this.getClass().getName(), true, 1) {
+ @Override
+ protected Dispatcher createDispatcher() {
+ return new DrainDispatcher();
+ }
+ protected ContainerAllocator createContainerAllocator(
+ ClientService clientService, AppContext context) {
+ return new MyContainerAllocator(rm, appAttemptId, context);
+ };
+ };
+
+ Assert.assertEquals(0.0, rmApp.getProgress(), 0.0);
+
+ mrApp.submit(conf);
+ Job job = mrApp.getContext().getAllJobs().entrySet().iterator().next()
+ .getValue();
+
+ DrainDispatcher amDispatcher = (DrainDispatcher) mrApp.getDispatcher();
+
+ MyContainerAllocator allocator = (MyContainerAllocator) mrApp
+ .getContainerAllocator();
+
+ mrApp.waitForState(job, JobState.RUNNING);
+
+ amDispatcher.await();
+ // Wait till all map-attempts request for containers
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.MAP) {
+ mrApp.waitForState(t.getAttempts().values().iterator().next(),
+ TaskAttemptState.UNASSIGNED);
+ }
+ }
+ amDispatcher.await();
+
+ allocator.schedule();
+ rmDispatcher.await();
+ amNodeManager.nodeHeartbeat(true);
+ rmDispatcher.await();
+ allocator.schedule();
+ rmDispatcher.await();
+
+ // Wait for all map-tasks to be running
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.MAP) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
+ }
allocator.schedule(); // Send heartbeat
- dispatcher.await();
- Assert.assertEquals(0.0, app.getProgress(), 0.0);
+ rmDispatcher.await();
+ Assert.assertEquals(0.05f, job.getProgress(), 0.001f);
+ Assert.assertEquals(0.05f, rmApp.getProgress(), 0.001f);
+
+ // Finish off 1 map.
+ Iterator<Task> it = job.getTasks().values().iterator();
+ finishNextNTasks(mrApp, it, 1);
+ allocator.schedule();
+ rmDispatcher.await();
+ Assert.assertEquals(0.095f, job.getProgress(), 0.001f);
+ Assert.assertEquals(0.095f, rmApp.getProgress(), 0.001f);
- job.setProgress(100, 10, 0, 0);
+ // Finish off 7 more so that map-progress is 80%
+ finishNextNTasks(mrApp, it, 7);
allocator.schedule();
- dispatcher.await();
- Assert.assertEquals(9.5f, app.getProgress(), 0.0);
+ rmDispatcher.await();
+ Assert.assertEquals(0.41f, job.getProgress(), 0.001f);
+ Assert.assertEquals(0.41f, rmApp.getProgress(), 0.001f);
+
+ // Finish off the 2 remaining maps
+ finishNextNTasks(mrApp, it, 2);
+
+ // Wait till all reduce-attempts request for containers
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.REDUCE) {
+ mrApp.waitForState(t.getAttempts().values().iterator().next(),
+ TaskAttemptState.UNASSIGNED);
+ }
+ }
- job.setProgress(100, 80, 0, 0);
allocator.schedule();
- dispatcher.await();
- Assert.assertEquals(41.0f, app.getProgress(), 0.0);
+ rmDispatcher.await();
+ amNodeManager.nodeHeartbeat(true);
+ rmDispatcher.await();
+ allocator.schedule();
+ rmDispatcher.await();
+
+ // Wait for all reduce-tasks to be running
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.REDUCE) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
+ }
+
+ // Finish off 2 reduces
+ finishNextNTasks(mrApp, it, 2);
- job.setProgress(100, 100, 20, 0);
allocator.schedule();
- dispatcher.await();
- Assert.assertEquals(59.0f, app.getProgress(), 0.0);
+ rmDispatcher.await();
+ Assert.assertEquals(0.59f, job.getProgress(), 0.001f);
+ Assert.assertEquals(0.59f, rmApp.getProgress(), 0.001f);
- job.setProgress(100, 100, 100, 100);
+ // Finish off the remaining 8 reduces.
+ finishNextNTasks(mrApp, it, 8);
allocator.schedule();
- dispatcher.await();
- Assert.assertEquals(100.0f, app.getProgress(), 0.0);
+ rmDispatcher.await();
+ // Remaining is JobCleanup
+ Assert.assertEquals(0.95f, job.getProgress(), 0.001f);
+ Assert.assertEquals(0.95f, rmApp.getProgress(), 0.001f);
+ }
+
+ private void finishNextNTasks(MRApp mrApp, Iterator<Task> it, int nextN)
+ throws Exception {
+ Task task;
+ for (int i=0; i<nextN; i++) {
+ task = it.next();
+ finishTask(mrApp, task);
+ }
+ }
+
+ private void finishTask(MRApp mrApp, Task task) throws Exception {
+ TaskAttempt attempt = task.getAttempts().values().iterator().next();
+ mrApp.getContext().getEventHandler().handle(
+ new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_DONE));
+ mrApp.waitForState(task, TaskState.SUCCEEDED);
}
@Test
@@ -438,46 +502,96 @@ public class TestRMContainerAllocator {
LOG.info("Running testReportedAppProgressWithOnlyMaps");
Configuration conf = new Configuration();
- MyResourceManager rm = new MyResourceManager(conf);
+ final MyResourceManager rm = new MyResourceManager(conf);
rm.start();
- DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
+ DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext()
.getDispatcher();
// Submit the application
- RMApp app = rm.submitApp(1024);
- dispatcher.await();
+ RMApp rmApp = rm.submitApp(1024);
+ rmDispatcher.await();
- MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 11264);
amNodeManager.nodeHeartbeat(true);
- dispatcher.await();
+ rmDispatcher.await();
- ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
+ final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt()
.getAppAttemptId();
rm.sendAMLaunched(appAttemptId);
- dispatcher.await();
+ rmDispatcher.await();
- FakeJob job = new FakeJob(appAttemptId, conf, 2, 0);
- MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
- appAttemptId, job);
+ MRApp mrApp = new MRApp(appAttemptId, BuilderUtils.newContainerId(
+ appAttemptId, 0), 10, 0, false, this.getClass().getName(), true, 1) {
+ @Override
+ protected Dispatcher createDispatcher() {
+ return new DrainDispatcher();
+ }
+ protected ContainerAllocator createContainerAllocator(
+ ClientService clientService, AppContext context) {
+ return new MyContainerAllocator(rm, appAttemptId, context);
+ };
+ };
+
+ Assert.assertEquals(0.0, rmApp.getProgress(), 0.0);
+
+ mrApp.submit(conf);
+ Job job = mrApp.getContext().getAllJobs().entrySet().iterator().next()
+ .getValue();
+
+ DrainDispatcher amDispatcher = (DrainDispatcher) mrApp.getDispatcher();
+
+ MyContainerAllocator allocator = (MyContainerAllocator) mrApp
+ .getContainerAllocator();
+
+ mrApp.waitForState(job, JobState.RUNNING);
+
+ amDispatcher.await();
+ // Wait till all map-attempts request for containers
+ for (Task t : job.getTasks().values()) {
+ mrApp.waitForState(t.getAttempts().values().iterator().next(),
+ TaskAttemptState.UNASSIGNED);
+ }
+ amDispatcher.await();
+
+ allocator.schedule();
+ rmDispatcher.await();
+ amNodeManager.nodeHeartbeat(true);
+ rmDispatcher.await();
+ allocator.schedule();
+ rmDispatcher.await();
+
+ // Wait for all map-tasks to be running
+ for (Task t : job.getTasks().values()) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
allocator.schedule(); // Send heartbeat
- dispatcher.await();
- Assert.assertEquals(0.0, app.getProgress(), 0.0);
+ rmDispatcher.await();
+ Assert.assertEquals(0.05f, job.getProgress(), 0.001f);
+ Assert.assertEquals(0.05f, rmApp.getProgress(), 0.001f);
+
+ Iterator<Task> it = job.getTasks().values().iterator();
- job.setProgress(100, 10, 0, 0);
+ // Finish off 1 map so that map-progress is 10%
+ finishNextNTasks(mrApp, it, 1);
allocator.schedule();
- dispatcher.await();
- Assert.assertEquals(14f, app.getProgress(), 0.0);
+ rmDispatcher.await();
+ Assert.assertEquals(0.14f, job.getProgress(), 0.001f);
+ Assert.assertEquals(0.14f, rmApp.getProgress(), 0.001f);
- job.setProgress(100, 60, 0, 0);
+ // Finish off 5 more map so that map-progress is 60%
+ finishNextNTasks(mrApp, it, 5);
allocator.schedule();
- dispatcher.await();
- Assert.assertEquals(59.0f, app.getProgress(), 0.0);
+ rmDispatcher.await();
+ Assert.assertEquals(0.59f, job.getProgress(), 0.001f);
+ Assert.assertEquals(0.59f, rmApp.getProgress(), 0.001f);
- job.setProgress(100, 100, 0, 100);
+ // Finish off remaining map so that map-progress is 100%
+ finishNextNTasks(mrApp, it, 4);
allocator.schedule();
- dispatcher.await();
- Assert.assertEquals(100.0f, app.getProgress(), 0.0);
+ rmDispatcher.await();
+ Assert.assertEquals(0.95f, job.getProgress(), 0.001f);
+ Assert.assertEquals(0.95f, rmApp.getProgress(), 0.001f);
}
@Test
@@ -1000,7 +1114,6 @@ public class TestRMContainerAllocator {
private MyResourceManager rm;
- @SuppressWarnings("rawtypes")
private static AppContext createAppContext(
ApplicationAttemptId appAttemptId, Job job) {
AppContext context = mock(AppContext.class);
@@ -1028,7 +1141,15 @@ public class TestRMContainerAllocator {
return service;
}
- MyContainerAllocator(MyResourceManager rm, Configuration conf,
+ // Use this constructor when using a real job.
+ MyContainerAllocator(MyResourceManager rm,
+ ApplicationAttemptId appAttemptId, AppContext context) {
+ super(createMockClientService(), context);
+ this.rm = rm;
+ }
+
+ // Use this constructor when you are using a mocked job.
+ public MyContainerAllocator(MyResourceManager rm, Configuration conf,
ApplicationAttemptId appAttemptId, Job job) {
super(createMockClientService(), createAppContext(appAttemptId, job));
this.rm = rm;
@@ -1090,6 +1211,7 @@ public class TestRMContainerAllocator {
return result;
}
+ @Override
protected void startAllocatorThread() {
// override to NOT start thread
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java Wed Dec 28 18:37:22 2011
@@ -394,6 +394,11 @@ public class TestRuntimeEstimators {
}
@Override
+ public float getProgress() {
+ return 0;
+ }
+
+ @Override
public Counters getCounters() {
throw new UnsupportedOperationException("Not supported yet.");
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java Wed Dec 28 18:37:22 2011
@@ -65,7 +65,7 @@ public class TestJobImpl {
Task mockTask = mock(Task.class);
Map<TaskId, Task> tasks = new HashMap<TaskId, Task>();
tasks.put(mockTask.getID(), mockTask);
- when(mockJob.getTasks()).thenReturn(tasks);
+ mockJob.tasks = tasks;
when(mockJob.getState()).thenReturn(JobState.ERROR);
JobEvent mockJobEvent = mock(JobEvent.class);
@@ -73,11 +73,12 @@ public class TestJobImpl {
Assert.assertEquals("Incorrect state returned from JobNoTasksCompletedTransition",
JobState.ERROR, state);
}
-
+
@Test
public void testCheckJobCompleteSuccess() {
JobImpl mockJob = mock(JobImpl.class);
+ mockJob.tasks = new HashMap<TaskId, Task>();
OutputCommitter mockCommitter = mock(OutputCommitter.class);
EventHandler mockEventHandler = mock(EventHandler.class);
JobContext mockJobContext = mock(JobContext.class);
@@ -110,7 +111,7 @@ public class TestJobImpl {
Task mockTask = mock(Task.class);
Map<TaskId, Task> tasks = new HashMap<TaskId, Task>();
tasks.put(mockTask.getID(), mockTask);
- when(mockJob.getTasks()).thenReturn(tasks);
+ mockJob.tasks = tasks;
try {
// Just in case the code breaks and reaches these calls
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java Wed Dec 28 18:37:22 2011
@@ -723,6 +723,8 @@ public class JobClient extends CLI {
* @param type the type of the task (map/reduce/setup/cleanup)
* @param state the state of the task
* (pending/running/completed/failed/killed)
+ * @throws IOException when there is an error communicating with the master
+ * @throws IllegalArgumentException if an invalid type/state is passed
*/
public void displayTasks(final JobID jobId, String type, String state)
throws IOException {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java Wed Dec 28 18:37:22 2011
@@ -20,6 +20,9 @@ package org.apache.hadoop.mapreduce.tool
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -56,6 +59,10 @@ import org.apache.hadoop.yarn.logaggrega
public class CLI extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(CLI.class);
protected Cluster cluster;
+ private final Set<String> taskTypes = new HashSet<String>(
+ Arrays.asList("map", "reduce", "setup", "cleanup"));
+ private final Set<String> taskStates = new HashSet<String>(
+ Arrays.asList("pending", "running", "completed", "failed", "killed"));
public CLI() {
}
@@ -545,9 +552,21 @@ public class CLI extends Configured impl
* @param type the type of the task (map/reduce/setup/cleanup)
* @param state the state of the task
* (pending/running/completed/failed/killed)
+ * @throws IOException when there is an error communicating with the master
+ * @throws InterruptedException
+ * @throws IllegalArgumentException if an invalid type/state is passed
*/
protected void displayTasks(Job job, String type, String state)
throws IOException, InterruptedException {
+ if (!taskTypes.contains(type)) {
+ throw new IllegalArgumentException("Invalid type: " + type +
+ ". Valid types for task are: map, reduce, setup, cleanup.");
+ }
+ if (!taskStates.contains(state)) {
+ throw new java.lang.IllegalArgumentException("Invalid state: " + state +
+ ". Valid states for task are: pending, running, completed, failed, killed.");
+ }
+
TaskReport[] reports = job.getTaskReports(TaskType.valueOf(type));
for (TaskReport report : reports) {
TIPStatus status = report.getCurrentStatus();
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1166973-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1166973-1225257
/hadoop/core/branches/branch-0.19/mapred/src/java/mapred-default.xml:713112
/hadoop/core/trunk/src/mapred/mapred-default.xml:776175-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java Wed Dec 28 18:37:22 2011
@@ -136,6 +136,11 @@ public class CompletedJob implements org
}
@Override
+ public float getProgress() {
+ return 1.0f;
+ }
+
+ @Override
public JobState getState() {
return report.getJobState();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java Wed Dec 28 18:37:22 2011
@@ -90,6 +90,11 @@ public class PartialJob implements org.a
}
@Override
+ public float getProgress() {
+ return 1.0f;
+ }
+
+ @Override
public Counters getCounters() {
return null;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java Wed Dec 28 18:37:22 2011
@@ -31,7 +31,9 @@ import java.util.Map;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -163,7 +165,8 @@ public class ContainerLocalizer {
ExecutorService exec = null;
try {
exec = createDownloadThreadPool();
- localizeFiles(nodeManager, exec, ugi);
+ CompletionService<Path> ecs = createCompletionService(exec);
+ localizeFiles(nodeManager, ecs, ugi);
return 0;
} catch (Throwable e) {
// Print traces to stdout so that they can be logged by the NM address
@@ -182,6 +185,10 @@ public class ContainerLocalizer {
.setNameFormat("ContainerLocalizer Downloader").build());
}
+ CompletionService<Path> createCompletionService(ExecutorService exec) {
+ return new ExecutorCompletionService<Path>(exec);
+ }
+
Callable<Path> download(LocalDirAllocator lda, LocalResource rsrc,
UserGroupInformation ugi) throws IOException {
Path destPath = lda.getLocalPathForWrite(".", getEstimatedSize(rsrc), conf);
@@ -206,7 +213,8 @@ public class ContainerLocalizer {
}
private void localizeFiles(LocalizationProtocol nodemanager,
- ExecutorService exec, UserGroupInformation ugi) throws IOException {
+ CompletionService<Path> cs, UserGroupInformation ugi)
+ throws IOException {
while (true) {
try {
LocalizerStatus status = createStatus();
@@ -231,7 +239,7 @@ public class ContainerLocalizer {
break;
}
// TODO: Synchronization??
- pendingResources.put(r, exec.submit(download(lda, r, ugi)));
+ pendingResources.put(r, cs.submit(download(lda, r, ugi)));
}
}
break;
@@ -247,8 +255,7 @@ public class ContainerLocalizer {
} catch (YarnRemoteException e) { }
return;
}
- // TODO HB immediately when rsrc localized
- sleep(1);
+ cs.poll(1000, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
return;
} catch (YarnRemoteException e) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java?rev=1225264&r1=1225263&r2=1225264&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java Wed Dec 28 18:37:22 2011
@@ -40,6 +40,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -146,7 +147,8 @@ public class TestContainerLocalizer {
// return result instantly for deterministic test
ExecutorService syncExec = mock(ExecutorService.class);
- when(syncExec.submit(isA(Callable.class)))
+ CompletionService<Path> cs = mock(CompletionService.class);
+ when(cs.submit(isA(Callable.class)))
.thenAnswer(new Answer<Future<Path>>() {
@Override
public Future<Path> answer(InvocationOnMock invoc)
@@ -159,6 +161,7 @@ public class TestContainerLocalizer {
}
});
doReturn(syncExec).when(localizer).createDownloadThreadPool();
+ doReturn(cs).when(localizer).createCompletionService(syncExec);
// run localization
assertEquals(0, localizer.runLocalization(nmAddr));
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1225257
/hadoop/core/branches/branch-0.19/mapred/src/c++:713112
/hadoop/core/trunk/src/c++:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1225257
/hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
/hadoop/core/trunk/src/contrib:784664-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1225257
/hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112
/hadoop/core/branches/branch-0.19/mapred/src/contrib/block_forensics:713112
/hadoop/core/trunk/src/contrib/block_forensics:784664-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1225257
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112
/hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1225257
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112
/hadoop/core/trunk/src/contrib/build.xml:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1225257
/hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112
/hadoop/core/trunk/src/contrib/data_join:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1225257
/hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112
/hadoop/core/branches/branch-0.19/mapred/src/contrib/eclipse-plugin:713112
/hadoop/core/trunk/src/contrib/eclipse-plugin:776175-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1225257
/hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112
/hadoop/core/trunk/src/contrib/index:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1225257
/hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
/hadoop/core/trunk/src/contrib/vaidya:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1225257
/hadoop/core/branches/branch-0.19/mapred/src/examples:713112
/hadoop/core/trunk/src/examples:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1225257
/hadoop/core/branches/branch-0.19/mapred/src/java:713112
/hadoop/core/trunk/src/mapred:776175-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1152502-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1152502-1225257
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112
/hadoop/core/trunk/src/test/mapred:776175-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1159757-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1159757-1225257
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/fs:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/fs:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1152502-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1152502-1225257
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/hdfs:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/hdfs:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/hdfs:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java:1161333-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java:1161333-1225257
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/FileBench.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1161333-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1161333-1225257
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1159757-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1159757-1225257
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs-with-mr/org/apache/hadoop/ipc:713112
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/ipc:713112
/hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/ipc:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1161333-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1161333-1225257
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1161333-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1161333-1225257
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 28 18:37:22 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1152502-1224959
+/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1152502-1225257
/hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
/hadoop/core/trunk/src/webapps/job:776175-785643