You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ma...@apache.org on 2011/05/02 00:35:52 UTC
svn commit: r1098425 - in /hadoop/mapreduce/branches/MR-279: ./
yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/
yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/...
Author: mahadev
Date: Sun May 1 22:35:51 2011
New Revision: 1098425
URL: http://svn.apache.org/viewvc?rev=1098425&view=rev
Log:
Fix to send finish application event only when the application is finished (mahadev)
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/events/ApplicationMasterEvents.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Sun May 1 22:35:51 2011
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Fix to send finish application event only when the application is finished (mahadev)
+
RM Restart Phase 2 - Completed the recovery of components in the RM (mahadev)
MAPREDUCE-2434. Metrics for ResourceManager. (Luke Lu via acmurthy)
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Sun May 1 22:35:51 2011
@@ -197,6 +197,8 @@ AMRMProtocol, EventHandler<ASMEvent<Appl
case REMOVE:
responseMap.remove(id);
break;
+ case EXPIRE:
+ responseMap.remove(id);
default:
break;
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationMasterInfo.java Sun May 1 22:35:51 2011
@@ -288,7 +288,7 @@ public class ApplicationMasterInfo imple
masterInfo.handler.handle(new ASMEvent<AMLauncherEventType>(
AMLauncherEventType.CLEANUP, masterInfo));
masterInfo.handler.handle(new ASMEvent<ApplicationTrackerEventType>(
- ApplicationTrackerEventType.REMOVE, masterInfo));
+ ApplicationTrackerEventType.EXPIRE, masterInfo));
masterInfo.numFailed++;
}
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/events/ApplicationMasterEvents.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/events/ApplicationMasterEvents.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/events/ApplicationMasterEvents.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/events/ApplicationMasterEvents.java Sun May 1 22:35:51 2011
@@ -41,7 +41,8 @@ public class ApplicationMasterEvents {
/* event generated for components tracking application adding/tracking/removal */
public enum ApplicationTrackerEventType {
ADD,
- REMOVE
+ REMOVE,
+ EXPIRE
}
public enum ApplicationEventType {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java Sun May 1 22:35:51 2011
@@ -63,8 +63,10 @@ public interface YarnScheduler {
/**
* A submitted application has completed.
* @param applicationId completed application
+ * @param finishApplication true if the application is completed and the
+ * scheduler needs to notify other components of application completion.
*/
- public void removeApplication(ApplicationId applicationId)
+ public void removeApplication(ApplicationId applicationId, boolean finishApplication)
throws IOException;
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Sun May 1 22:35:51 2011
@@ -252,7 +252,7 @@ implements ResourceScheduler, CapacitySc
}
@Override
- public void removeApplication(ApplicationId applicationId)
+ public void removeApplication(ApplicationId applicationId, boolean finishApplication)
throws IOException {
Application application = getApplication(applicationId);
@@ -270,11 +270,11 @@ implements ResourceScheduler, CapacitySc
// Release containers and update queue capacities
processReleasedContainers(application, application.getCurrentContainers());
-
- // Inform all NodeManagers about completion of application
- finishedApplication(applicationId,
- application.getAllNodesForApplication());
-
+ if (finishApplication) {
+ // Inform all NodeManagers about completion of application
+ finishedApplication(applicationId,
+ application.getAllNodesForApplication());
+ }
// Remove from our data-structure
applications.remove(applicationId);
}
@@ -462,7 +462,16 @@ implements ResourceScheduler, CapacitySc
break;
case REMOVE:
try {
- removeApplication(event.getAppContext().getApplicationID());
+ removeApplication(event.getAppContext().getApplicationID(), true);
+ } catch(IOException ie) {
+ LOG.error("Error in removing application", ie);
+ //TODO have to be shutdown the RM in case of this.
+ // do a graceful shutdown.
+ }
+ break;
+ case EXPIRE:
+ try {
+ removeApplication(event.getAppContext().getApplicationID(), false);
} catch(IOException ie) {
LOG.error("Error in removing application", ie);
//TODO have to be shutdown the RM in case of this.
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Sun May 1 22:35:51 2011
@@ -238,7 +238,7 @@ public class FifoScheduler implements Re
}
@Override
- public synchronized void removeApplication(ApplicationId applicationId)
+ public synchronized void removeApplication(ApplicationId applicationId, boolean finishApplication)
throws IOException {
Application application = getApplication(applicationId);
if (application == null) {
@@ -252,11 +252,11 @@ public class FifoScheduler implements Re
// Update metrics
metrics.finishApp(application);
application.finish();
-
- // Let the cluster know that the applications are done
- finishedApplication(applicationId,
- application.getAllNodesForApplication());
-
+ if (finishApplication) {
+ // Let the cluster know that the applications are done
+ finishedApplication(applicationId,
+ application.getAllNodesForApplication());
+ }
// Remove the application
applications.remove(applicationId);
}
@@ -529,12 +529,18 @@ public class FifoScheduler implements Re
break;
case REMOVE:
try {
-
- removeApplication(event.getAppContext().getApplicationID());
+ removeApplication(event.getAppContext().getApplicationID(), true);
} catch(IOException ie) {
LOG.error("Unable to remove application " + event.getAppContext().getApplicationID(), ie);
}
break;
+ case EXPIRE:
+ try {
+ removeApplication(event.getAppContext().getApplicationID(), false);
+ } catch(IOException ie) {
+ LOG.error("Unable to remove application " + event.getAppContext().getApplicationID(), ie);
+ }
+ break;
}
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java Sun May 1 22:35:51 2011
@@ -75,7 +75,7 @@ public class TestAMLaunchFailure extends
}
@Override
- public void removeApplication(ApplicationId applicationId)
+ public void removeApplication(ApplicationId applicationId, boolean finishApplication)
throws IOException {
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java Sun May 1 22:35:51 2011
@@ -85,7 +85,7 @@ public class TestAMRMRPCResponseId exten
}
@Override
- public void removeApplication(ApplicationId applicationId)
+ public void removeApplication(ApplicationId applicationId, boolean finishApplication)
throws IOException {
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Sun May 1 22:35:51 2011
@@ -156,7 +156,7 @@ public class TestAMRestart extends TestC
case ADD:
schedulerAddApplication++;
break;
- case REMOVE:
+ case EXPIRE:
schedulerRemoveApplication++;
LOG.info("REMOVING app : " + schedulerRemoveApplication);
if (schedulerRemoveApplication == maxFailures) {
@@ -172,7 +172,7 @@ public class TestAMRestart extends TestC
}
@Override
- public void removeApplication(ApplicationId applicationId)
+ public void removeApplication(ApplicationId applicationId, boolean finishApplication)
throws IOException {
}
@Override
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java?rev=1098425&r1=1098424&r2=1098425&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java Sun May 1 22:35:51 2011
@@ -96,7 +96,7 @@ public class TestSchedulerNegotiator ext
}
@Override
- public void removeApplication(ApplicationId applicationId)
+ public void removeApplication(ApplicationId applicationId, boolean finishApplication)
throws IOException {
}
@Override