You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2014/08/29 08:21:50 UTC
git commit: YARN-2405. NPE in FairSchedulerAppsBlock. (Tsuyoshi Ozawa
via kasha)
Repository: hadoop
Updated Branches:
refs/heads/trunk 9d6844571 -> fa80ca49b
YARN-2405. NPE in FairSchedulerAppsBlock. (Tsuyoshi Ozawa via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa80ca49
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa80ca49
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa80ca49
Branch: refs/heads/trunk
Commit: fa80ca49bdd741823ff012ddbd7a0f1aecf26195
Parents: 9d68445
Author: Karthik Kambatla <ka...@apache.org>
Authored: Thu Aug 28 23:21:37 2014 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Thu Aug 28 23:21:37 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../webapp/FairSchedulerAppsBlock.java | 4 +
.../webapp/dao/FairSchedulerInfo.java | 15 ++-
.../webapp/TestRMWebAppFairScheduler.java | 96 +++++++++++++++++++-
4 files changed, 114 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa80ca49/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fa4c8c5..fa47c8e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -249,6 +249,8 @@ Release 2.6.0 - UNRELEASED
YARN-2035. FileSystemApplicationHistoryStore should not make working dir
when it already exists. (Jonathan Eagles via zjshen)
+ YARN-2405. NPE in FairSchedulerAppsBlock. (Tsuyoshi Ozawa via kasha)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa80ca49/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index b1aff90..2a1442e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -110,6 +110,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
String percent = String.format("%.1f", appInfo.getProgress());
ApplicationAttemptId attemptId = app.getCurrentAppAttempt().getAppAttemptId();
int fairShare = fsinfo.getAppFairShare(attemptId);
+ if (fairShare == FairSchedulerInfo.INVALID_FAIR_SHARE) {
+ // FairScheduler#applications don't have the entry. Skip it.
+ continue;
+ }
//AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
appsTableData.append("[\"<a href='")
.append(url("app", appInfo.getAppId())).append("'>")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa80ca49/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
index 23f8c01..f97ff8a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
@@ -25,12 +25,14 @@ import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
@XmlRootElement(name = "fairScheduler")
@XmlType(name = "fairScheduler")
@XmlAccessorType(XmlAccessType.FIELD)
public class FairSchedulerInfo extends SchedulerInfo {
+ public static final int INVALID_FAIR_SHARE = -1;
private FairSchedulerQueueInfo rootQueue;
@XmlTransient
@@ -44,9 +46,18 @@ public class FairSchedulerInfo extends SchedulerInfo {
rootQueue = new FairSchedulerQueueInfo(scheduler.getQueueManager().
getRootQueue(), scheduler);
}
-
+
+ /**
+ * Get the fair share assigned to the appAttemptId.
+ * @param appAttemptId
+ * @return The fair share assigned to the appAttemptId,
+ * <code>FairSchedulerInfo#INVALID_FAIR_SHARE</code> if the scheduler does
+ * not know about this application attempt.
+ */
public int getAppFairShare(ApplicationAttemptId appAttemptId) {
- return scheduler.getSchedulerApp(appAttemptId).getFairShare().getMemory();
+ FSAppAttempt fsAppAttempt = scheduler.getSchedulerApp(appAttemptId);
+ return fsAppAttempt == null ?
+ INVALID_FAIR_SHARE : fsAppAttempt.getFairShare().getMemory();
}
public FairSchedulerQueueInfo getRootQueueInfo() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa80ca49/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
index 1de6489..111bf47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
@@ -22,20 +22,29 @@ import com.google.common.collect.Maps;
import com.google.inject.Binder;
import com.google.inject.Injector;
import com.google.inject.Module;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
@@ -75,12 +84,67 @@ public class TestRMWebAppFairScheduler {
WebAppTests.flushOutput(injector);
}
+
+ /**
+ * Testing inconsistent state between AbstractYarnScheduler#applications and
+ * RMContext#applications
+ */
+ @Test
+ public void testFairSchedulerWebAppPageInInconsistentState() {
+ List<RMAppState> appStates = Arrays.asList(
+ RMAppState.NEW,
+ RMAppState.NEW_SAVING,
+ RMAppState.SUBMITTED,
+ RMAppState.RUNNING,
+ RMAppState.FINAL_SAVING,
+ RMAppState.ACCEPTED,
+ RMAppState.FINISHED
+ );
+ final RMContext rmContext = mockRMContext(appStates);
+ Injector injector = WebAppTests.createMockInjector(RMContext.class,
+ rmContext,
+ new Module() {
+ @Override
+ public void configure(Binder binder) {
+ try {
+ ResourceManager mockRmWithFairScheduler =
+ mockRmWithApps(rmContext);
+ binder.bind(ResourceManager.class).toInstance
+ (mockRmWithFairScheduler);
+
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ });
+ FairSchedulerPage fsViewInstance =
+ injector.getInstance(FairSchedulerPage.class);
+ try {
+ fsViewInstance.render();
+ } catch (Exception e) {
+ Assert.fail("Failed to render FairSchedulerPage: " +
+ StringUtils.stringifyException(e));
+ }
+ WebAppTests.flushOutput(injector);
+ }
+
private static RMContext mockRMContext(List<RMAppState> states) {
final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
.newConcurrentMap();
int i = 0;
for (RMAppState state : states) {
- MockRMApp app = new MockRMApp(i, i, state);
+ MockRMApp app = new MockRMApp(i, i, state) {
+ @Override
+ public RMAppMetrics getRMAppMetrics() {
+ return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0);
+ }
+ @Override
+ public YarnApplicationState createApplicationState() {
+ return YarnApplicationState.ACCEPTED;
+ }
+ };
+ RMAppAttempt attempt = mock(RMAppAttempt.class);
+ app.setCurrentAppAttempt(attempt);
applicationsMaps.put(app.getApplicationId(), app);
i++;
}
@@ -113,4 +177,34 @@ public class TestRMWebAppFairScheduler {
fs.init(conf);
return fs;
}
+
+ private static ResourceManager mockRmWithApps(RMContext rmContext) throws
+ IOException {
+ ResourceManager rm = mock(ResourceManager.class);
+ ResourceScheduler rs = mockFairSchedulerWithoutApps(rmContext);
+ when(rm.getResourceScheduler()).thenReturn(rs);
+ when(rm.getRMContext()).thenReturn(rmContext);
+ return rm;
+ }
+
+ private static FairScheduler mockFairSchedulerWithoutApps(RMContext rmContext)
+ throws IOException {
+ FairScheduler fs = new FairScheduler() {
+ @Override
+ public FSAppAttempt getSchedulerApp(ApplicationAttemptId
+ applicationAttemptId) {
+ return null ;
+ }
+ @Override
+ public FSAppAttempt getApplicationAttempt(ApplicationAttemptId
+ applicationAttemptId) {
+ return null;
+ }
+ };
+ FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
+ fs.setRMContext(rmContext);
+ fs.init(conf);
+ return fs;
+ }
+
}