You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2014/11/17 22:16:36 UTC
hadoop git commit: YARN-2414. RM web UI: app page will crash if app
is failed before any attempt has been created. Contributed by Wangda Tan
Repository: hadoop
Updated Branches:
refs/heads/trunk bd8196e85 -> 81c9d17af
YARN-2414. RM web UI: app page will crash if app is failed before any attempt has been created. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81c9d17a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81c9d17a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81c9d17a
Branch: refs/heads/trunk
Commit: 81c9d17af84ed87b9ded7057cb726a3855ddd32d
Parents: bd8196e
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Nov 17 21:13:22 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Nov 17 21:15:48 2014 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/security/ApplicationACLsManager.java | 7 ++
.../security/QueueACLsManager.java | 7 ++
.../server/resourcemanager/webapp/AppBlock.java | 30 +++++--
.../resourcemanager/webapp/TestAppPage.java | 90 ++++++++++++++++++++
5 files changed, 130 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81c9d17a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b303e53..0f526dd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -99,6 +99,9 @@ Release 2.7.0 - UNRELEASED
YARN-1703. Fixed ResourceManager web-proxy to close connections correctly.
(Rohith Sharma via vinodkv)
+ YARN-2414. RM web UI: app page will crash if app is failed before any
+ attempt has been created (Wangda Tan via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81c9d17a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
index e8e3cb5..4cacfca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AdminACLsManager;
+import com.google.common.annotations.VisibleForTesting;
+
@InterfaceAudience.Private
public class ApplicationACLsManager {
@@ -48,6 +50,11 @@ public class ApplicationACLsManager {
private final ConcurrentMap<ApplicationId, Map<ApplicationAccessType, AccessControlList>> applicationACLS
= new ConcurrentHashMap<ApplicationId, Map<ApplicationAccessType, AccessControlList>>();
+ @VisibleForTesting
+ public ApplicationACLsManager() {
+ this(new Configuration());
+ }
+
public ApplicationACLsManager(Configuration conf) {
this.conf = conf;
this.adminAclsManager = new AdminACLsManager(this.conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81c9d17a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
index eb5037a..fb8279d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
@@ -24,9 +24,16 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import com.google.common.annotations.VisibleForTesting;
+
public class QueueACLsManager {
private ResourceScheduler scheduler;
private boolean isACLsEnable;
+
+ @VisibleForTesting
+ public QueueACLsManager() {
+ this(null, new Configuration());
+ }
public QueueACLsManager(ResourceScheduler scheduler, Configuration conf) {
this.scheduler = scheduler;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81c9d17a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
index c427ccf..a108e43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
@@ -113,8 +115,23 @@ public class AppBlock extends HtmlBlock {
setTitle(join("Application ", aid));
RMAppMetrics appMerics = rmApp.getRMAppMetrics();
- RMAppAttemptMetrics attemptMetrics =
- rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
+
+ // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
+ // null. In that case, we will assume resource preempted and number of Non
+ // AM container preempted on that attempt is 0
+ RMAppAttemptMetrics attemptMetrics;
+ if (null == rmApp.getCurrentAppAttempt()) {
+ attemptMetrics = null;
+ } else {
+ attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
+ }
+ Resource attemptResourcePreempted =
+ attemptMetrics == null ? Resources.none() : attemptMetrics
+ .getResourcePreempted();
+ int attemptNumNonAMContainerPreempted =
+ attemptMetrics == null ? 0 : attemptMetrics
+ .getNumNonAMContainersPreempted();
+
info("Application Overview")
._("User:", app.getUser())
._("Name:", app.getName())
@@ -143,13 +160,12 @@ public class AppBlock extends HtmlBlock {
._("Total Number of AM Containers Preempted:",
String.valueOf(appMerics.getNumAMContainersPreempted()))
._("Resource Preempted from Current Attempt:",
- attemptMetrics.getResourcePreempted())
+ attemptResourcePreempted)
._("Number of Non-AM Containers Preempted from Current Attempt:",
- String.valueOf(attemptMetrics
- .getNumNonAMContainersPreempted()))
+ attemptNumNonAMContainerPreempted)
._("Aggregate Resource Allocation:",
- String.format("%d MB-seconds, %d vcore-seconds",
- appMerics.getMemorySeconds(), appMerics.getVcoreSeconds()));
+ String.format("%d MB-seconds, %d vcore-seconds",
+ appMerics.getMemorySeconds(), appMerics.getVcoreSeconds()));
pdiv._();
Collection<RMAppAttempt> attempts = rmApp.getAppAttempts().values();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81c9d17a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
new file mode 100644
index 0000000..9732c19
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.junit.Test;
+
+import com.google.inject.Binder;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+public class TestAppPage {
+ @Test
+ public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
+ final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0);
+ Injector injector;
+
+ // init app
+ RMApp app = mock(RMApp.class);
+ when(app.getTrackingUrl()).thenReturn("http://host:123");
+ when(app.getState()).thenReturn(RMAppState.FAILED);
+ when(app.getApplicationId()).thenReturn(APP_ID);
+ when(app.getApplicationType()).thenReturn("Type");
+ when(app.getUser()).thenReturn("user");
+ when(app.getName()).thenReturn("Name");
+ when(app.getQueue()).thenReturn("queue");
+ when(app.getDiagnostics()).thenReturn(new StringBuilder());
+ when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
+ when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
+ when(app.getStartTime()).thenReturn(0L);
+ when(app.getFinishTime()).thenReturn(0L);
+ when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
+
+ RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
+ when(app.getRMAppMetrics()).thenReturn(appMetrics);
+
+ // initialize RM Context, and create RMApp, without creating RMAppAttempt
+ final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
+ rmContext.getRMApps().put(APP_ID, app);
+
+ injector =
+ WebAppTests.createMockInjector(RMContext.class, rmContext,
+ new Module() {
+ @Override
+ public void configure(Binder binder) {
+ try {
+ binder.bind(ResourceManager.class).toInstance(
+ TestRMWebApp.mockRm(rmContext));
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ });
+
+ AppBlock instance = injector.getInstance(AppBlock.class);
+ instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString());
+ instance.render();
+ }
+}