You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2022/10/24 16:35:23 UTC
[hadoop] branch trunk updated: YARN-11345. [Federation] Refactoring Yarn Router's Application Web Page. (#5030)
This is an automated email from the ASF dual-hosted git repository.
inigoiri pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 454157a3844 YARN-11345. [Federation] Refactoring Yarn Router's Application Web Page. (#5030)
454157a3844 is described below
commit 454157a3844cdd6c92ef650af6c3b323cbec88af
Author: slfan1989 <55...@users.noreply.github.com>
AuthorDate: Tue Oct 25 00:35:16 2022 +0800
YARN-11345. [Federation] Refactoring Yarn Router's Application Web Page. (#5030)
---
.../apache/hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../yarn/server/router/webapp/AppsBlock.java | 178 +++++++++++++++------
.../hadoop/yarn/server/router/webapp/AppsPage.java | 11 +-
.../hadoop/yarn/server/router/webapp/NavBlock.java | 21 ++-
.../yarn/server/router/webapp/NodesBlock.java | 1 +
.../yarn/server/router/webapp/RouterWebApp.java | 2 +-
.../server/router/webapp/TestFederationWebApp.java | 18 +++
7 files changed, 178 insertions(+), 54 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 67d9b8512f1..f084adbd9a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -35,6 +35,7 @@ public interface YarnWebParams {
String APP_STATE = "app.state";
String APP_START_TIME_BEGIN = "app.started-time.begin";
String APP_START_TIME_END = "app.started-time.end";
+ String APP_SC = "app.subcluster";
String APPS_NUM = "apps.num";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
index 87f20c81bb8..7f277ae3ae7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
@@ -21,11 +21,18 @@ package org.apache.hadoop.yarn.server.router.webapp;
import static org.apache.commons.text.StringEscapeUtils.escapeHtml4;
import static org.apache.commons.text.StringEscapeUtils.escapeEcmaScript;
import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_SC;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
import com.sun.jersey.api.client.Client;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
@@ -34,34 +41,102 @@ import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
/**
* Applications block for the Router Web UI.
*/
-public class AppsBlock extends HtmlBlock {
+public class AppsBlock extends RouterBlock {
+
private final Router router;
+ private final Configuration conf;
@Inject
AppsBlock(Router router, ViewContext ctx) {
- super(ctx);
+ super(router, ctx);
this.router = router;
+ this.conf = this.router.getConfig();
}
@Override
protected void render(Block html) {
- // Get the applications from the Resource Managers
- Configuration conf = this.router.getConfig();
+
+ boolean isEnabled = isYarnFederationEnabled();
+
+ // Get subClusterName
+ String subClusterName = $(APP_SC);
+ String reqState = $(APP_STATE);
+
+ // We will try to get the subClusterName.
+ // If the subClusterName is not empty,
+ // it means that we need to get the Node list of a subCluster.
+ AppsInfo appsInfo = null;
+ if (subClusterName != null && !subClusterName.isEmpty()) {
+ initSubClusterMetricsOverviewTable(html, subClusterName);
+ appsInfo = getSubClusterAppsInfo(subClusterName, reqState);
+ } else {
+ // Metrics Overview Table
+ html.__(MetricsOverviewTable.class);
+ appsInfo = getYarnFederationAppsInfo(isEnabled);
+ }
+
+ initYarnFederationAppsOfCluster(appsInfo, html);
+ }
+
+ private static String escape(String str) {
+ return escapeEcmaScript(escapeHtml4(str));
+ }
+
+ private AppsInfo getYarnFederationAppsInfo(boolean isEnabled) {
+ if (isEnabled) {
+ String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf);
+ return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY);
+ }
+ return null;
+ }
+
+ private AppsInfo getSubClusterAppsInfo(String subCluster, String states) {
+ try {
+ SubClusterId subClusterId = SubClusterId.newInstance(subCluster);
+ FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
+ SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
+
+ if (subClusterInfo != null) {
+ // Prepare webAddress
+ String webAddress = subClusterInfo.getRMWebServiceAddress();
+ String herfWebAppAddress = "";
+ if (webAddress != null && !webAddress.isEmpty()) {
+ herfWebAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + webAddress;
+ return getSubClusterAppsInfoByWebAddress(herfWebAppAddress, states);
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("get AppsInfo From SubCluster = {} error.", subCluster, e);
+ }
+ return null;
+ }
+
+ private AppsInfo getSubClusterAppsInfoByWebAddress(String webAddress, String states) {
Client client = RouterWebServiceUtil.createJerseyClient(conf);
- String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+ Map<String, String[]> queryParams = new HashMap<>();
+ if (StringUtils.isNotBlank(states)) {
+ queryParams.put("states", new String[]{states});
+ }
AppsInfo apps = RouterWebServiceUtil
- .genericForward(webAppAddress, null, AppsInfo.class, HTTPMethods.GET,
- RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, conf,
- client);
+ .genericForward(webAddress, null, AppsInfo.class, HTTPMethods.GET,
+ RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, queryParams, conf,
+ client);
+ client.destroy();
+ return apps;
+ }
- setTitle("Applications");
+ private void initYarnFederationAppsOfCluster(AppsInfo appsInfo, Block html) {
TBODY<TABLE<Hamlet>> tbody = html.table("#apps").thead()
.tr()
@@ -81,45 +156,18 @@ public class AppsBlock extends HtmlBlock {
// Render the applications
StringBuilder appsTableData = new StringBuilder("[\n");
- for (AppInfo app : apps.getApps()) {
- try {
-
- String percent = String.format("%.1f", app.getProgress() * 100.0F);
- String trackingURL =
- app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
- // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
- appsTableData.append("[\"")
- .append("<a href='").append(trackingURL).append("'>")
- .append(app.getAppId()).append("</a>\",\"")
- .append(escape(app.getUser())).append("\",\"")
- .append(escape(app.getName())).append("\",\"")
- .append(escape(app.getApplicationType())).append("\",\"")
- .append(escape(app.getQueue())).append("\",\"")
- .append(String.valueOf(app.getPriority())).append("\",\"")
- .append(app.getStartTime()).append("\",\"")
- .append(app.getFinishTime()).append("\",\"")
- .append(app.getState()).append("\",\"")
- .append(app.getFinalStatus()).append("\",\"")
- // Progress bar
- .append("<br title='").append(percent).append("'> <div class='")
- .append(C_PROGRESSBAR).append("' title='")
- .append(join(percent, '%')).append("'> ").append("<div class='")
- .append(C_PROGRESSBAR_VALUE).append("' style='")
- .append(join("width:", percent, '%')).append("'> </div> </div>")
- // History link
- .append("\",\"<a href='").append(trackingURL).append("'>")
- .append("History").append("</a>");
- appsTableData.append("\"],\n");
-
- } catch (Exception e) {
- LOG.info(
- "Cannot add application {}: {}", app.getAppId(), e.getMessage());
+
+ if (appsInfo != null && CollectionUtils.isNotEmpty(appsInfo.getApps())) {
+
+ List<String> appInfoList =
+ appsInfo.getApps().stream().map(this::parseAppInfoData).collect(Collectors.toList());
+
+ if (CollectionUtils.isNotEmpty(appInfoList)) {
+ String formattedAppInfo = StringUtils.join(appInfoList, ",");
+ appsTableData.append(formattedAppInfo);
}
}
- if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
- appsTableData.delete(appsTableData.length() - 2,
- appsTableData.length() - 1);
- }
+
appsTableData.append("]");
html.script().$type("text/javascript")
.__("var appsTableData=" + appsTableData).__();
@@ -127,7 +175,39 @@ public class AppsBlock extends HtmlBlock {
tbody.__().__();
}
- private static String escape(String str) {
- return escapeEcmaScript(escapeHtml4(str));
+ private String parseAppInfoData(AppInfo app) {
+ StringBuilder appsDataBuilder = new StringBuilder();
+ try {
+ String percent = String.format("%.1f", app.getProgress() * 100.0F);
+ String trackingURL = app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
+
+ // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
+ appsDataBuilder.append("[\"")
+ .append("<a href='").append(trackingURL).append("'>")
+ .append(app.getAppId()).append("</a>\",\"")
+ .append(escape(app.getUser())).append("\",\"")
+ .append(escape(app.getName())).append("\",\"")
+ .append(escape(app.getApplicationType())).append("\",\"")
+ .append(escape(app.getQueue())).append("\",\"")
+ .append(app.getPriority()).append("\",\"")
+ .append(app.getStartTime()).append("\",\"")
+ .append(app.getFinishTime()).append("\",\"")
+ .append(app.getState()).append("\",\"")
+ .append(app.getFinalStatus()).append("\",\"")
+ // Progress bar
+ .append("<br title='").append(percent).append("'> <div class='")
+ .append(C_PROGRESSBAR).append("' title='")
+ .append(join(percent, '%')).append("'> ").append("<div class='")
+ .append(C_PROGRESSBAR_VALUE).append("' style='")
+ .append(join("width:", percent, '%')).append("'> </div> </div>")
+ // History link
+ .append("\",\"<a href='").append(trackingURL).append("'>")
+ .append("History").append("</a>");
+ appsDataBuilder.append("\"]\n");
+
+ } catch (Exception e) {
+ LOG.warn("Cannot add application {}: {}", app.getAppId(), e.getMessage());
+ }
+ return appsDataBuilder.toString();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
index 12d0b5b4ede..f820aed05f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.yarn.server.router.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_SC;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.yarn.webapp.SubView;
class AppsPage extends RouterView {
@@ -37,9 +39,14 @@ class AppsPage extends RouterView {
setTableStyles(html, "apps", ".queue {width:6em}", ".ui {width:8em}");
// Set the correct title.
+ String subClusterName = $(APP_SC);
String reqState = $(APP_STATE);
- reqState = (reqState == null || reqState.isEmpty() ? "All" : reqState);
- setTitle(sjoin(reqState, "Applications"));
+
+ if(StringUtils.isBlank(subClusterName)){
+ subClusterName = "Federation ";
+ }
+ reqState = (StringUtils.isBlank(reqState) ? "All" : reqState);
+ setTitle(sjoin(subClusterName, reqState, "Applications"));
}
private String appsTableInit() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
index 44a9ab6a512..b1d3b61ab4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
@@ -20,12 +20,12 @@ package org.apache.hadoop.yarn.server.router.webapp;
import com.google.inject.Inject;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.router.Router;
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import java.util.List;
-
/**
* Navigation block for the Router Web UI.
*/
@@ -60,7 +60,24 @@ public class NavBlock extends RouterBlock {
subAppsList1.__().__();
// ### applications info
- mainList.li().a(url("apps"), "Applications").__();
+ Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>> subAppsList2 =
+ mainList.li().a(url("apps"), "Applications").ul();
+
+ subAppsList2.li().__();
+ for (String subClusterId : subClusterIds) {
+ Hamlet.LI<Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>>> subAppsList3 = subAppsList2.
+ li().a(url("apps", subClusterId), subClusterId);
+ Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>>>> subAppsList4 =
+ subAppsList3.ul().$style("padding:0.3em 1em 0.1em 2em");
+ subAppsList4.li().__();
+ for (YarnApplicationState state : YarnApplicationState.values()) {
+ subAppsList4.
+ li().a(url("apps", subClusterId, state.toString()), state.toString()).__();
+ }
+ subAppsList4.li().__().__();
+ subAppsList3.__();
+ }
+ subAppsList2.__().__();
// ### tools
Hamlet.DIV<Hamlet> sectionBefore = mainList.__();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
index 61f72fb2b2f..7e92506e0d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
@@ -116,6 +116,7 @@ public class NodesBlock extends RouterBlock {
.genericForward(webAddress, null, NodesInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf,
client);
+ client.destroy();
return nodes;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
index d9a9a5896ee..ec99415dc3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
@@ -49,7 +49,7 @@ public class RouterWebApp extends WebApp implements YarnWebParams {
route("/", RouterController.class);
route("/cluster", RouterController.class, "about");
route("/about", RouterController.class, "about");
- route("/apps", RouterController.class, "apps");
+ route(pajoin("/apps", APP_SC, APP_STATE), RouterController.class, "apps");
route(pajoin("/nodes", NODE_SC), RouterController.class, "nodes");
route("/federation", RouterController.class, "federation");
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
index 860e266ced1..dad73b206b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
@@ -81,4 +81,22 @@ public class TestFederationWebApp extends TestRouterWebServicesREST {
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
WebAppTests.testPage(NodesPage.class, Router.class, new MockRouter(config));
}
+
+ @Test
+ public void testFederationAppViewEnable()
+ throws InterruptedException, YarnException, IOException {
+ // Test Federation Enabled
+ Configuration config = new YarnConfiguration();
+ config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+ WebAppTests.testPage(AppsPage.class, Router.class, new MockRouter(config));
+ }
+
+ @Test
+ public void testFederationAppViewNotEnable()
+ throws InterruptedException, YarnException, IOException {
+ // Test Federation Not Enabled
+ Configuration config = new YarnConfiguration();
+ config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
+ WebAppTests.testPage(AppsPage.class, Router.class, new MockRouter(config));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org