You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2019/09/10 21:06:03 UTC
[hadoop] branch trunk updated: YARN-9728. Bugfix for escaping
illegal xml characters for Resource Manager REST API. Contributed by Prabhu
Joseph
This is an automated email from the ASF dual-hosted git repository.
eyang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 10144a5 YARN-9728. Bugfix for escaping illegal xml characters for Resource Manager REST API. Contributed by Prabhu Joseph
10144a5 is described below
commit 10144a580e4647eb7d3d59d043608ffbf8cf090d
Author: Eric Yang <ey...@apache.org>
AuthorDate: Tue Sep 10 17:04:39 2019 -0400
YARN-9728. Bugfix for escaping illegal xml characters for Resource Manager REST API.
Contributed by Prabhu Joseph
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++
.../src/main/resources/yarn-default.xml | 9 +++
.../resourcemanager/webapp/RMWebServices.java | 61 +++++++++++++++-
.../server/resourcemanager/webapp/dao/AppInfo.java | 4 ++
.../resourcemanager/webapp/TestRMWebServices.java | 82 ++++++++++++++++++++++
5 files changed, 158 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f140d6f..9c62827 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3960,6 +3960,10 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER =
false;
+ public static final String FILTER_INVALID_XML_CHARS =
+ "yarn.webapp.filter-invalid-xml-chars";
+ public static final boolean DEFAULT_FILTER_INVALID_XML_CHARS = false;
+
// RM and NM CSRF props
public static final String REST_CSRF = "webapp.rest-csrf.";
public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index eb6bf14..b856536 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3794,6 +3794,15 @@
</property>
<property>
+ <name>yarn.webapp.filter-invalid-xml-chars</name>
+ <value>false</value>
+ <description>
+ Flag to enable filter of invalid xml 1.0 characters present in the
+ value of diagnostics field of apps output from RM WebService.
+ </description>
+ </property>
+
+ <property>
<description>
The type of configuration store to use for scheduler configurations.
Default is "file", which uses file based capacity-scheduler.xml to
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index aa29ee6..6a413d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -242,6 +242,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
@VisibleForTesting
boolean isCentralizedNodeLabelConfiguration = true;
private boolean filterAppsByUser = false;
+ private boolean filterInvalidXMLChars = false;
public final static String DELEGATION_TOKEN_HEADER =
"Hadoop-YARN-RM-Delegation-Token";
@@ -257,6 +258,9 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
this.filterAppsByUser = conf.getBoolean(
YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
+ this.filterInvalidXMLChars = conf.getBoolean(
+ YarnConfiguration.FILTER_INVALID_XML_CHARS,
+ YarnConfiguration.DEFAULT_FILTER_INVALID_XML_CHARS);
}
RMWebServices(ResourceManager rm, Configuration conf,
@@ -551,6 +555,38 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
return ni;
}
+ /**
+ * This method ensures that the output String has only
+ * valid XML unicode characters as specified by the
+ * XML 1.0 standard. For reference, please see
+ * <a href="http://www.w3.org/TR/2000/REC-xml-20001006#NT-Char">
+ * the standard</a>.
+ *
+ * @param str The String whose invalid xml characters we want to escape.
+ * @return The str String after escaping invalid xml characters.
+ */
+ public static String escapeInvalidXMLCharacters(String str) {
+ StringBuffer out = new StringBuffer();
+ final int strlen = str.length();
+ final String substitute = "\uFFFD";
+ int idx = 0;
+ while (idx < strlen) {
+ final int cpt = str.codePointAt(idx);
+ idx += Character.isSupplementaryCodePoint(cpt) ? 2 : 1;
+ if ((cpt == 0x9) ||
+ (cpt == 0xA) ||
+ (cpt == 0xD) ||
+ ((cpt >= 0x20) && (cpt <= 0xD7FF)) ||
+ ((cpt >= 0xE000) && (cpt <= 0xFFFD)) ||
+ ((cpt >= 0x10000) && (cpt <= 0x10FFFF))) {
+ out.append(Character.toChars(cpt));
+ } else {
+ out.append(substitute);
+ }
+ }
+ return out.toString();
+ }
+
@GET
@Path(RMWSConsts.APPS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
@@ -629,6 +665,17 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
WebAppUtils.getHttpSchemePrefix(conf), deSelectFields);
allApps.add(app);
}
+
+ if (filterInvalidXMLChars) {
+ final String format = hsr.getHeader(HttpHeaders.ACCEPT);
+ if (format != null &&
+ format.toLowerCase().contains(MediaType.APPLICATION_XML)) {
+ for (AppInfo appInfo : allApps.getApps()) {
+ appInfo.setNote(escapeInvalidXMLCharacters(appInfo.getNote()));
+ }
+ }
+ }
+
return allApps;
}
@@ -985,8 +1032,18 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
DeSelectFields deSelectFields = new DeSelectFields();
deSelectFields.initFields(unselectedFields);
- return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://",
- deSelectFields);
+ AppInfo appInfo = new AppInfo(rm, app, hasAccess(app, hsr),
+ hsr.getScheme() + "://", deSelectFields);
+
+ if (filterInvalidXMLChars) {
+ final String format = hsr.getHeader(HttpHeaders.ACCEPT);
+ if (format != null &&
+ format.toLowerCase().contains(MediaType.APPLICATION_XML)) {
+ appInfo.setNote(escapeInvalidXMLCharacters(appInfo.getNote()));
+ }
+ }
+
+ return appInfo;
}
@GET
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 63b6fe0..a1d701a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -390,6 +390,10 @@ public class AppInfo {
return this.diagnostics;
}
+ public void setNote(String diagnosticsMsg) {
+ this.diagnostics = diagnosticsMsg;
+ }
+
public FinalApplicationStatus getFinalStatus() {
return this.finalStatus;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
index 9e9fe6e..d9d1d9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseS
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -30,11 +31,17 @@ import java.io.File;
import java.io.StringReader;
import java.security.Principal;
import java.util.Arrays;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
import java.util.Set;
+import java.util.HashMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -49,12 +56,21 @@ import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.*;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -870,6 +886,72 @@ public class TestRMWebServices extends JerseyTestBase {
verifyClusterUserInfo(userInfo, "yarn", "admin");
}
+ @Test
+ public void testInvalidXMLChars() throws Exception {
+ ResourceManager mockRM = mock(ResourceManager.class);
+
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationReport appReport = ApplicationReport.newInstance(
+ applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.FAILED, "java.lang.Exception: \u0001", "url",
+ 0, 0, 0, FinalApplicationStatus.FAILED, null, "N/A", 0.53789f, "YARN",
+ null, null, false, Priority.newInstance(0), "high-mem", "high-mem");
+ List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
+ appReports.add(appReport);
+
+ GetApplicationsResponse response = mock(GetApplicationsResponse.class);
+ when(response.getApplicationList()).thenReturn(appReports);
+ ClientRMService clientRMService = mock(ClientRMService.class);
+ when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
+ .thenReturn(response);
+ when(mockRM.getClientRMService()).thenReturn(clientRMService);
+
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getDispatcher()).thenReturn(mock(Dispatcher.class));
+
+ ApplicationSubmissionContext applicationSubmissionContext = mock(
+ ApplicationSubmissionContext.class);
+ when(applicationSubmissionContext.getUnmanagedAM()).thenReturn(true);
+
+ RMApp app = mock(RMApp.class);
+ RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0),
+ 0, 0, new HashMap<>(), new HashMap<>());
+ when(app.getDiagnostics()).thenReturn(
+ new StringBuilder("java.lang.Exception: \u0001"));
+ when(app.getApplicationId()).thenReturn(applicationId);
+ when(app.getUser()).thenReturn("user");
+ when(app.getName()).thenReturn("appname");
+ when(app.getQueue()).thenReturn("queue");
+ when(app.getRMAppMetrics()).thenReturn(appMetrics);
+ when(app.getApplicationSubmissionContext()).thenReturn(
+ applicationSubmissionContext);
+
+ ConcurrentMap<ApplicationId, RMApp> applications =
+ new ConcurrentHashMap<>();
+ applications.put(applicationId, app);
+
+ when(rmContext.getRMApps()).thenReturn(applications);
+ when(mockRM.getRMContext()).thenReturn(rmContext);
+
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.FILTER_INVALID_XML_CHARS, true);
+ RMWebServices webSvc = new RMWebServices(mockRM, conf, mock(
+ HttpServletResponse.class));
+
+ HttpServletRequest mockHsr = mock(HttpServletRequest.class);
+ when(mockHsr.getHeader(HttpHeaders.ACCEPT)).
+ thenReturn(MediaType.APPLICATION_XML);
+ Set<String> emptySet = Collections.unmodifiableSet(Collections.emptySet());
+
+ AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
+ null, null, null, null, null, null, null, emptySet, emptySet, null);
+
+ assertEquals("Incorrect Number of Apps", 1, appsInfo.getApps().size());
+ assertEquals("Invalid XML Characters Present",
+ "java.lang.Exception: \uFFFD", appsInfo.getApps().get(0).getNote());
+ }
+
public void verifyClusterUserInfo(ClusterUserInfo userInfo,
String rmLoginUser, String requestedUser) {
assertEquals("rmLoginUser doesn't match: ",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org