You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2022/05/18 12:09:00 UTC
[hadoop] branch branch-3.3 updated: YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new b4550b33564 YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke
b4550b33564 is described below
commit b4550b3356439f012ca6f97383889e236067e689
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed May 18 14:08:41 2022 +0200
YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke
---
.../hadoop/yarn/client/api/impl/AHSv2ClientImpl.java | 5 ++---
.../client/api/impl/TimelineReaderClientImpl.java | 16 ++++++++++++++--
.../api/impl/TestTimelineReaderClientImpl.java | 20 +++++++++++++++++++-
3 files changed, 35 insertions(+), 6 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
index b6a0c591c90..5fd3ee21fc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
@@ -137,9 +137,8 @@ public class AHSv2ClientImpl extends AHSClient {
ApplicationId appId = applicationAttemptId.getApplicationId();
ApplicationReport appReport = getApplicationReport(appId);
Map<String, String> filters = new HashMap<>();
- filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
- applicationAttemptId.toString() +
- "\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}");
+ filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq "
+ + "{\"type\":\"YARN_APPLICATION_ATTEMPT\",\"id\":\"" + applicationAttemptId + "\"}");
List<TimelineEntity> entities = readerClient.getContainerEntities(
appId, "ALL", filters, 0, null);
List<ContainerReport> containers =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
index d5af8a0df56..de30fb84be0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
@@ -38,7 +38,10 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -208,12 +211,21 @@ public class TimelineReaderClientImpl extends TimelineReaderClient {
return Arrays.asList(entity);
}
+ @VisibleForTesting
+ protected String encodeValue(String value) throws UnsupportedEncodingException {
+ // Since URLEncoder doesn't use and doesn't have an option for percent-encoding
+ // (as specified in RFC 3986) the spaces are encoded to + signs, which need to be replaced
+ // manually
+ return URLEncoder.encode(value, StandardCharsets.UTF_8.toString())
+ .replaceAll("\\+", "%20");
+ }
+
private void mergeFilters(MultivaluedMap<String, String> defaults,
- Map<String, String> filters) {
+ Map<String, String> filters) throws UnsupportedEncodingException {
if (filters != null && !filters.isEmpty()) {
for (Map.Entry<String, String> entry : filters.entrySet()) {
if (!defaults.containsKey(entry.getKey())) {
- defaults.add(entry.getKey(), filters.get(entry.getValue()));
+ defaults.add(entry.getKey(), encodeValue(entry.getValue()));
}
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
index f668472256a..757aeb8c31d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.when;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -47,6 +48,8 @@ import java.util.List;
*/
public class TestTimelineReaderClientImpl {
+ private final String appAttemptInfoFilter = "{\"type\":\"YARN_APPLICATION_ATTEMPT\"," +
+ "\"id\":\"appattempt_1234_0001_000001\"}";
private TimelineReaderClient client;
@Before
@@ -107,6 +110,16 @@ public class TestTimelineReaderClientImpl {
Assert.assertEquals("mockContainer2", entities.get(1).getId());
}
+ @Test
+ public void testGetContainersForAppAttempt() throws Exception {
+ ApplicationId appId =
+ ApplicationId.fromString("application_1234_0001");
+ List<TimelineEntity> entities = client.getContainerEntities(appId,
+ null, ImmutableMap.of("infofilters", appAttemptInfoFilter), 0, null);
+ Assert.assertEquals(2, entities.size());
+ Assert.assertEquals("mockContainer4", entities.get(1).getId());
+ }
+
@After
public void tearDown() {
if (client != null) {
@@ -135,11 +148,15 @@ public class TestTimelineReaderClientImpl {
protected ClientResponse doGetUri(URI base, String path,
MultivaluedMap<String, String> params) throws IOException {
ClientResponse mockClientResponse = mock(ClientResponse.class);
- if (path.contains(YARN_CONTAINER.toString())) {
+ if (path.contains(YARN_CONTAINER.toString()) && !params.containsKey("infofilters")) {
when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
createTimelineEntity("mockContainer1"));
when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
createTimelineEntities("mockContainer1", "mockContainer2"));
+ } else if (path.contains(YARN_CONTAINER.toString()) && params.containsKey("infofilters")) {
+ Assert.assertEquals(encodeValue(appAttemptInfoFilter), params.get("infofilters").get(0));
+ when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
+ createTimelineEntities("mockContainer3", "mockContainer4"));
} else if (path.contains(YARN_APPLICATION_ATTEMPT.toString())) {
when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
createTimelineEntity("mockAppAttempt1"));
@@ -151,6 +168,7 @@ public class TestTimelineReaderClientImpl {
when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
createTimelineEntities("mockApp1", "mockApp2"));
}
+
return mockClientResponse;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org