You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bt...@apache.org on 2022/04/12 14:31:26 UTC
[hadoop] branch YARN-10850 created (now 4b7e83625db)
This is an automated email from the ASF dual-hosted git repository.
bteke pushed a change to branch YARN-10850
in repository https://gitbox.apache.org/repos/asf/hadoop.git
at 4b7e83625db YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one
This branch includes the following new commits:
new 4b7e83625db YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/01: YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bteke pushed a commit to branch YARN-10850
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 4b7e83625dbabeae326ec69e60cd46bc17692dd4
Author: Benjamin Teke <bt...@cloudera.com>
AuthorDate: Tue Apr 12 16:30:55 2022 +0200
YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one
---
.../hadoop/yarn/client/api/impl/AHSv2ClientImpl.java | 5 ++---
.../client/api/impl/TimelineReaderClientImpl.java | 16 ++++++++++++++--
.../api/impl/TestTimelineReaderClientImpl.java | 20 +++++++++++++++++++-
3 files changed, 35 insertions(+), 6 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
index e3653baa2b7..26f45baaee0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
@@ -137,9 +137,8 @@ public class AHSv2ClientImpl extends AHSClient {
ApplicationId appId = applicationAttemptId.getApplicationId();
ApplicationReport appReport = getApplicationReport(appId);
Map<String, String> filters = new HashMap<>();
- filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
- applicationAttemptId.toString() +
- "\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}");
+ filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq "
+ + "{\"type\":\"YARN_APPLICATION_ATTEMPT\",\"id\":\"" + applicationAttemptId + "\"}");
List<TimelineEntity> entities = readerClient.getContainerEntities(
appId, "ALL", filters, 0, null);
List<ContainerReport> containers =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
index 29609b955fb..71bf13220b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
@@ -38,7 +38,10 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -208,12 +211,21 @@ public class TimelineReaderClientImpl extends TimelineReaderClient {
return Arrays.asList(entity);
}
+ @VisibleForTesting
+ protected String encodeValue(String value) throws UnsupportedEncodingException {
+ // Since URLEncoder doesn't use and doesn't have an option for percent-encoding
+ // (as specified in RFC 3986) the spaces are encoded to + signs, which need to be replaced
+ // manually
+ return URLEncoder.encode(value, StandardCharsets.UTF_8.toString())
+ .replaceAll("\\+", "%20");
+ }
+
private void mergeFilters(MultivaluedMap<String, String> defaults,
- Map<String, String> filters) {
+ Map<String, String> filters) throws UnsupportedEncodingException {
if (filters != null && !filters.isEmpty()) {
for (Map.Entry<String, String> entry : filters.entrySet()) {
if (!defaults.containsKey(entry.getKey())) {
- defaults.add(entry.getKey(), filters.get(entry.getValue()));
+ defaults.add(entry.getKey(), encodeValue(entry.getValue()));
}
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
index f668472256a..757aeb8c31d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.when;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -47,6 +48,8 @@ import java.util.List;
*/
public class TestTimelineReaderClientImpl {
+ private final String appAttemptInfoFilter = "{\"type\":\"YARN_APPLICATION_ATTEMPT\"," +
+ "\"id\":\"appattempt_1234_0001_000001\"}";
private TimelineReaderClient client;
@Before
@@ -107,6 +110,16 @@ public class TestTimelineReaderClientImpl {
Assert.assertEquals("mockContainer2", entities.get(1).getId());
}
+ @Test
+ public void testGetContainersForAppAttempt() throws Exception {
+ ApplicationId appId =
+ ApplicationId.fromString("application_1234_0001");
+ List<TimelineEntity> entities = client.getContainerEntities(appId,
+ null, ImmutableMap.of("infofilters", appAttemptInfoFilter), 0, null);
+ Assert.assertEquals(2, entities.size());
+ Assert.assertEquals("mockContainer4", entities.get(1).getId());
+ }
+
@After
public void tearDown() {
if (client != null) {
@@ -135,11 +148,15 @@ public class TestTimelineReaderClientImpl {
protected ClientResponse doGetUri(URI base, String path,
MultivaluedMap<String, String> params) throws IOException {
ClientResponse mockClientResponse = mock(ClientResponse.class);
- if (path.contains(YARN_CONTAINER.toString())) {
+ if (path.contains(YARN_CONTAINER.toString()) && !params.containsKey("infofilters")) {
when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
createTimelineEntity("mockContainer1"));
when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
createTimelineEntities("mockContainer1", "mockContainer2"));
+ } else if (path.contains(YARN_CONTAINER.toString()) && params.containsKey("infofilters")) {
+ Assert.assertEquals(encodeValue(appAttemptInfoFilter), params.get("infofilters").get(0));
+ when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
+ createTimelineEntities("mockContainer3", "mockContainer4"));
} else if (path.contains(YARN_APPLICATION_ATTEMPT.toString())) {
when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
createTimelineEntity("mockAppAttempt1"));
@@ -151,6 +168,7 @@ public class TestTimelineReaderClientImpl {
when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
createTimelineEntities("mockApp1", "mockApp2"));
}
+
return mockClientResponse;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org