You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2014/11/05 03:00:35 UTC
git commit: YARN-2804. Fixed Timeline service to not fill the logs
with JAXB bindings exceptions. Contributed by Zhijie Shen.
Repository: hadoop
Updated Branches:
refs/heads/trunk d794f785d -> b76179895
YARN-2804. Fixed Timeline service to not fill the logs with JAXB bindings exceptions. Contributed by Zhijie Shen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7617989
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7617989
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7617989
Branch: refs/heads/trunk
Commit: b76179895dd2ef4d56e8de31e9f673375faa2afa
Parents: d794f78
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Tue Nov 4 17:45:46 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Nov 4 18:00:06 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../api/records/timeline/TimelineEntity.java | 49 +++++--
.../api/records/timeline/TimelineEvent.java | 17 ++-
.../records/timeline/TestTimelineRecords.java | 128 ++++++++++++++++++-
.../timeline/webapp/TimelineWebServices.java | 9 +-
5 files changed, 182 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7617989/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f9fcf5b..b5aec42 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -859,6 +859,9 @@ Release 2.6.0 - UNRELEASED
YARN-2010. Handle app-recovery failures gracefully.
(Jian He and Karthik Kambatla via kasha)
+ YARN-2804. Fixed Timeline service to not fill the logs with JAXB bindings
+ exceptions. (Zhijie Shen via vinodkv)
+
Release 2.5.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7617989/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
index d66c253..03ed4af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
@@ -31,6 +31,7 @@ import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -58,11 +59,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
private String entityId;
private Long startTime;
private List<TimelineEvent> events = new ArrayList<TimelineEvent>();
- private Map<String, Set<String>> relatedEntities =
+ private HashMap<String, Set<String>> relatedEntities =
new HashMap<String, Set<String>>();
- private Map<String, Set<Object>> primaryFilters =
+ private HashMap<String, Set<Object>> primaryFilters =
new HashMap<String, Set<Object>>();
- private Map<String, Object> otherInfo =
+ private HashMap<String, Object> otherInfo =
new HashMap<String, Object>();
private String domainId;
@@ -175,11 +176,17 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
*
* @return the related entities
*/
- @XmlElement(name = "relatedentities")
public Map<String, Set<String>> getRelatedEntities() {
return relatedEntities;
}
+ // Required by JAXB
+ @Private
+ @XmlElement(name = "relatedentities")
+ public HashMap<String, Set<String>> getRelatedEntitiesJAXB() {
+ return relatedEntities;
+ }
+
/**
* Add an entity to the existing related entity map
*
@@ -224,7 +231,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
*/
public void setRelatedEntities(
Map<String, Set<String>> relatedEntities) {
- this.relatedEntities = relatedEntities;
+ if (relatedEntities != null && !(relatedEntities instanceof HashMap)) {
+ this.relatedEntities = new HashMap<String, Set<String>>(relatedEntities);
+ } else {
+ this.relatedEntities = (HashMap<String, Set<String>>) relatedEntities;
+ }
}
/**
@@ -232,11 +243,17 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
*
* @return the primary filters
*/
- @XmlElement(name = "primaryfilters")
public Map<String, Set<Object>> getPrimaryFilters() {
return primaryFilters;
}
+ // Required by JAXB
+ @Private
+ @XmlElement(name = "primaryfilters")
+ public HashMap<String, Set<Object>> getPrimaryFiltersJAXB() {
+ return primaryFilters;
+ }
+
/**
* Add a single piece of primary filter to the existing primary filter map
*
@@ -280,7 +297,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
* a map of primary filters
*/
public void setPrimaryFilters(Map<String, Set<Object>> primaryFilters) {
- this.primaryFilters = primaryFilters;
+ if (primaryFilters != null && !(primaryFilters instanceof HashMap)) {
+ this.primaryFilters = new HashMap<String, Set<Object>>(primaryFilters);
+ } else {
+ this.primaryFilters = (HashMap<String, Set<Object>>) primaryFilters;
+ }
}
/**
@@ -288,11 +309,17 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
*
* @return the other information of the entity
*/
- @XmlElement(name = "otherinfo")
public Map<String, Object> getOtherInfo() {
return otherInfo;
}
+ // Required by JAXB
+ @Private
+ @XmlElement(name = "otherinfo")
+ public HashMap<String, Object> getOtherInfoJAXB() {
+ return otherInfo;
+ }
+
/**
* Add one piece of other information of the entity to the existing other info
* map
@@ -323,7 +350,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
* a map of other information
*/
public void setOtherInfo(Map<String, Object> otherInfo) {
- this.otherInfo = otherInfo;
+ if (otherInfo != null && !(otherInfo instanceof HashMap)) {
+ this.otherInfo = new HashMap<String, Object>(otherInfo);
+ } else {
+ this.otherInfo = (HashMap<String, Object>) otherInfo;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7617989/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
index aa49538..0a09e78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
@@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -43,7 +44,7 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
private long timestamp;
private String eventType;
- private Map<String, Object> eventInfo = new HashMap<String, Object>();
+ private HashMap<String, Object> eventInfo = new HashMap<String, Object>();
public TimelineEvent() {
}
@@ -93,11 +94,17 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
*
* @return the information of the event
*/
- @XmlElement(name = "eventinfo")
public Map<String, Object> getEventInfo() {
return eventInfo;
}
+ // Required by JAXB
+ @Private
+ @XmlElement(name = "eventinfo")
+ public HashMap<String, Object> getEventInfoJAXB() {
+ return eventInfo;
+ }
+
/**
* Add one piece of the information of the event to the existing information
* map
@@ -128,7 +135,11 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
* a map of of the information of the event
*/
public void setEventInfo(Map<String, Object> eventInfo) {
- this.eventInfo = eventInfo;
+ if (eventInfo != null && !(eventInfo instanceof HashMap)) {
+ this.eventInfo = new HashMap<String, Object>(eventInfo);
+ } else {
+ this.eventInfo = (HashMap<String, Object>) eventInfo;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7617989/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
index 2b59ff5..9d16edb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
@@ -19,19 +19,19 @@
package org.apache.hadoop.yarn.api.records.timeline;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
-
-import org.junit.Assert;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.WeakHashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.junit.Assert;
import org.junit.Test;
public class TestTimelineRecords {
@@ -202,4 +202,118 @@ public class TestTimelineRecords {
}
}
+ @Test
+ public void testMapInterfaceOrTimelineRecords() throws Exception {
+ TimelineEntity entity = new TimelineEntity();
+ List<Map<String, Set<Object>>> primaryFiltersList =
+ new ArrayList<Map<String, Set<Object>>>();
+ primaryFiltersList.add(
+ Collections.singletonMap("pkey", Collections.singleton((Object) "pval")));
+ Map<String, Set<Object>> primaryFilters = new TreeMap<String, Set<Object>>();
+ primaryFilters.put("pkey1", Collections.singleton((Object) "pval1"));
+ primaryFilters.put("pkey2", Collections.singleton((Object) "pval2"));
+ primaryFiltersList.add(primaryFilters);
+ entity.setPrimaryFilters(null);
+ for (Map<String, Set<Object>> primaryFiltersToSet : primaryFiltersList) {
+ entity.setPrimaryFilters(primaryFiltersToSet);
+ assertPrimaryFilters(entity);
+
+ Map<String, Set<Object>> primaryFiltersToAdd =
+ new WeakHashMap<String, Set<Object>>();
+ primaryFiltersToAdd.put("pkey3", Collections.singleton((Object) "pval3"));
+ entity.addPrimaryFilters(primaryFiltersToAdd);
+ assertPrimaryFilters(entity);
+ }
+
+ List<Map<String, Set<String>>> relatedEntitiesList =
+ new ArrayList<Map<String, Set<String>>>();
+ relatedEntitiesList.add(
+ Collections.singletonMap("rkey", Collections.singleton("rval")));
+ Map<String, Set<String>> relatedEntities = new TreeMap<String, Set<String>>();
+ relatedEntities.put("rkey1", Collections.singleton("rval1"));
+ relatedEntities.put("rkey2", Collections.singleton("rval2"));
+ relatedEntitiesList.add(relatedEntities);
+ entity.setRelatedEntities(null);
+ for (Map<String, Set<String>> relatedEntitiesToSet : relatedEntitiesList) {
+ entity.setRelatedEntities(relatedEntitiesToSet);
+ assertRelatedEntities(entity);
+
+ Map<String, Set<String>> relatedEntitiesToAdd =
+ new WeakHashMap<String, Set<String>>();
+ relatedEntitiesToAdd.put("rkey3", Collections.singleton("rval3"));
+ entity.addRelatedEntities(relatedEntitiesToAdd);
+ assertRelatedEntities(entity);
+ }
+
+ List<Map<String, Object>> otherInfoList =
+ new ArrayList<Map<String, Object>>();
+ otherInfoList.add(Collections.singletonMap("okey", (Object) "oval"));
+ Map<String, Object> otherInfo = new TreeMap<String, Object>();
+ otherInfo.put("okey1", "oval1");
+ otherInfo.put("okey2", "oval2");
+ otherInfoList.add(otherInfo);
+ entity.setOtherInfo(null);
+ for (Map<String, Object> otherInfoToSet : otherInfoList) {
+ entity.setOtherInfo(otherInfoToSet);
+ assertOtherInfo(entity);
+
+ Map<String, Object> otherInfoToAdd = new WeakHashMap<String, Object>();
+ otherInfoToAdd.put("okey3", "oval3");
+ entity.addOtherInfo(otherInfoToAdd);
+ assertOtherInfo(entity);
+ }
+
+ TimelineEvent event = new TimelineEvent();
+ List<Map<String, Object>> eventInfoList =
+ new ArrayList<Map<String, Object>>();
+ eventInfoList.add(Collections.singletonMap("ekey", (Object) "eval"));
+ Map<String, Object> eventInfo = new TreeMap<String, Object>();
+ eventInfo.put("ekey1", "eval1");
+ eventInfo.put("ekey2", "eval2");
+ eventInfoList.add(eventInfo);
+ event.setEventInfo(null);
+ for (Map<String, Object> eventInfoToSet : eventInfoList) {
+ event.setEventInfo(eventInfoToSet);
+ assertEventInfo(event);
+
+ Map<String, Object> eventInfoToAdd = new WeakHashMap<String, Object>();
+ eventInfoToAdd.put("ekey3", "eval3");
+ event.addEventInfo(eventInfoToAdd);
+ assertEventInfo(event);
+ }
+ }
+
+ private static void assertPrimaryFilters(TimelineEntity entity) {
+ Assert.assertNotNull(entity.getPrimaryFilters());
+ Assert.assertNotNull(entity.getPrimaryFiltersJAXB());
+ Assert.assertTrue(entity.getPrimaryFilters() instanceof HashMap);
+ Assert.assertTrue(entity.getPrimaryFiltersJAXB() instanceof HashMap);
+ Assert.assertEquals(
+ entity.getPrimaryFilters(), entity.getPrimaryFiltersJAXB());
+ }
+
+ private static void assertRelatedEntities(TimelineEntity entity) {
+ Assert.assertNotNull(entity.getRelatedEntities());
+ Assert.assertNotNull(entity.getRelatedEntitiesJAXB());
+ Assert.assertTrue(entity.getRelatedEntities() instanceof HashMap);
+ Assert.assertTrue(entity.getRelatedEntitiesJAXB() instanceof HashMap);
+ Assert.assertEquals(
+ entity.getRelatedEntities(), entity.getRelatedEntitiesJAXB());
+ }
+
+ private static void assertOtherInfo(TimelineEntity entity) {
+ Assert.assertNotNull(entity.getOtherInfo());
+ Assert.assertNotNull(entity.getOtherInfoJAXB());
+ Assert.assertTrue(entity.getOtherInfo() instanceof HashMap);
+ Assert.assertTrue(entity.getOtherInfoJAXB() instanceof HashMap);
+ Assert.assertEquals(entity.getOtherInfo(), entity.getOtherInfoJAXB());
+ }
+
+ private static void assertEventInfo(TimelineEvent event) {
+ Assert.assertNotNull(event);
+ Assert.assertNotNull(event.getEventInfoJAXB());
+ Assert.assertTrue(event.getEventInfo() instanceof HashMap);
+ Assert.assertTrue(event.getEventInfoJAXB() instanceof HashMap);
+ Assert.assertEquals(event.getEventInfo(), event.getEventInfoJAXB());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7617989/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
index f290930..0907f2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
@@ -42,7 +42,6 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
@@ -53,11 +52,11 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.EntityIdentifier;
@@ -272,7 +271,7 @@ public class TimelineWebServices {
@PUT
@Path("/domain")
@Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
- public Response putDomain(
+ public TimelinePutResponse putDomain(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
TimelineDomain domain) {
@@ -295,7 +294,7 @@ public class TimelineWebServices {
throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
}
- return Response.status(Status.OK).build();
+ return new TimelinePutResponse();
}
/**