You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/02/03 21:07:56 UTC
svn commit: r1564038 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/
hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records...
Author: vinodkv
Date: Mon Feb 3 20:07:55 2014
New Revision: 1564038
URL: http://svn.apache.org/r1564038
Log:
YARN-1659. Defined the ApplicationTimelineStore store as an abstraction for implementing different storage impls for storing timeline information. Contributed by Billie Rinaldi.
svn merge --ignore-ancestry -c 1564037 ../../trunk/
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java
- copied unchanged from r1564037, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/
- copied from r1564037, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1564038&r1=1564037&r2=1564038&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Mon Feb 3 20:07:55 2014
@@ -63,6 +63,10 @@ Release 2.4.0 - UNRELEASED
used by ResourceManager to read configuration locally or from remote systems
so as to help RM failover. (Xuan Gong via vinodkv)
+ YARN-1659. Defined the ApplicationTimelineStore store as an abstraction for
+ implementing different storage impls for storing timeline information.
+ (Billie Rinaldi via vinodkv)
+
IMPROVEMENTS
YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java?rev=1564038&r1=1564037&r2=1564038&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java Mon Feb 3 20:07:55 2014
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -53,10 +54,10 @@ public class ATSEntity {
private String entityType;
private String entityId;
- private long startTime;
+ private Long startTime;
private List<ATSEvent> events = new ArrayList<ATSEvent>();
- private Map<String, List<Object>> relatedEntities =
- new HashMap<String, List<Object>>();
+ private Map<String, List<String>> relatedEntities =
+ new HashMap<String, List<String>>();
private Map<String, Object> primaryFilters =
new HashMap<String, Object>();
private Map<String, Object> otherInfo =
@@ -112,7 +113,7 @@ public class ATSEntity {
* @return the start time of the entity
*/
@XmlElement(name = "starttime")
- public long getStartTime() {
+ public Long getStartTime() {
return startTime;
}
@@ -122,7 +123,7 @@ public class ATSEntity {
* @param startTime
* the start time of the entity
*/
- public void setStartTime(long startTime) {
+ public void setStartTime(Long startTime) {
this.startTime = startTime;
}
@@ -172,26 +173,25 @@ public class ATSEntity {
* @return the related entities
*/
@XmlElement(name = "relatedentities")
- public Map<String, List<Object>> getRelatedEntities() {
+ public Map<String, List<String>> getRelatedEntities() {
return relatedEntities;
}
/**
- * Add a list of entity of the same type to the existing related entity map
+ * Add an entity to the existing related entity map
*
* @param entityType
* the entity type
- * @param entityIds
- * a list of entity Ids
+ * @param entityId
+ * the entity Id
*/
- public void addRelatedEntity(String entityType, List<Object> entityIds) {
- List<Object> thisRelatedEntity = relatedEntities.get(entityType);
- relatedEntities.put(entityType, entityIds);
+ public void addRelatedEntity(String entityType, String entityId) {
+ List<String> thisRelatedEntity = relatedEntities.get(entityType);
if (thisRelatedEntity == null) {
- relatedEntities.put(entityType, entityIds);
- } else {
- thisRelatedEntity.addAll(entityIds);
+ thisRelatedEntity = new ArrayList<String>();
+ relatedEntities.put(entityType, thisRelatedEntity);
}
+ thisRelatedEntity.add(entityId);
}
/**
@@ -200,11 +200,10 @@ public class ATSEntity {
* @param relatedEntities
* a map of related entities
*/
- public void addRelatedEntities(
- Map<String, List<Object>> relatedEntities) {
- for (Map.Entry<String, List<Object>> relatedEntity : relatedEntities
- .entrySet()) {
- List<Object> thisRelatedEntity =
+ public void addRelatedEntities(Map<String, List<String>> relatedEntities) {
+ for (Entry<String, List<String>> relatedEntity :
+ relatedEntities.entrySet()) {
+ List<String> thisRelatedEntity =
this.relatedEntities.get(relatedEntity.getKey());
if (thisRelatedEntity == null) {
this.relatedEntities.put(
@@ -222,7 +221,7 @@ public class ATSEntity {
* a map of related entities
*/
public void setRelatedEntities(
- Map<String, List<Object>> relatedEntities) {
+ Map<String, List<String>> relatedEntities) {
this.relatedEntities = relatedEntities;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java?rev=1564038&r1=1564037&r2=1564038&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java Mon Feb 3 20:07:55 2014
@@ -39,7 +39,7 @@ import org.apache.hadoop.classification.
@XmlAccessorType(XmlAccessType.NONE)
@Public
@Unstable
-public class ATSEvent {
+public class ATSEvent implements Comparable<ATSEvent> {
private long timestamp;
private String eventType;
@@ -131,4 +131,42 @@ public class ATSEvent {
this.eventInfo = eventInfo;
}
+ @Override
+ public int compareTo(ATSEvent other) {
+ if (timestamp > other.timestamp) {
+ return -1;
+ } else if (timestamp < other.timestamp) {
+ return 1;
+ } else {
+ return eventType.compareTo(other.eventType);
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ ATSEvent atsEvent = (ATSEvent) o;
+
+ if (timestamp != atsEvent.timestamp)
+ return false;
+ if (!eventType.equals(atsEvent.eventType))
+ return false;
+ if (eventInfo != null ? !eventInfo.equals(atsEvent.eventInfo) :
+ atsEvent.eventInfo != null)
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = (int) (timestamp ^ (timestamp >>> 32));
+ result = 31 * result + eventType.hashCode();
+ result = 31 * result + (eventInfo != null ? eventInfo.hashCode() : 0);
+ return result;
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java?rev=1564038&r1=1564037&r2=1564038&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java Mon Feb 3 20:07:55 2014
@@ -160,7 +160,7 @@ public class ATSEvents {
* @param event
* a single event
*/
- public void addEntity(ATSEvent event) {
+ public void addEvent(ATSEvent event) {
events.add(event);
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java?rev=1564038&r1=1564037&r2=1564038&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java Mon Feb 3 20:07:55 2014
@@ -18,10 +18,13 @@
package org.apache.hadoop.yarn.api.records.apptimeline;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import junit.framework.Assert;
+import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError;
import org.junit.Test;
public class TestApplicationTimelineRecords {
@@ -42,10 +45,8 @@ public class TestApplicationTimelineReco
event.addEventInfo("key2", "val2");
entity.addEvent(event);
}
- entity.addRelatedEntity(
- "test ref type 1", Arrays.asList((Object) "test ref id 1"));
- entity.addRelatedEntity(
- "test ref type 2", Arrays.asList((Object) "test ref id 2"));
+ entity.addRelatedEntity("test ref type 1", "test ref id 1");
+ entity.addRelatedEntity("test ref type 2", "test ref id 2");
entity.addPrimaryFilter("pkey1", "pval1");
entity.addPrimaryFilter("pkey2", "pval2");
entity.addOtherInfo("okey1", "oval1");
@@ -83,7 +84,7 @@ public class TestApplicationTimelineReco
event.setEventType("event type " + i);
event.addEventInfo("key1", "val1");
event.addEventInfo("key2", "val2");
- partEvents.addEntity(event);
+ partEvents.addEvent(event);
}
events.addEvent(partEvents);
}
@@ -110,4 +111,36 @@ public class TestApplicationTimelineReco
Assert.assertEquals(2, event22.getEventInfo().size());
}
+ @Test
+ public void testATSPutErrors() {
+ ATSPutErrors atsPutErrors = new ATSPutErrors();
+ ATSPutError error1 = new ATSPutError();
+ error1.setEntityId("entity id 1");
+ error1.setEntityId("entity type 1");
+ error1.setErrorCode(1);
+ atsPutErrors.addError(error1);
+ List<ATSPutError> errors = new ArrayList<ATSPutError>();
+ errors.add(error1);
+ ATSPutError error2 = new ATSPutError();
+ error2.setEntityId("entity id 2");
+ error2.setEntityId("entity type 2");
+ error2.setErrorCode(2);
+ errors.add(error2);
+ atsPutErrors.addErrors(errors);
+
+ Assert.assertEquals(3, atsPutErrors.getErrors().size());
+ ATSPutError e = atsPutErrors.getErrors().get(0);
+ Assert.assertEquals(error1.getEntityId(), e.getEntityId());
+ Assert.assertEquals(error1.getEntityType(), e.getEntityType());
+ Assert.assertEquals(error1.getErrorCode(), e.getErrorCode());
+ e = atsPutErrors.getErrors().get(1);
+ Assert.assertEquals(error1.getEntityId(), e.getEntityId());
+ Assert.assertEquals(error1.getEntityType(), e.getEntityType());
+ Assert.assertEquals(error1.getErrorCode(), e.getErrorCode());
+ e = atsPutErrors.getErrors().get(2);
+ Assert.assertEquals(error2.getEntityId(), e.getEntityId());
+ Assert.assertEquals(error2.getEntityType(), e.getEntityType());
+ Assert.assertEquals(error2.getErrorCode(), e.getErrorCode());
+ }
+
}