You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by zj...@apache.org on 2014/02/08 03:18:06 UTC
svn commit: r1565869 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-y...
Author: zjshen
Date: Sat Feb 8 02:18:06 2014
New Revision: 1565869
URL: http://svn.apache.org/r1565869
Log:
YARN-1635. Implemented a Leveldb based ApplicationTimelineStore. Contributed by Billie Rinaldi.
svn merge --ignore-ancestry -c 1565868 ../../trunk/
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityIdentifier.java
- copied unchanged from r1565868, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityIdentifier.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/GenericObjectMapper.java
- copied unchanged from r1565868, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/GenericObjectMapper.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/LeveldbApplicationTimelineStore.java
- copied unchanged from r1565868, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/LeveldbApplicationTimelineStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestGenericObjectMapper.java
- copied unchanged from r1565868, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestGenericObjectMapper.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestLeveldbApplicationTimelineStore.java
- copied unchanged from r1565868, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestLeveldbApplicationTimelineStore.java
Removed:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Sat Feb 8 02:18:06 2014
@@ -87,6 +87,9 @@ Release 2.4.0 - UNRELEASED
YARN-1566. Changed Distributed Shell to retain containers across application
attempts. (Jian He via vinodkv)
+ YARN-1635. Implemented a Leveldb based ApplicationTimelineStore. (Billie
+ Rinaldi via zjshen)
+
IMPROVEMENTS
YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java Sat Feb 8 02:18:06 2014
@@ -94,9 +94,21 @@ public class ATSPutErrors {
@Public
@Unstable
public static class ATSPutError {
+ /**
+ * Error code returned when no start time can be found when putting an
+ * entity. This occurs when the entity does not already exist in the
+ * store and it is put with no start time or events specified.
+ */
+ public static final int NO_START_TIME = 1;
+ /**
+ * Error code returned if an IOException is encountered when putting an
+ * entity.
+ */
+ public static final int IO_EXCEPTION = 2;
+
private String entityId;
private String entityType;
- private Integer errorCode;
+ private int errorCode;
/**
* Get the entity Id
@@ -144,7 +156,7 @@ public class ATSPutErrors {
* @return an error code
*/
@XmlElement(name = "errorcode")
- public Integer getErrorCode() {
+ public int getErrorCode() {
return errorCode;
}
@@ -154,7 +166,7 @@ public class ATSPutErrors {
* @param errorCode
* an error code
*/
- public void setErrorCode(Integer errorCode) {
+ public void setErrorCode(int errorCode) {
this.errorCode = errorCode;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Sat Feb 8 02:18:06 2014
@@ -1041,6 +1041,10 @@ public class YarnConfiguration extends C
/** ATS store class */
public static final String ATS_STORE = ATS_PREFIX + "store.class";
+ /** ATS leveldb path */
+ public static final String ATS_LEVELDB_PATH_PROPERTY =
+ ATS_PREFIX + "leveldb-apptimeline-store.path";
+
////////////////////////////////
// Other Configs
////////////////////////////////
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Sat Feb 8 02:18:06 2014
@@ -1145,7 +1145,13 @@
<property>
<description>Store class name for application timeline store</description>
<name>yarn.ats.store.class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore</value>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.LeveldbApplicationTimelineStore</value>
+ </property>
+
+ <property>
+ <description>Store file name for leveldb application timeline store</description>
+ <name>yarn.ats.leveldb-apptimeline-store.path</name>
+ <value>${yarn.log.dir}/ats</value>
</property>
<!-- Other configuration -->
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java Sat Feb 8 02:18:06 2014
@@ -117,14 +117,14 @@ public class TestApplicationTimelineReco
ATSPutError error1 = new ATSPutError();
error1.setEntityId("entity id 1");
error1.setEntityId("entity type 1");
- error1.setErrorCode(1);
+ error1.setErrorCode(ATSPutError.NO_START_TIME);
atsPutErrors.addError(error1);
List<ATSPutError> errors = new ArrayList<ATSPutError>();
errors.add(error1);
ATSPutError error2 = new ATSPutError();
error2.setEntityId("entity id 2");
error2.setEntityId("entity type 2");
- error2.setErrorCode(2);
+ error2.setErrorCode(ATSPutError.IO_EXCEPTION);
errors.add(error2);
atsPutErrors.addErrors(errors);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml Sat Feb 8 02:18:06 2014
@@ -167,6 +167,25 @@
<artifactId>jersey-test-framework-grizzly2</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </dependency>
</dependencies>
</project>
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java Sat Feb 8 02:18:06 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline;
+import java.io.IOException;
import java.util.Collection;
import java.util.EnumSet;
import java.util.Set;
@@ -78,13 +79,15 @@ public interface ApplicationTimelineRead
* retrieve (see {@link Field}). If the set of fields
* contains {@link Field#LAST_EVENT_ONLY} and not
* {@link Field#EVENTS}, the most recent event for
- * each entity is retrieved.
+ * each entity is retrieved. If null, retrieves all
+ * fields.
* @return An {@link ATSEntities} object.
+ * @throws IOException
*/
ATSEntities getEntities(String entityType,
Long limit, Long windowStart, Long windowEnd,
NameValuePair primaryFilter, Collection<NameValuePair> secondaryFilters,
- EnumSet<Field> fieldsToRetrieve);
+ EnumSet<Field> fieldsToRetrieve) throws IOException;
/**
* This method retrieves the entity information for a given entity.
@@ -95,11 +98,13 @@ public interface ApplicationTimelineRead
* retrieve (see {@link Field}). If the set of
* fields contains {@link Field#LAST_EVENT_ONLY} and
* not {@link Field#EVENTS}, the most recent event
- * for each entity is retrieved.
+ * for each entity is retrieved. If null, retrieves
+ * all fields.
* @return An {@link ATSEntity} object.
+ * @throws IOException
*/
ATSEntity getEntity(String entity, String entityType, EnumSet<Field>
- fieldsToRetrieve);
+ fieldsToRetrieve) throws IOException;
/**
* This method retrieves the events for a list of entities all of the same
@@ -118,8 +123,9 @@ public interface ApplicationTimelineRead
* @param eventTypes Restricts the events returned to the given types. If
* null, events of all types will be returned.
* @return An {@link ATSEvents} object.
+ * @throws IOException
*/
ATSEvents getEntityTimelines(String entityType,
SortedSet<String> entityIds, Long limit, Long windowStart,
- Long windowEnd, Set<String> eventTypes);
+ Long windowEnd, Set<String> eventTypes) throws IOException;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java Sat Feb 8 02:18:06 2014
@@ -23,6 +23,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities;
import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors;
+import java.io.IOException;
+
/**
* This interface is for storing application timeline information.
*/
@@ -37,7 +39,8 @@ public interface ApplicationTimelineWrit
*
* @param data An {@link ATSEntities} object.
* @return An {@link ATSPutErrors} object.
+ * @throws IOException
*/
- ATSPutErrors put(ATSEntities data);
+ ATSPutErrors put(ATSEntities data) throws IOException;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java Sat Feb 8 02:18:06 2014
@@ -53,8 +53,8 @@ import org.apache.hadoop.yarn.api.record
public class MemoryApplicationTimelineStore
extends AbstractService implements ApplicationTimelineStore {
- private Map<EntityId, ATSEntity> entities =
- new HashMap<EntityId, ATSEntity>();
+ private Map<EntityIdentifier, ATSEntity> entities =
+ new HashMap<EntityIdentifier, ATSEntity>();
public MemoryApplicationTimelineStore() {
super(MemoryApplicationTimelineStore.class.getName());
@@ -125,7 +125,7 @@ public class MemoryApplicationTimelineSt
if (fieldsToRetrieve == null) {
fieldsToRetrieve = EnumSet.allOf(Field.class);
}
- ATSEntity entity = entities.get(new EntityId(entityId, entityType));
+ ATSEntity entity = entities.get(new EntityIdentifier(entityId, entityType));
if (entity == null) {
return null;
} else {
@@ -152,7 +152,7 @@ public class MemoryApplicationTimelineSt
windowEnd = Long.MAX_VALUE;
}
for (String entityId : entityIds) {
- EntityId entityID = new EntityId(entityId, entityType);
+ EntityIdentifier entityID = new EntityIdentifier(entityId, entityType);
ATSEntity entity = entities.get(entityID);
if (entity == null) {
continue;
@@ -184,8 +184,8 @@ public class MemoryApplicationTimelineSt
public ATSPutErrors put(ATSEntities data) {
ATSPutErrors errors = new ATSPutErrors();
for (ATSEntity entity : data.getEntities()) {
- EntityId entityId =
- new EntityId(entity.getEntityId(), entity.getEntityType());
+ EntityIdentifier entityId =
+ new EntityIdentifier(entity.getEntityId(), entity.getEntityType());
// store entity info in memory
ATSEntity existingEntity = entities.get(entityId);
if (existingEntity == null) {
@@ -210,7 +210,7 @@ public class MemoryApplicationTimelineSt
ATSPutError error = new ATSPutError();
error.setEntityId(entityId.getId());
error.setEntityType(entityId.getType());
- error.setErrorCode(1);
+ error.setErrorCode(ATSPutError.NO_START_TIME);
errors.addError(error);
entities.remove(entityId);
continue;
@@ -242,12 +242,20 @@ public class MemoryApplicationTimelineSt
continue;
}
for (String idStr : partRelatedEntities.getValue()) {
- EntityId relatedEntityId =
- new EntityId(idStr, partRelatedEntities.getKey());
+ EntityIdentifier relatedEntityId =
+ new EntityIdentifier(idStr, partRelatedEntities.getKey());
ATSEntity relatedEntity = entities.get(relatedEntityId);
if (relatedEntity != null) {
relatedEntity.addRelatedEntity(
existingEntity.getEntityType(), existingEntity.getEntityId());
+ } else {
+ relatedEntity = new ATSEntity();
+ relatedEntity.setEntityId(relatedEntityId.getId());
+ relatedEntity.setEntityType(relatedEntityId.getType());
+ relatedEntity.setStartTime(existingEntity.getStartTime());
+ relatedEntity.addRelatedEntity(existingEntity.getEntityType(),
+ existingEntity.getEntityId());
+ entities.put(relatedEntityId, relatedEntity);
}
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java Sat Feb 8 02:18:06 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
@@ -45,6 +46,8 @@ import javax.xml.bind.annotation.XmlAcce
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities;
@@ -64,6 +67,8 @@ import com.google.inject.Singleton;
//TODO: support XML serialization/deserialization
public class ATSWebServices {
+ private static final Log LOG = LogFactory.getLog(ATSWebServices.class);
+
private ApplicationTimelineStore store;
@Inject
@@ -143,6 +148,10 @@ public class ATSWebServices {
"windowStart, windowEnd or limit is not a numeric value.");
} catch (IllegalArgumentException e) {
throw new BadRequestException("requested invalid field.");
+ } catch (IOException e) {
+ LOG.error("Error getting entities", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
}
if (entities == null) {
return new ATSEntities();
@@ -171,6 +180,10 @@ public class ATSWebServices {
} catch (IllegalArgumentException e) {
throw new BadRequestException(
"requested invalid field.");
+ } catch (IOException e) {
+ LOG.error("Error getting entity", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
}
if (entity == null) {
throw new WebApplicationException(Response.Status.NOT_FOUND);
@@ -206,6 +219,10 @@ public class ATSWebServices {
} catch (NumberFormatException e) {
throw new BadRequestException(
"windowStart, windowEnd or limit is not a numeric value.");
+ } catch (IOException e) {
+ LOG.error("Error getting entity timelines", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
}
if (events == null) {
return new ATSEvents();
@@ -228,7 +245,13 @@ public class ATSWebServices {
if (entities == null) {
return new ATSPutErrors();
}
- return store.put(entities);
+ try {
+ return store.put(entities);
+ } catch (IOException e) {
+ LOG.error("Error putting entities", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
}
private void init(HttpServletResponse response) {
@@ -275,7 +298,17 @@ public class ATSWebServices {
String[] strs = str.split(delimiter);
List<Field> fieldList = new ArrayList<Field>();
for (String s : strs) {
- fieldList.add(Field.valueOf(s.toUpperCase()));
+ s = s.trim().toUpperCase();
+ if (s.equals("EVENTS"))
+ fieldList.add(Field.EVENTS);
+ else if (s.equals("LASTEVENTONLY"))
+ fieldList.add(Field.LAST_EVENT_ONLY);
+ else if (s.equals("RELATEDENTITIES"))
+ fieldList.add(Field.RELATED_ENTITIES);
+ else if (s.equals("PRIMARYFILTERS"))
+ fieldList.add(Field.PRIMARY_FILTERS);
+ else if (s.equals("OTHERINFO"))
+ fieldList.add(Field.OTHER_INFO);
}
if (fieldList.size() == 0)
return null;
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java Sat Feb 8 02:18:06 2014
@@ -21,6 +21,8 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -71,7 +73,7 @@ public class ApplicationTimelineStoreTes
/**
* Load test data into the given store
*/
- protected void loadTestData() {
+ protected void loadTestData() throws IOException {
ATSEntities atsEntities = new ATSEntities();
Map<String, Object> primaryFilters = new HashMap<String, Object>();
primaryFilters.put("user", "username");
@@ -126,7 +128,7 @@ public class ApplicationTimelineStoreTes
response = store.put(atsEntities);
assertEquals(0, response.getErrors().size());
atsEntities.setEntities(Collections.singletonList(createEntity(entity1b,
- entityType1, 123l, Collections.singletonList(ev2), null,
+ entityType1, 789l, Collections.singletonList(ev2), null,
primaryFilters, otherInfo2)));
response = store.put(atsEntities);
assertEquals(0, response.getErrors().size());
@@ -138,11 +140,11 @@ public class ApplicationTimelineStoreTes
ATSPutError error = response.getErrors().get(0);
assertEquals("badentityid", error.getEntityId());
assertEquals("badentity", error.getEntityType());
- assertEquals((Integer) 1, error.getErrorCode());
+ assertEquals(ATSPutError.NO_START_TIME, error.getErrorCode());
}
/**
- * Load veification data
+ * Load verification data
*/
protected void loadVerificationData() throws Exception {
userFilter = new NameValuePair("user",
@@ -197,7 +199,7 @@ public class ApplicationTimelineStoreTes
events2.add(ev4);
}
- public void testGetSingleEntity() {
+ public void testGetSingleEntity() throws IOException {
// test getting entity info
verifyEntityInfo(null, null, null, null, null, null,
store.getEntity("id_1", "type_2", EnumSet.allOf(Field.class)));
@@ -222,6 +224,10 @@ public class ApplicationTimelineStoreTes
null, null, null, store.getEntity(entity1, entityType1,
EnumSet.of(Field.LAST_EVENT_ONLY)));
+ verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES,
+ primaryFilters, otherInfo, store.getEntity(entity1b, entityType1,
+ null));
+
verifyEntityInfo(entity1, entityType1, null, null, primaryFilters, null,
store.getEntity(entity1, entityType1,
EnumSet.of(Field.PRIMARY_FILTERS)));
@@ -234,7 +240,7 @@ public class ApplicationTimelineStoreTes
EnumSet.of(Field.RELATED_ENTITIES)));
}
- public void testGetEntities() {
+ public void testGetEntities() throws IOException {
// test getting entities
assertEquals("nonzero entities size for nonexistent type", 0,
store.getEntities("type_0", null, null, null, null, null,
@@ -305,7 +311,7 @@ public class ApplicationTimelineStoreTes
primaryFilters, otherInfo, entities.get(1));
}
- public void testGetEntitiesWithPrimaryFilters() {
+ public void testGetEntitiesWithPrimaryFilters() throws IOException {
// test using primary filter
assertEquals("nonzero entities size for primary filter", 0,
store.getEntities("type_1", null, null, null,
@@ -361,7 +367,7 @@ public class ApplicationTimelineStoreTes
primaryFilters, otherInfo, entities.get(1));
}
- public void testGetEntitiesWithSecondaryFilters() {
+ public void testGetEntitiesWithSecondaryFilters() throws IOException {
// test using secondary filter
List<ATSEntity> entities = store.getEntities("type_1", null, null, null,
null, goodTestingFilters, EnumSet.allOf(Field.class)).getEntities();
@@ -388,7 +394,7 @@ public class ApplicationTimelineStoreTes
assertEquals(0, entities.size());
}
- public void testGetEvents() {
+ public void testGetEvents() throws IOException {
// test getting entity timelines
SortedSet<String> sortedSet = new TreeSet<String>();
sortedSet.add(entity1);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java Sat Feb 8 02:18:06 2014
@@ -23,6 +23,7 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
public class TestMemoryApplicationTimelineStore
extends ApplicationTimelineStoreTestUtils {
@@ -46,27 +47,27 @@ public class TestMemoryApplicationTimeli
}
@Test
- public void testGetSingleEntity() {
+ public void testGetSingleEntity() throws IOException {
super.testGetSingleEntity();
}
@Test
- public void testGetEntities() {
+ public void testGetEntities() throws IOException {
super.testGetEntities();
}
@Test
- public void testGetEntitiesWithPrimaryFilters() {
+ public void testGetEntitiesWithPrimaryFilters() throws IOException {
super.testGetEntitiesWithPrimaryFilters();
}
@Test
- public void testGetEntitiesWithSecondaryFilters() {
+ public void testGetEntitiesWithSecondaryFilters() throws IOException {
super.testGetEntitiesWithSecondaryFilters();
}
@Test
- public void testGetEvents() {
+ public void testGetEvents() throws IOException {
super.testGetEvents();
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java?rev=1565869&r1=1565868&r2=1565869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java Sat Feb 8 02:18:06 2014
@@ -157,6 +157,43 @@ public class TestATSWebServices extends
}
@Test
+ public void testGetEntityFields1() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("apptimeline")
+ .path("type_1").path("id_1").queryParam("fields", "events,otherinfo")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ ATSEntity entity = response.getEntity(ATSEntity.class);
+ Assert.assertNotNull(entity);
+ Assert.assertEquals("id_1", entity.getEntityId());
+ Assert.assertEquals("type_1", entity.getEntityType());
+ Assert.assertEquals(123l, entity.getStartTime().longValue());
+ Assert.assertEquals(2, entity.getEvents().size());
+ Assert.assertEquals(0, entity.getPrimaryFilters().size());
+ Assert.assertEquals(4, entity.getOtherInfo().size());
+ }
+
+ @Test
+ public void testGetEntityFields2() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("apptimeline")
+ .path("type_1").path("id_1").queryParam("fields", "lasteventonly," +
+ "primaryfilters,relatedentities")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ ATSEntity entity = response.getEntity(ATSEntity.class);
+ Assert.assertNotNull(entity);
+ Assert.assertEquals("id_1", entity.getEntityId());
+ Assert.assertEquals("type_1", entity.getEntityType());
+ Assert.assertEquals(123l, entity.getStartTime().longValue());
+ Assert.assertEquals(1, entity.getEvents().size());
+ Assert.assertEquals(2, entity.getPrimaryFilters().size());
+ Assert.assertEquals(0, entity.getOtherInfo().size());
+ }
+
+ @Test
public void testGetEvents() throws Exception {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("apptimeline")