You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/02/05 18:02:26 UTC
svn commit: r1564829 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/
hadoop-yarn/hadoop-yarn-common/src...
Author: vinodkv
Date: Wed Feb 5 17:02:26 2014
New Revision: 1564829
URL: http://svn.apache.org/r1564829
Log:
YARN-1636. Augmented Application-history server's web-services to also expose new APIs for retrieving and storing timeline information. Contributed by Zhijie Shen.
Added:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1564829&r1=1564828&r2=1564829&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Wed Feb 5 17:02:26 2014
@@ -96,6 +96,10 @@ Release 2.4.0 - UNRELEASED
YARN-1461. Added tags for YARN applications and changed RM to handle them.
(Karthik Kambatla via zjshen)
+ YARN-1636. Augmented Application-history server's web-services to also expose
+ new APIs for retrieving and storing timeline information. (Zhijie Shen via
+ vinodkv)
+
IMPROVEMENTS
YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1564829&r1=1564828&r2=1564829&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Wed Feb 5 17:02:26 2014
@@ -1029,6 +1029,15 @@ public class YarnConfiguration extends C
AHS_PREFIX + "webapp.spnego-keytab-file";
////////////////////////////////
+ // ATS Configs
+ ////////////////////////////////
+
+ public static final String ATS_PREFIX = YARN_PREFIX + "ats.";
+
+ /** ATS store class */
+ public static final String ATS_STORE = ATS_PREFIX + "store.class";
+
+ ////////////////////////////////
// Other Configs
////////////////////////////////
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java?rev=1564829&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java Wed Feb 5 17:02:26 2014
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.webapp;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider;
+import org.codehaus.jackson.map.AnnotationIntrospector;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion;
+import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+
+import com.google.inject.Singleton;
+
+/**
+ * YARN's implementation of JAX-RS abstractions based on
+ * {@link JacksonJaxbJsonProvider} needed for deserialize JSON content to or
+ * serialize it from POJO objects.
+ */
+@Singleton
+@Provider
+@Unstable
+@Private
+public class YarnJacksonJaxbJsonProvider extends JacksonJaxbJsonProvider {
+
+ public YarnJacksonJaxbJsonProvider() {
+ super();
+ }
+
+ @Override
+ public ObjectMapper locateMapper(Class<?> type, MediaType mediaType) {
+ ObjectMapper mapper = super.locateMapper(type, mediaType);
+ AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
+ mapper.setAnnotationIntrospector(introspector);
+ mapper.getSerializationConfig()
+ .setSerializationInclusion(Inclusion.NON_NULL);
+ return mapper;
+ }
+}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1564829&r1=1564828&r2=1564829&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Wed Feb 5 17:02:26 2014
@@ -1137,6 +1137,14 @@
<value>org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore</value>
</property>
+ <!-- Application Timeline Service's Configuration-->
+
+ <property>
+ <description>Store class name for application timeline store</description>
+ <name>yarn.ats.store.class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore</value>
+ </property>
+
<!-- Other configuration -->
<property>
<description>The interval that the yarn client library uses to poll the
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java?rev=1564829&r1=1564828&r2=1564829&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java Wed Feb 5 17:02:26 2014
@@ -27,11 +27,14 @@ import org.apache.hadoop.metrics2.source
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
@@ -51,6 +54,7 @@ public class ApplicationHistoryServer ex
ApplicationHistoryClientService ahsClientService;
ApplicationHistoryManager historyManager;
+ ApplicationTimelineStore timelineStore;
private WebApp webApp;
public ApplicationHistoryServer() {
@@ -63,6 +67,8 @@ public class ApplicationHistoryServer ex
ahsClientService = createApplicationHistoryClientService(historyManager);
addService(ahsClientService);
addService((Service) historyManager);
+ timelineStore = createApplicationTimelineStore(conf);
+ addIfService(timelineStore);
super.serviceInit(conf);
}
@@ -135,6 +141,15 @@ public class ApplicationHistoryServer ex
return new ApplicationHistoryManagerImpl();
}
+ protected ApplicationTimelineStore createApplicationTimelineStore(
+ Configuration conf) {
+ // TODO: need to replace the MemoryApplicationTimelineStore.class with the
+ // LevelDB implementation
+ return ReflectionUtils.newInstance(conf.getClass(
+ YarnConfiguration.ATS_STORE, MemoryApplicationTimelineStore.class,
+ ApplicationTimelineStore.class), conf);
+ }
+
protected void startWebApp() {
String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig());
LOG.info("Instantiating AHSWebApp at " + bindAddress);
@@ -148,7 +163,8 @@ public class ApplicationHistoryServer ex
YarnConfiguration.AHS_WEBAPP_SPNEGO_USER_NAME_KEY)
.withHttpSpnegoKeytabKey(
YarnConfiguration.AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
- .at(bindAddress).start(new AHSWebApp(historyManager));
+ .at(bindAddress)
+ .start(new AHSWebApp(historyManager, timelineStore));
} catch (Exception e) {
String msg = "AHSWebApp failed to start.";
LOG.error(msg, e);
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java?rev=1564829&r1=1564828&r2=1564829&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java Wed Feb 5 17:02:26 2014
@@ -21,24 +21,31 @@ import static org.apache.hadoop.yarn.uti
import org.apache.hadoop.yarn.server.api.ApplicationContext;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
public class AHSWebApp extends WebApp implements YarnWebParams {
private final ApplicationHistoryManager applicationHistoryManager;
+ private final ApplicationTimelineStore applicationTimelineStore;
- public AHSWebApp(ApplicationHistoryManager applicationHistoryManager) {
+ public AHSWebApp(ApplicationHistoryManager applicationHistoryManager,
+ ApplicationTimelineStore applicationTimelineStore) {
this.applicationHistoryManager = applicationHistoryManager;
+ this.applicationTimelineStore = applicationTimelineStore;
}
@Override
public void setup() {
- bind(JAXBContextResolver.class);
+ bind(YarnJacksonJaxbJsonProvider.class);
bind(AHSWebServices.class);
+ bind(ATSWebServices.class);
bind(GenericExceptionHandler.class);
bind(ApplicationContext.class).toInstance(applicationHistoryManager);
+ bind(ApplicationTimelineStore.class).toInstance(applicationTimelineStore);
route("/", AHSController.class);
route(pajoin("/apps", APP_STATE), AHSController.class);
route(pajoin("/app", APPLICATION_ID), AHSController.class, "app");
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java?rev=1564829&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java Wed Feb 5 17:02:26 2014
@@ -0,0 +1,297 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities;
+import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity;
+import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents;
+import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineReader.Field;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.NameValuePair;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+@Singleton
+@Path("/ws/v1/apptimeline")
+//TODO: support XML serialization/deserialization
+public class ATSWebServices {
+
+ private ApplicationTimelineStore store;
+
+ @Inject
+ public ATSWebServices(ApplicationTimelineStore store) {
+ this.store = store;
+ }
+
+ @XmlRootElement(name = "about")
+ @XmlAccessorType(XmlAccessType.NONE)
+ @Public
+ @Unstable
+ public static class AboutInfo {
+
+ private String about;
+
+ public AboutInfo() {
+
+ }
+
+ public AboutInfo(String about) {
+ this.about = about;
+ }
+
+ @XmlElement(name = "About")
+ public String getAbout() {
+ return about;
+ }
+
+ public void setAbout(String about) {
+ this.about = about;
+ }
+
+ }
+
+ /**
+ * Return the description of the application timeline web services.
+ */
+ @GET
+ @Path("/")
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public AboutInfo about(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res) {
+ init(res);
+ return new AboutInfo("Application Timeline API");
+ }
+
+ /**
+ * Return a list of entities that match the given parameters.
+ */
+ @GET
+ @Path("/{entityType}")
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public ATSEntities getEntities(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("entityType") String entityType,
+ @QueryParam("primaryFilter") String primaryFilter,
+ @QueryParam("secondaryFilter") String secondaryFilter,
+ @QueryParam("windowStart") String windowStart,
+ @QueryParam("windowEnd") String windowEnd,
+ @QueryParam("limit") String limit,
+ @QueryParam("fields") String fields) {
+ init(res);
+ ATSEntities entities = null;
+ try {
+ entities = store.getEntities(
+ parseStr(entityType),
+ parseLongStr(limit),
+ parseLongStr(windowStart),
+ parseLongStr(windowEnd),
+ parsePairStr(primaryFilter, ":"),
+ parsePairsStr(secondaryFilter, ",", ":"),
+ parseFieldsStr(fields, ","));
+ } catch (NumberFormatException e) {
+ throw new BadRequestException(
+ "windowStart, windowEnd or limit is not a numeric value.");
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("requested invalid field.");
+ }
+ if (entities == null) {
+ return new ATSEntities();
+ }
+ return entities;
+ }
+
+ /**
+ * Return a single entity of the given entity type and Id.
+ */
+ @GET
+ @Path("/{entityType}/{entityId}")
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public ATSEntity getEntity(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("entityType") String entityType,
+ @PathParam("entityId") String entityId,
+ @QueryParam("fields") String fields) {
+ init(res);
+ ATSEntity entity = null;
+ try {
+ entity =
+ store.getEntity(parseStr(entityId), parseStr(entityType),
+ parseFieldsStr(fields, ","));
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException(
+ "requested invalid field.");
+ }
+ if (entity == null) {
+ throw new WebApplicationException(Response.Status.NOT_FOUND);
+ }
+ return entity;
+ }
+
+ /**
+ * Return the events that match the given parameters.
+ */
+ @GET
+ @Path("/{entityType}/events")
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public ATSEvents getEvents(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("entityType") String entityType,
+ @QueryParam("entityId") String entityId,
+ @QueryParam("eventType") String eventType,
+ @QueryParam("windowStart") String windowStart,
+ @QueryParam("windowEnd") String windowEnd,
+ @QueryParam("limit") String limit) {
+ init(res);
+ ATSEvents events = null;
+ try {
+ events = store.getEntityTimelines(
+ parseStr(entityType),
+ parseArrayStr(entityId, ","),
+ parseLongStr(limit),
+ parseLongStr(windowStart),
+ parseLongStr(windowEnd),
+ parseArrayStr(eventType, ","));
+ } catch (NumberFormatException e) {
+ throw new BadRequestException(
+ "windowStart, windowEnd or limit is not a numeric value.");
+ }
+ if (events == null) {
+ return new ATSEvents();
+ }
+ return events;
+ }
+
+ /**
+ * Store the given entities into the timeline store, and return the errors
+ * that happen during storing.
+ */
+ @POST
+ @Path("/")
+ @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public ATSPutErrors postEntities(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ ATSEntities entities) {
+ init(res);
+ if (entities == null) {
+ return new ATSPutErrors();
+ }
+ return store.put(entities);
+ }
+
+ private void init(HttpServletResponse response) {
+ response.setContentType(null);
+ }
+
+ private static SortedSet<String> parseArrayStr(String str, String delimiter) {
+ if (str == null) {
+ return null;
+ }
+ SortedSet<String> strSet = new TreeSet<String>();
+ String[] strs = str.split(delimiter);
+ for (String aStr : strs) {
+ strSet.add(aStr.trim());
+ }
+ return strSet;
+ }
+
+ private static NameValuePair parsePairStr(String str, String delimiter) {
+ if (str == null) {
+ return null;
+ }
+ String[] strs = str.split(delimiter, 2);
+ return new NameValuePair(strs[0].trim(), strs[1].trim());
+ }
+
+ private static Collection<NameValuePair> parsePairsStr(
+ String str, String aDelimiter, String pDelimiter) {
+ if (str == null) {
+ return null;
+ }
+ String[] strs = str.split(aDelimiter);
+ Set<NameValuePair> pairs = new HashSet<NameValuePair>();
+ for (String aStr : strs) {
+ pairs.add(parsePairStr(aStr, pDelimiter));
+ }
+ return pairs;
+ }
+
+ private static EnumSet<Field> parseFieldsStr(String str, String delimiter) {
+ if (str == null) {
+ return null;
+ }
+ String[] strs = str.split(delimiter);
+ List<Field> fieldList = new ArrayList<Field>();
+ for (String s : strs) {
+ fieldList.add(Field.valueOf(s.toUpperCase()));
+ }
+ if (fieldList.size() == 0)
+ return null;
+ Field f1 = fieldList.remove(fieldList.size() - 1);
+ if (fieldList.size() == 0)
+ return EnumSet.of(f1);
+ else
+ return EnumSet.of(f1, fieldList.toArray(new Field[fieldList.size()]));
+ }
+
+ private static Long parseLongStr(String str) {
+ return str == null ? null : Long.parseLong(str.trim());
+ }
+
+ private static String parseStr(String str) {
+ return str == null ? null : str.trim();
+ }
+
+}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java?rev=1564829&r1=1564828&r2=1564829&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java Wed Feb 5 17:02:26 2014
@@ -40,7 +40,7 @@ public class TestApplicationHistoryServe
Configuration config = new YarnConfiguration();
historyServer.init(config);
assertEquals(STATE.INITED, historyServer.getServiceState());
- assertEquals(2, historyServer.getServices().size());
+ assertEquals(3, historyServer.getServices().size());
ApplicationHistoryClientService historyService =
historyServer.getClientService();
assertNotNull(historyServer.getClientService());
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java?rev=1564829&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java Wed Feb 5 17:02:26 2014
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
+
+import static org.junit.Assert.assertEquals;
+
+import javax.ws.rs.core.MediaType;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities;
+import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity;
+import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent;
+import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents;
+import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.TestMemoryApplicationTimelineStore;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.junit.Test;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+
+public class TestATSWebServices extends JerseyTest {
+
+ private static ApplicationTimelineStore store;
+
+ private Injector injector = Guice.createInjector(new ServletModule() {
+
+ @Override
+ protected void configureServlets() {
+ bind(YarnJacksonJaxbJsonProvider.class);
+ bind(ATSWebServices.class);
+ bind(GenericExceptionHandler.class);
+ try{
+ store = mockApplicationTimelineStore();
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ bind(ApplicationTimelineStore.class).toInstance(store);
+ serve("/*").with(GuiceContainer.class);
+ }
+
+ });
+
+ public class GuiceServletConfig extends GuiceServletContextListener {
+
+ @Override
+ protected Injector getInjector() {
+ return injector;
+ }
+ }
+
+ private ApplicationTimelineStore mockApplicationTimelineStore()
+ throws Exception {
+ TestMemoryApplicationTimelineStore store =
+ new TestMemoryApplicationTimelineStore();
+ store.setup();
+ return store.getApplicationTimelineStore();
+ }
+
+ public TestATSWebServices() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.applicationhistoryservice.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter")
+ .servletPath("/")
+ .clientConfig(new DefaultClientConfig(YarnJacksonJaxbJsonProvider.class))
+ .build());
+ }
+
+ @Test
+ public void testAbout() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("apptimeline")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ ATSWebServices.AboutInfo about =
+ response.getEntity(ATSWebServices.AboutInfo.class);
+ Assert.assertNotNull(about);
+ Assert.assertEquals("Application Timeline API", about.getAbout());
+ }
+
+ @Test
+ public void testGetEntities() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("apptimeline")
+ .path("type_1")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ ATSEntities entities = response.getEntity(ATSEntities.class);
+ Assert.assertNotNull(entities);
+ Assert.assertEquals(2, entities.getEntities().size());
+ ATSEntity entity1 = entities.getEntities().get(0);
+ Assert.assertNotNull(entity1);
+ Assert.assertEquals("id_1", entity1.getEntityId());
+ Assert.assertEquals("type_1", entity1.getEntityType());
+ Assert.assertEquals(123l, entity1.getStartTime().longValue());
+ Assert.assertEquals(2, entity1.getEvents().size());
+ Assert.assertEquals(2, entity1.getPrimaryFilters().size());
+ Assert.assertEquals(4, entity1.getOtherInfo().size());
+ ATSEntity entity2 = entities.getEntities().get(1);
+ Assert.assertNotNull(entity2);
+ Assert.assertEquals("id_2", entity2.getEntityId());
+ Assert.assertEquals("type_1", entity2.getEntityType());
+ Assert.assertEquals(123l, entity2.getStartTime().longValue());
+ Assert.assertEquals(2, entity2.getEvents().size());
+ Assert.assertEquals(2, entity2.getPrimaryFilters().size());
+ Assert.assertEquals(4, entity2.getOtherInfo().size());
+ }
+
+ @Test
+ public void testGetEntity() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("apptimeline")
+ .path("type_1").path("id_1")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ ATSEntity entity = response.getEntity(ATSEntity.class);
+ Assert.assertNotNull(entity);
+ Assert.assertEquals("id_1", entity.getEntityId());
+ Assert.assertEquals("type_1", entity.getEntityType());
+ Assert.assertEquals(123l, entity.getStartTime().longValue());
+ Assert.assertEquals(2, entity.getEvents().size());
+ Assert.assertEquals(2, entity.getPrimaryFilters().size());
+ Assert.assertEquals(4, entity.getOtherInfo().size());
+ }
+
+ @Test
+ public void testGetEvents() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("apptimeline")
+ .path("type_1").path("events")
+ .queryParam("entityId", "id_1")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ ATSEvents events = response.getEntity(ATSEvents.class);
+ Assert.assertNotNull(events);
+ Assert.assertEquals(1, events.getAllEvents().size());
+ ATSEvents.ATSEventsOfOneEntity partEvents = events.getAllEvents().get(0);
+ Assert.assertEquals(2, partEvents.getEvents().size());
+ ATSEvent event1 = partEvents.getEvents().get(0);
+ Assert.assertEquals(456l, event1.getTimestamp());
+ Assert.assertEquals("end_event", event1.getEventType());
+ Assert.assertEquals(1, event1.getEventInfo().size());
+ ATSEvent event2 = partEvents.getEvents().get(1);
+ Assert.assertEquals(123l, event2.getTimestamp());
+ Assert.assertEquals("start_event", event2.getEventType());
+ Assert.assertEquals(0, event2.getEventInfo().size());
+ }
+
+ @Test
+ public void testPostEntities() throws Exception {
+ ATSEntities entities = new ATSEntities();
+ ATSEntity entity = new ATSEntity();
+ entity.setEntityId("test id");
+ entity.setEntityType("test type");
+ entity.setStartTime(System.currentTimeMillis());
+ entities.addEntity(entity);
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("apptimeline")
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class, entities);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ ATSPutErrors errors = response.getEntity(ATSPutErrors.class);
+ Assert.assertNotNull(errors);
+ Assert.assertEquals(0, errors.getErrors().size());
+ // verify the entity exists in the store
+ response = r.path("ws").path("v1").path("apptimeline")
+ .path("test type").path("test id")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ entity = response.getEntity(ATSEntity.class);
+ Assert.assertNotNull(entity);
+ Assert.assertEquals("test id", entity.getEntityId());
+ Assert.assertEquals("test type", entity.getEntityType());
+ }
+
+}