You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/21 18:39:06 UTC
[04/52] [abbrv] hadoop git commit: YARN-6395. Integrate service app
master to write data into ATSv2. Contributed by Rohith Sharma K S
YARN-6395. Integrate service app master to write data into ATSv2. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ff8a082
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ff8a082
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ff8a082
Branch: refs/heads/yarn-native-services
Commit: 6ff8a0826c615c396e64b735a153a6635affc6a3
Parents: 4c242ac
Author: Jian He <ji...@apache.org>
Authored: Thu Mar 30 15:58:51 2017 +0800
Committer: Jian He <ji...@apache.org>
Committed: Fri Jul 21 11:38:12 2017 -0700
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 63 ++++
.../slider/server/appmaster/state/AppState.java | 11 +
.../ServiceTimelinePublisher.java | 365 +++++++++++++++++++
.../timelineservice/SliderMetricsSink.java | 102 ++++++
.../SliderTimelineEntityType.java | 39 ++
.../timelineservice/SliderTimelineEvent.java | 34 ++
.../SliderTimelineMetricsConstants.java | 91 +++++
.../appmaster/timelineservice/package-info.java | 27 ++
.../TestServiceTimelinePublisher.java | 285 +++++++++++++++
.../appmaster/timelineservice/package-info.java | 26 ++
10 files changed, 1043 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff8a082/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 4922c2d..4fa2769 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
@@ -146,6 +147,8 @@ import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
+import org.apache.slider.server.appmaster.timelineservice.SliderMetricsSink;
import org.apache.slider.server.appmaster.web.SliderAMWebApp;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
@@ -240,6 +243,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private AMRMClientAsync asyncRMClient;
+ /** Handle to communicate with the timeline service */
+ private TimelineClient timelineClient;
+
+ private boolean timelineServiceEnabled = false;
+
+ ServiceTimelinePublisher serviceTimelinePublisher;
+
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private RMOperationHandler rmOperationHandler;
@@ -483,6 +493,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
addService(executorService);
addService(actionQueues);
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ timelineServiceEnabled = true;
+ log.info("Enabled YARN timeline service v2. ");
+ }
//init all child services
super.serviceInit(conf);
@@ -650,6 +664,20 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//now bring it up
deployChildService(asyncRMClient);
+ if (timelineServiceEnabled) {
+ timelineClient = TimelineClient.createTimelineClient(appid);
+ asyncRMClient.registerTimelineClient(timelineClient);
+ timelineClient.init(getConfig());
+ timelineClient.start();
+ log.info("Timeline client started.");
+
+ serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
+ serviceTimelinePublisher.init(getConfig());
+ serviceTimelinePublisher.start();
+ appState.setServiceTimelinePublisher(serviceTimelinePublisher);
+ log.info("ServiceTimelinePublisher started.");
+ }
+
// nmclient relays callbacks back to this class
nmClientAsync = new NMClientAsyncImpl("nmclient", this);
@@ -781,6 +809,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
liveContainers = amRegistrationData.getContainersFromPreviousAttempts();
DefaultMetricsSystem.initialize("SliderAppMaster");
+ if (timelineServiceEnabled) {
+ DefaultMetricsSystem.instance().register("SliderMetricsSink",
+ "For processing metrics to ATS",
+ new SliderMetricsSink(serviceTimelinePublisher));
+ log.info("SliderMetricsSink registered.");
+ }
//determine the location for the role history data
Path historyDir = new Path(appDir, HISTORY_DIR_NAME);
@@ -1132,6 +1166,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
yarnRegistryOperations.getSelfRegistrationPath(),
true);
}
+ if (timelineServiceEnabled) {
+ serviceTimelinePublisher.serviceAttemptRegistered(appState);
+ }
}
/**
@@ -1184,6 +1221,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
container.setState(org.apache.slider.api.resource.ContainerState.INIT);
container.setBareHost(instance.host);
instance.providerRole.component.addContainer(container);
+
+ if (timelineServiceEnabled) {
+ serviceTimelinePublisher.componentInstanceStarted(container,
+ instance.providerRole.component.getName());
+ }
return true;
}
@@ -1345,6 +1387,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
releaseAllContainers(application);
DefaultMetricsSystem.shutdown();
+ if (timelineServiceEnabled) {
+ serviceTimelinePublisher.serviceAttemptUnregistered(appState, stopAction);
+ serviceTimelinePublisher.stop();
+ timelineClient.stop();
+ }
+
// When the application completes, it should send a finish application
// signal to the RM
log.info("Application completed. Signalling finish to RM");
@@ -1490,6 +1538,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
if(!result.unknownNode) {
queue(new UnregisterComponentInstance(containerId, 0,
TimeUnit.MILLISECONDS));
+ if (timelineServiceEnabled && result.roleInstance != null) {
+ serviceTimelinePublisher
+ .componentInstanceFinished(result.roleInstance);
+ }
}
}
@@ -1967,6 +2019,17 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
nmClientAsync.getContainerStatusAsync(containerId,
cinfo.container.getNodeId());
}
+ } else if (timelineServiceEnabled) {
+ RoleInstance instance = appState.getOwnedContainer(containerId);
+ if (instance != null) {
+ org.apache.slider.api.resource.Container container =
+ instance.providerRole.component
+ .getContainer(containerId.toString());
+ if (container != null) {
+ serviceTimelinePublisher.componentInstanceUpdated(container,
+ instance.providerRole.component.getName());
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff8a082/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index e891a27..84b8140 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -64,6 +64,7 @@ import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
+import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -207,6 +208,8 @@ public class AppState {
private Resource maxResource;
private SliderMetrics appMetrics;
+
+ private ServiceTimelinePublisher serviceTimelinePublisher;
/**
* Create an instance
* @param recordFactory factory for YARN records
@@ -1762,6 +1765,10 @@ public class AppState {
log.info("Releasing container. Log: " + url);
try {
containerReleaseSubmitted(possible);
+ // update during finish call
+ if (serviceTimelinePublisher != null) {
+ serviceTimelinePublisher.componentInstanceFinished(instance);
+ }
} catch (SliderInternalStateException e) {
log.warn("when releasing container {} :", possible, e);
}
@@ -1948,4 +1955,8 @@ public class AppState {
}
return naming;
}
+
+ public void setServiceTimelinePublisher(ServiceTimelinePublisher serviceTimelinePublisher) {
+ this.serviceTimelinePublisher = serviceTimelinePublisher;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff8a082/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
new file mode 100644
index 0000000..3ff4200
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.timelineservice;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.api.resource.Configuration;
+import org.apache.slider.api.resource.Container;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.server.appmaster.actions.ActionStopSlider;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A single service that publishes all the Timeline Entities.
+ */
+public class ServiceTimelinePublisher extends CompositeService {
+
+ // Number of bytes of config which can be published in one shot to ATSv2.
+ public static final int ATS_CONFIG_PUBLISH_SIZE_BYTES = 10 * 1024;
+
+ private TimelineClient timelineClient;
+
+ private volatile boolean stopped = false;
+
+ private static final Logger log =
+ LoggerFactory.getLogger(ServiceTimelinePublisher.class);
+
+ @Override
+ protected void serviceStop() throws Exception {
+ stopped = true;
+ }
+
+ public boolean isStopped() {
+ return stopped;
+ }
+
+ public ServiceTimelinePublisher(TimelineClient client) {
+ super(ServiceTimelinePublisher.class.getName());
+ timelineClient = client;
+ }
+
+ public void serviceAttemptRegistered(AppState appState) {
+ Application application = appState.getClusterStatus();
+ long currentTimeMillis = application.getLaunchTime() == null
+ ? System.currentTimeMillis() : application.getLaunchTime().getTime();
+
+ TimelineEntity entity = createServiceAttemptEntity(application.getId());
+ entity.setCreatedTime(currentTimeMillis);
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(SliderTimelineMetricsConstants.NAME, application.getName());
+ entityInfos.put(SliderTimelineMetricsConstants.STATE,
+ application.getState().toString());
+ entityInfos.put(SliderTimelineMetricsConstants.LAUNCH_TIME,
+ currentTimeMillis);
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent startEvent = new TimelineEvent();
+ startEvent.setId(SliderTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString());
+ startEvent.setTimestamp(currentTimeMillis);
+ entity.addEvent(startEvent);
+
+ // publish before configurations published
+ putEntity(entity);
+
+ // publish application specific configurations
+ publishConfigurations(application.getConfiguration(), application.getId(),
+ SliderTimelineEntityType.SERVICE_ATTEMPT.toString(), true);
+
+ // publish component as separate entity.
+ publishComponents(application.getComponents());
+ }
+
+ public void serviceAttemptUnregistered(AppState appState,
+ ActionStopSlider stopAction) {
+ long currentTimeMillis = System.currentTimeMillis();
+
+ TimelineEntity entity =
+ createServiceAttemptEntity(appState.getClusterStatus().getId());
+
+ // add info
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(SliderTimelineMetricsConstants.EXIT_STATUS_CODE,
+ stopAction.getExitCode());
+ entityInfos.put(SliderTimelineMetricsConstants.STATE,
+ stopAction.getFinalApplicationStatus().toString());
+ if (stopAction.getMessage() != null) {
+ entityInfos.put(SliderTimelineMetricsConstants.EXIT_REASON,
+ stopAction.getMessage());
+ }
+ if (stopAction.getEx() != null) {
+ entityInfos.put(SliderTimelineMetricsConstants.DIAGNOSTICS_INFO,
+ stopAction.getEx().toString());
+ }
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent startEvent = new TimelineEvent();
+ startEvent
+ .setId(SliderTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString());
+ startEvent.setTimestamp(currentTimeMillis);
+ entity.addEvent(startEvent);
+
+ putEntity(entity);
+ }
+
+ public void componentInstanceStarted(Container container,
+ String componentName) {
+
+ TimelineEntity entity = createComponentInstanceEntity(container.getId());
+ entity.setCreatedTime(container.getLaunchTime().getTime());
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(SliderTimelineMetricsConstants.BARE_HOST,
+ container.getBareHost());
+ entityInfos.put(SliderTimelineMetricsConstants.STATE,
+ container.getState().toString());
+ entityInfos.put(SliderTimelineMetricsConstants.LAUNCH_TIME,
+ container.getLaunchTime().getTime());
+ entityInfos.put(SliderTimelineMetricsConstants.COMPONENT_NAME,
+ componentName);
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent startEvent = new TimelineEvent();
+ startEvent
+ .setId(SliderTimelineEvent.COMPONENT_INSTANCE_REGISTERED.toString());
+ startEvent.setTimestamp(container.getLaunchTime().getTime());
+ entity.addEvent(startEvent);
+
+ putEntity(entity);
+ }
+
+ public void componentInstanceFinished(RoleInstance instance) {
+ TimelineEntity entity = createComponentInstanceEntity(instance.id);
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(SliderTimelineMetricsConstants.EXIT_STATUS_CODE,
+ instance.exitCode);
+ entityInfos.put(SliderTimelineMetricsConstants.DIAGNOSTICS_INFO,
+ instance.diagnostics);
+ // TODO need to change the state based on enum value.
+ entityInfos.put(SliderTimelineMetricsConstants.STATE, "FINISHED");
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent startEvent = new TimelineEvent();
+ startEvent
+ .setId(SliderTimelineEvent.COMPONENT_INSTANCE_UNREGISTERED.toString());
+ startEvent.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(startEvent);
+
+ putEntity(entity);
+ }
+
+ public void componentInstanceUpdated(Container container,
+ String componentName) {
+ TimelineEntity entity = createComponentInstanceEntity(container.getId());
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(SliderTimelineMetricsConstants.IP, container.getIp());
+ entityInfos.put(SliderTimelineMetricsConstants.HOSTNAME,
+ container.getHostname());
+ entityInfos.put(SliderTimelineMetricsConstants.STATE,
+ container.getState().toString());
+ entity.addInfo(entityInfos);
+
+ TimelineEvent updateEvent = new TimelineEvent();
+ updateEvent
+ .setId(SliderTimelineEvent.COMPONENT_INSTANCE_UPDATED.toString());
+ updateEvent.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(updateEvent);
+
+ putEntity(entity);
+ }
+
+ private void publishComponents(List<Component> components) {
+ long currentTimeMillis = System.currentTimeMillis();
+ for (Component component : components) {
+ TimelineEntity entity = createComponentEntity(component.getName());
+ entity.setCreatedTime(currentTimeMillis);
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(SliderTimelineMetricsConstants.ARTIFACT_ID,
+ component.getArtifact().getId());
+ entityInfos.put(SliderTimelineMetricsConstants.ARTIFACT_TYPE,
+ component.getArtifact().getType().toString());
+ if (component.getResource().getProfile() != null) {
+ entityInfos.put(SliderTimelineMetricsConstants.RESOURCE_PROFILE,
+ component.getResource().getProfile());
+ }
+ entityInfos.put(SliderTimelineMetricsConstants.RESOURCE_CPU,
+ component.getResource().getCpus());
+ entityInfos.put(SliderTimelineMetricsConstants.RESOURCE_MEMORY,
+ component.getResource().getMemory());
+
+ if (component.getLaunchCommand() != null) {
+ entityInfos.put(SliderTimelineMetricsConstants.LAUNCH_COMMAND,
+ component.getLaunchCommand());
+ }
+ entityInfos.put(SliderTimelineMetricsConstants.UNIQUE_COMPONENT_SUPPORT,
+ component.getUniqueComponentSupport().toString());
+ entityInfos.put(SliderTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER,
+ component.getRunPrivilegedContainer().toString());
+ if (component.getPlacementPolicy() != null) {
+ entityInfos.put(SliderTimelineMetricsConstants.PLACEMENT_POLICY,
+ component.getPlacementPolicy().getLabel());
+ }
+ entity.addInfo(entityInfos);
+
+ putEntity(entity);
+
+ // publish component specific configurations
+ publishConfigurations(component.getConfiguration(), component.getName(),
+ SliderTimelineEntityType.COMPONENT.toString(), false);
+ }
+ }
+
+ private void publishConfigurations(Configuration configuration,
+ String entityId, String entityType, boolean isServiceAttemptEntity) {
+ if (isServiceAttemptEntity) {
+ // publish slider-client.xml properties at service level
+ publishConfigurations(SliderUtils.loadSliderClientXML().iterator(),
+ entityId, entityType);
+ }
+ publishConfigurations(configuration.getProperties().entrySet().iterator(),
+ entityId, entityType);
+
+ publishConfigurations(configuration.getEnv().entrySet().iterator(),
+ entityId, entityType);
+
+ for (ConfigFile configFile : configuration.getFiles()) {
+ publishConfigurations(configFile.getProps().entrySet().iterator(),
+ entityId, entityType);
+ }
+ }
+
+ private void publishConfigurations(Iterator<Entry<String, String>> iterator,
+ String entityId, String entityType) {
+ int configSize = 0;
+ TimelineEntity entity = createTimelineEntity(entityId, entityType);
+ while (iterator.hasNext()) {
+ Entry<String, String> entry = iterator.next();
+ int size = entry.getKey().length() + entry.getValue().length();
+ configSize += size;
+ // Configs are split into multiple entities if they exceed 100kb in size.
+ if (configSize > ATS_CONFIG_PUBLISH_SIZE_BYTES) {
+ if (entity.getConfigs().size() > 0) {
+ putEntity(entity);
+ entity = createTimelineEntity(entityId, entityType);
+ }
+ configSize = size;
+ }
+ entity.addConfig(entry.getKey(), entry.getValue());
+ }
+ if (configSize > 0) {
+ putEntity(entity);
+ }
+ }
+
+ /**
+ * Called from SliderMetricsSink at regular interval of time.
+ * @param metrics of service or components
+ * @param entityId Id of entity
+ * @param entityType Type of entity
+ * @param timestamp
+ */
+ public void publishMetrics(Iterable<AbstractMetric> metrics, String entityId,
+ String entityType, long timestamp) {
+ TimelineEntity entity = createTimelineEntity(entityId, entityType);
+ Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
+ for (AbstractMetric metric : metrics) {
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setId(metric.name());
+ timelineMetric.addValue(timestamp, metric.value());
+ entityMetrics.add(timelineMetric);
+ }
+ entity.setMetrics(entityMetrics);
+ putEntity(entity);
+ }
+
+ private TimelineEntity createServiceAttemptEntity(String serviceId) {
+ TimelineEntity entity = createTimelineEntity(serviceId,
+ SliderTimelineEntityType.SERVICE_ATTEMPT.toString());
+ return entity;
+ }
+
+ private TimelineEntity createComponentInstanceEntity(String instanceId) {
+ TimelineEntity entity = createTimelineEntity(instanceId,
+ SliderTimelineEntityType.COMPONENT_INSTANCE.toString());
+ return entity;
+ }
+
+ private TimelineEntity createComponentEntity(String componentId) {
+ TimelineEntity entity = createTimelineEntity(componentId,
+ SliderTimelineEntityType.COMPONENT.toString());
+ return entity;
+ }
+
+ private TimelineEntity createTimelineEntity(String entityId,
+ String entityType) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(entityId);
+ entity.setType(entityType);
+ return entity;
+ }
+
+ private void putEntity(TimelineEntity entity) {
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug("Publishing the entity " + entity + ", JSON-style content: "
+ + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ }
+ if (timelineClient != null) {
+ timelineClient.putEntitiesAsync(entity);
+ } else {
+ log.error("Seems like client has been removed before the entity "
+ + "could be published for " + entity);
+ }
+ } catch (Exception e) {
+ log.error("Error when publishing entity " + entity, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff8a082/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java
new file mode 100644
index 0000000..869ae26
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.timelineservice;
+
+import org.apache.commons.configuration2.SubsetConfiguration;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsSink;
+import org.apache.hadoop.metrics2.MetricsTag;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Write the metrics to a ATSv2. Generally, this class is instantiated via
+ * hadoop-metrics2 property files. Specifically, you would create this class by
+ * adding the following to by This would actually be set as: <code>
+ * [prefix].sink.[some instance name].class
+ * =org.apache.slider.server.appmaster.timelineservice.SliderMetricsSink
+ * </code>, where <tt>prefix</tt> is "atsv2": and <tt>some instance name</tt> is
+ * just any unique name, so properties can be differentiated if there are
+ * multiple sinks of the same type created
+ */
+public class SliderMetricsSink implements MetricsSink {
+
+ private static final Logger log =
+ LoggerFactory.getLogger(SliderMetricsSink.class);
+
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+
+ public SliderMetricsSink() {
+
+ }
+
+ public SliderMetricsSink(ServiceTimelinePublisher publisher) {
+ serviceTimelinePublisher = publisher;
+ }
+
+ /**
+ * Publishes service and component metrics to ATS.
+ */
+ @Override
+ public void putMetrics(MetricsRecord record) {
+ if (serviceTimelinePublisher.isStopped()) {
+ log.warn("ServiceTimelinePublisher has stopped. "
+ + "Not publishing any more metrics to ATS.");
+ return;
+ }
+
+ boolean isServiceMetrics = false;
+ boolean isComponentMetrics = false;
+ String appId = null;
+ for (MetricsTag tag : record.tags()) {
+ if (tag.name().equals("type") && tag.value().equals("service")) {
+ isServiceMetrics = true;
+ } else if (tag.name().equals("type") && tag.value().equals("component")) {
+ isComponentMetrics = true;
+ break; // if component metrics, no more information required from tag so
+ // break the loop
+ } else if (tag.name().equals("appId")) {
+ appId = tag.value();
+ }
+ }
+
+ if (isServiceMetrics && appId != null) {
+ if (log.isDebugEnabled()) {
+ log.debug("Publishing service metrics. " + record);
+ }
+ serviceTimelinePublisher.publishMetrics(record.metrics(), appId,
+ SliderTimelineEntityType.SERVICE_ATTEMPT.toString(),
+ record.timestamp());
+ } else if (isComponentMetrics) {
+ if (log.isDebugEnabled()) {
+ log.debug("Publishing Component metrics. " + record);
+ }
+ serviceTimelinePublisher.publishMetrics(record.metrics(), record.name(),
+ SliderTimelineEntityType.COMPONENT.toString(), record.timestamp());
+ }
+ }
+
+ @Override
+ public void init(SubsetConfiguration conf) {
+ }
+
+ @Override
+ public void flush() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff8a082/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java
new file mode 100644
index 0000000..908754f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.timelineservice;
+
+/**
+ * Slider entities that are published to ATS.
+ */
+public enum SliderTimelineEntityType {
+ /**
+ * Used for publishing service entity information.
+ */
+ SERVICE_ATTEMPT,
+
+ /**
+ * Used for publishing component entity information.
+ */
+ COMPONENT,
+
+ /**
+ * Used for publishing component instance entity information.
+ */
+ COMPONENT_INSTANCE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff8a082/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java
new file mode 100644
index 0000000..04f0219
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.timelineservice;
+
+/**
+ * Events that are used to store in ATS.
+ */
+public enum SliderTimelineEvent {
+ SERVICE_ATTEMPT_REGISTERED,
+
+ SERVICE_ATTEMPT_UNREGISTERED,
+
+ COMPONENT_INSTANCE_REGISTERED,
+
+ COMPONENT_INSTANCE_UNREGISTERED,
+
+ COMPONENT_INSTANCE_UPDATED
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff8a082/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
new file mode 100644
index 0000000..23e059d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.timelineservice;
+
+/**
+ * Constants which are stored as key in ATS
+ */
+public final class SliderTimelineMetricsConstants {
+
+ public static final String URI = "URI";
+
+ public static final String NAME = "NAME";
+
+ public static final String STATE = "STATE";
+
+ public static final String EXIT_STATUS_CODE = "EXIT_STATUS_CODE";
+
+ public static final String EXIT_REASON = "EXIT_REASON";
+
+ public static final String DIAGNOSTICS_INFO = "DIAGNOSTICS_INFO";
+
+ public static final String LAUNCH_TIME = "LAUNCH_TIME";
+
+ public static final String LAUNCH_COMMAND = "LAUNCH_COMMAND";
+
+ public static final String TOTAL_CONTAINERS = "NUMBER_OF_CONTAINERS";
+
+ public static final String RUNNING_CONTAINERS =
+ "NUMBER_OF_RUNNING_CONTAINERS";
+
+ /**
+ * Artifacts constants.
+ */
+ public static final String ARTIFACT_ID = "ARTIFACT_ID";
+
+ public static final String ARTIFACT_TYPE = "ARTIFACT_TYPE";
+
+ public static final String ARTIFACT_URI = "ARTIFACT_URI";
+
+ /**
+ * Resource constants.
+ */
+ public static final String RESOURCE_CPU = "RESOURCE_CPU";
+
+ public static final String RESOURCE_MEMORY = "RESOURCE_MEMORY";
+
+ public static final String RESOURCE_PROFILE = "RESOURCE_PROFILE";
+
+ /**
+ * component instance constants.
+ */
+ public static final String IP = "IP";
+
+ public static final String HOSTNAME = "HOSTNAME";
+
+ public static final String BARE_HOST = "BARE_HOST";
+
+ public static final String COMPONENT_NAME = "COMPONENT_NAME";
+
+ /**
+ * component constants.
+ */
+ public static final String DEPENDENCIES = "DEPENDENCIES";
+
+ public static final String DESCRIPTION = "DESCRIPTION";
+
+ public static final String UNIQUE_COMPONENT_SUPPORT =
+ "UNIQUE_COMPONENT_SUPPORT";
+
+ public static final String RUN_PRIVILEGED_CONTAINER =
+ "RUN_PRIVILEGED_CONTAINER";
+
+ public static final String PLACEMENT_POLICY = "PLACEMENT_POLICY";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff8a082/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/package-info.java
new file mode 100644
index 0000000..0bffc90
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * ATS implementation
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.slider.server.appmaster.timelineservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff8a082/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
new file mode 100644
index 0000000..1209aef
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.timelineservice;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.ApplicationState;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.Container;
+import org.apache.slider.api.resource.ContainerState;
+import org.apache.slider.api.resource.PlacementPolicy;
+import org.apache.slider.api.resource.Resource;
+import org.apache.slider.server.appmaster.actions.ActionStopSlider;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for ServiceTimelinePublisher.
+ */
+public class TestServiceTimelinePublisher {
+ private TimelineClient timelineClient;
+ private Configuration config;
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+ private static String SERVICE_NAME = "HBASE";
+ private static String SERVICEID = "application_1490093646524_0005";
+ private static String ARTIFACTID = "ARTIFACTID";
+ private static String COMPONENT_NAME = "DEFAULT";
+ private static String CONTAINER_ID =
+ "container_e02_1490093646524_0005_01_000001";
+ private static String CONTAINER_IP =
+ "localhost";
+ private static String CONTAINER_HOSTNAME =
+ "cnl124-localhost.site";
+ private static String CONTAINER_BAREHOST =
+ "localhost.com";
+
+ @Before
+ public void setUp() throws Exception {
+ config = new Configuration();
+ timelineClient = new DummyTimelineClient();
+ serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
+ timelineClient.init(config);
+ serviceTimelinePublisher.init(config);
+ timelineClient.start();
+ serviceTimelinePublisher.start();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ serviceTimelinePublisher.stop();
+ timelineClient.stop();
+ }
+
+ @Test
+ public void testServiceAttemptEntity() {
+ AppState appState = createMockAppState();
+ int exitCode = 0;
+ String message = "Stopped by user";
+ ActionStopSlider stopAction = mock(ActionStopSlider.class);
+ when(stopAction.getExitCode()).thenReturn(exitCode);
+ when(stopAction.getFinalApplicationStatus())
+ .thenReturn(FinalApplicationStatus.SUCCEEDED);
+ when(stopAction.getMessage()).thenReturn(message);
+
+ serviceTimelinePublisher.serviceAttemptRegistered(appState);
+
+ Collection<TimelineEntity> lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ // 2 entities because during registration component also registered.
+ assertEquals(2, lastPublishedEntities.size());
+ for (TimelineEntity timelineEntity : lastPublishedEntities) {
+ if (timelineEntity.getType() == SliderTimelineEntityType.COMPONENT
+ .toString()) {
+ verifyComponentTimelineEntity(timelineEntity);
+ } else {
+ verifyServiceAttemptTimelineEntity(timelineEntity, 0, null, true);
+ }
+ }
+
+ serviceTimelinePublisher.serviceAttemptUnregistered(appState, stopAction);
+ lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ for (TimelineEntity timelineEntity : lastPublishedEntities) {
+ if (timelineEntity.getType() == SliderTimelineEntityType.SERVICE_ATTEMPT
+ .toString()) {
+ verifyServiceAttemptTimelineEntity(timelineEntity, exitCode, message,
+ false);
+ }
+ }
+ }
+
+ @Test
+ public void testComponentInstanceEntity() {
+ Container container = new Container();
+ container.id(CONTAINER_ID).ip(CONTAINER_IP).bareHost(CONTAINER_BAREHOST)
+ .hostname(CONTAINER_HOSTNAME).state(ContainerState.INIT)
+ .launchTime(new Date());
+ serviceTimelinePublisher.componentInstanceStarted(container,
+ COMPONENT_NAME);
+
+ Collection<TimelineEntity> lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ assertEquals(1, lastPublishedEntities.size());
+ TimelineEntity entity = lastPublishedEntities.iterator().next();
+
+ assertEquals(1, entity.getEvents().size());
+ assertEquals(CONTAINER_ID, entity.getId());
+ assertEquals(CONTAINER_BAREHOST,
+ entity.getInfo().get(SliderTimelineMetricsConstants.BARE_HOST));
+ assertEquals(COMPONENT_NAME,
+ entity.getInfo().get(SliderTimelineMetricsConstants.COMPONENT_NAME));
+ assertEquals(ContainerState.INIT.toString(),
+ entity.getInfo().get(SliderTimelineMetricsConstants.STATE));
+
+ // updated container state
+ container.setState(ContainerState.READY);
+ serviceTimelinePublisher.componentInstanceUpdated(container,
+ COMPONENT_NAME);
+ lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ assertEquals(1, lastPublishedEntities.size());
+ entity = lastPublishedEntities.iterator().next();
+ assertEquals(2, entity.getEvents().size());
+ assertEquals(ContainerState.READY.toString(),
+ entity.getInfo().get(SliderTimelineMetricsConstants.STATE));
+
+ }
+
+ private void verifyServiceAttemptTimelineEntity(TimelineEntity timelineEntity,
+ int exitCode, String message, boolean isRegistedEntity) {
+ assertEquals(SERVICEID, timelineEntity.getId());
+ assertEquals(SERVICE_NAME,
+ timelineEntity.getInfo().get(SliderTimelineMetricsConstants.NAME));
+ if (isRegistedEntity) {
+ assertEquals(ApplicationState.STARTED.toString(),
+ timelineEntity.getInfo().get(SliderTimelineMetricsConstants.STATE));
+ assertEquals(SliderTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString(),
+ timelineEntity.getEvents().iterator().next().getId());
+ } else {
+ assertEquals("SUCCEEDED",
+ timelineEntity.getInfo().get(SliderTimelineMetricsConstants.STATE));
+ assertEquals(exitCode, timelineEntity.getInfo()
+ .get(SliderTimelineMetricsConstants.EXIT_STATUS_CODE));
+ assertEquals(message, timelineEntity.getInfo()
+ .get(SliderTimelineMetricsConstants.EXIT_REASON));
+
+ assertEquals(2, timelineEntity.getEvents().size());
+ assertEquals(SliderTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString(),
+ timelineEntity.getEvents().iterator().next().getId());
+ }
+ }
+
+ private void verifyComponentTimelineEntity(TimelineEntity entity) {
+ Map<String, Object> info = entity.getInfo();
+ assertEquals("DEFAULT", entity.getId());
+ assertEquals(ARTIFACTID,
+ info.get(SliderTimelineMetricsConstants.ARTIFACT_ID));
+ assertEquals("DOCKER",
+ info.get(SliderTimelineMetricsConstants.ARTIFACT_TYPE));
+ assertEquals("medium",
+ info.get(SliderTimelineMetricsConstants.RESOURCE_PROFILE));
+ assertEquals(1, info.get(SliderTimelineMetricsConstants.RESOURCE_CPU));
+ assertEquals("1024",
+ info.get(SliderTimelineMetricsConstants.RESOURCE_MEMORY));
+ assertEquals("sleep 1",
+ info.get(SliderTimelineMetricsConstants.LAUNCH_COMMAND));
+ assertEquals("false",
+ info.get(SliderTimelineMetricsConstants.UNIQUE_COMPONENT_SUPPORT));
+ assertEquals("false",
+ info.get(SliderTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER));
+ assertEquals("label",
+ info.get(SliderTimelineMetricsConstants.PLACEMENT_POLICY));
+ }
+
+ private static AppState createMockAppState() {
+ AppState appState = mock(AppState.class);
+ Application application = mock(Application.class);
+
+ when(application.getId()).thenReturn(SERVICEID);
+ when(application.getLaunchTime()).thenReturn(new Date());
+ when(application.getState()).thenReturn(ApplicationState.STARTED);
+ when(application.getName()).thenReturn(SERVICE_NAME);
+ when(application.getConfiguration())
+ .thenReturn(new org.apache.slider.api.resource.Configuration());
+
+ Component component = mock(Component.class);
+ Artifact artifact = new Artifact();
+ artifact.setId(ARTIFACTID);
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory(1024 + "");
+ resource.setProfile("medium");
+ when(component.getArtifact()).thenReturn(artifact);
+ when(component.getName()).thenReturn(COMPONENT_NAME);
+ when(component.getResource()).thenReturn(resource);
+ when(component.getLaunchCommand()).thenReturn("sleep 1");
+ PlacementPolicy placementPolicy = new PlacementPolicy();
+ placementPolicy.setLabel("label");
+ when(component.getPlacementPolicy()).thenReturn(placementPolicy);
+ when(component.getConfiguration())
+ .thenReturn(new org.apache.slider.api.resource.Configuration());
+ List<Component> components = new ArrayList<Component>();
+ components.add(component);
+
+ when(application.getComponents()).thenReturn(components);
+ when(appState.getClusterStatus()).thenReturn(application);
+ return appState;
+ }
+
+ public static void main(String[] args) {
+ Application application = createMockAppState().getClusterStatus();
+ System.out.println(application.getConfiguration());
+ }
+
+ protected static class DummyTimelineClient extends TimelineClientImpl {
+ private Map<Identifier, TimelineEntity> lastPublishedEntities =
+ new HashMap<>();
+
+ @Override
+ public void putEntitiesAsync(TimelineEntity... entities)
+ throws IOException, YarnException {
+ for (TimelineEntity timelineEntity : entities) {
+ TimelineEntity entity =
+ lastPublishedEntities.get(timelineEntity.getIdentifier());
+ if (entity == null) {
+ lastPublishedEntities.put(timelineEntity.getIdentifier(),
+ timelineEntity);
+ } else {
+ entity.addMetrics(timelineEntity.getMetrics());
+ entity.addEvents(timelineEntity.getEvents());
+ entity.addInfo(timelineEntity.getInfo());
+ entity.addConfigs(timelineEntity.getConfigs());
+ entity.addRelatesToEntities(timelineEntity.getRelatesToEntities());
+ entity
+ .addIsRelatedToEntities(timelineEntity.getIsRelatedToEntities());
+ }
+ }
+ }
+
+ public Collection<TimelineEntity> getLastPublishedEntities() {
+ return lastPublishedEntities.values();
+ }
+
+ public void reset() {
+ lastPublishedEntities = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff8a082/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/package-info.java
new file mode 100644
index 0000000..f274cd0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/package-info.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * ATS tests
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.slider.server.appmaster.timelineservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org