You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/21 18:39:23 UTC
[21/52] [abbrv] hadoop git commit: YARN-6446. Revisit ATSv2
integration to ensure all required information is published. Contributed by
Rohith Sharma K S
YARN-6446. Revisit ATSv2 integration to ensure all required information is published. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8be2e50f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8be2e50f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8be2e50f
Branch: refs/heads/yarn-native-services
Commit: 8be2e50f6922931dca7b3a1e370dc5781d745125
Parents: 7f037ee
Author: Jian He <ji...@apache.org>
Authored: Thu May 4 09:15:01 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Fri Jul 21 11:38:13 2017 -0700
----------------------------------------------------------------------
.../apache/slider/providers/ProviderService.java | 8 ++++++++
.../providers/docker/DockerProviderService.java | 10 ++++++++++
.../slider/server/appmaster/SliderAppMaster.java | 4 +++-
.../ServiceTimelinePublisher.java | 8 +++++++-
.../SliderTimelineMetricsConstants.java | 2 ++
.../TestServiceTimelinePublisher.java | 19 +++++++++++++++----
6 files changed, 45 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 7e92bfa..edd313b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -28,6 +28,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import java.io.IOException;
@@ -57,4 +58,11 @@ public interface ProviderService extends Service {
*/
boolean processContainerStatus(ContainerId containerId,
ContainerStatus status);
+
+ /**
+ * Set service publisher.
+ * @param serviceTimelinePublisher service publisher.
+ */
+ void setServiceTimelinePublisher(
+ ServiceTimelinePublisher serviceTimelinePublisher);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index 93a481c..482bb27 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -38,6 +38,7 @@ import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +58,7 @@ public class DockerProviderService extends AbstractService
private static final String QUICK_LINKS = "quicklinks";
protected StateAccessForProviders amState;
protected YarnRegistryViewForProviders yarnRegistry;
+ private ServiceTimelinePublisher serviceTimelinePublisher;
protected DockerProviderService() {
super("DockerProviderService");
@@ -126,6 +128,9 @@ public class DockerProviderService extends AbstractService
PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS,
application.getQuicklinks().entrySet());
amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf);
+ if (serviceTimelinePublisher != null) {
+ serviceTimelinePublisher.serviceAttemptUpdated(application);
+ }
}
public boolean processContainerStatus(ContainerId containerId,
@@ -155,4 +160,9 @@ public class DockerProviderService extends AbstractService
}
return false;
}
+
+ @Override
+ public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) {
+ this.serviceTimelinePublisher = publisher;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 1f379ea..ffa07fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -661,11 +661,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
asyncRMClient.registerTimelineV2Client(timelineClient);
timelineClient.init(getConfig());
timelineClient.start();
- log.info("Timeline client started.");
+ log.info("Timeline v2 client started.");
serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
serviceTimelinePublisher.init(getConfig());
serviceTimelinePublisher.start();
+
+ providerService.setServiceTimelinePublisher(serviceTimelinePublisher);
appState.setServiceTimelinePublisher(serviceTimelinePublisher);
log.info("ServiceTimelinePublisher started.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
index 0e04af5..7504140 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.TimelineV2Client;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.slider.api.resource.Application;
@@ -109,6 +108,13 @@ public class ServiceTimelinePublisher extends CompositeService {
publishComponents(application.getComponents());
}
+ public void serviceAttemptUpdated(Application application) {
+ TimelineEntity entity = createServiceAttemptEntity(application.getId());
+ entity.addInfo(SliderTimelineMetricsConstants.QUICK_LINKS,
+ application.getQuicklinks());
+ putEntity(entity);
+ }
+
public void serviceAttemptUnregistered(AppState appState,
ActionStopSlider stopAction) {
long currentTimeMillis = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
index 23e059d..58d77ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
@@ -37,6 +37,8 @@ public final class SliderTimelineMetricsConstants {
public static final String LAUNCH_TIME = "LAUNCH_TIME";
+ public static final String QUICK_LINKS = "QUICK_LINKS";
+
public static final String LAUNCH_COMMAND = "LAUNCH_COMMAND";
public static final String TOTAL_CONTAINERS = "NUMBER_OF_CONTAINERS";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
index 9f3ebba..e4fcbe8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
@@ -23,10 +23,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.ApplicationState;
@@ -77,6 +76,8 @@ public class TestServiceTimelinePublisher {
@Before
public void setUp() throws Exception {
config = new Configuration();
+ config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
timelineClient =
new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
@@ -88,8 +89,12 @@ public class TestServiceTimelinePublisher {
@After
public void tearDown() throws Exception {
- serviceTimelinePublisher.stop();
- timelineClient.stop();
+ if (serviceTimelinePublisher != null) {
+ serviceTimelinePublisher.stop();
+ }
+ if (timelineClient != null) {
+ timelineClient.stop();
+ }
}
@Test
@@ -264,6 +269,12 @@ public class TestServiceTimelinePublisher {
@Override
public void putEntitiesAsync(TimelineEntity... entities)
throws IOException, YarnException {
+ putEntities(entities);
+ }
+
+ @Override
+ public void putEntities(TimelineEntity... entities)
+ throws IOException, YarnException {
for (TimelineEntity timelineEntity : entities) {
TimelineEntity entity =
lastPublishedEntities.get(timelineEntity.getIdentifier());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org