You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2017/08/14 21:13:45 UTC
[07/15] hadoop git commit: YARN-6903. Yarn-native-service framework
core rewrite. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java
deleted file mode 100644
index 3e020cf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.tarball;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.util.RestApiErrorMessages;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-
-public class TarballClientProvider extends AbstractClientProvider
- implements SliderKeys {
-
- public TarballClientProvider() {
- }
-
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fs)
- throws IOException {
- if (artifact == null) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
- }
- if (StringUtils.isEmpty(artifact.getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- Path p = new Path(artifact.getId());
- if (!fs.exists(p)) {
- throw new IllegalArgumentException( "Artifact tarball does not exist "
- + artifact.getId());
- }
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException {
- // validate dest_file is not absolute
- if (Paths.get(configFile.getDestFile()).isAbsolute()) {
- throw new IllegalArgumentException(
- "Dest_file must not be absolute path: " + configFile.getDestFile());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java
deleted file mode 100644
index d9b7450..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.tarball;
-
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.SliderProviderFactory;
-
-public class TarballProviderFactory extends SliderProviderFactory {
- private static final SliderProviderFactory FACTORY = new
- TarballProviderFactory();
-
- private TarballProviderFactory() {
- }
-
- private static class Client {
- static final AbstractClientProvider PROVIDER = new TarballClientProvider();
- }
-
- private static class Server {
- static final ProviderService PROVIDER = new TarballProviderService();
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return Client.PROVIDER;
- }
-
- @Override
- public ProviderService createServerProvider() {
- return Server.PROVIDER;
- }
-
- public static SliderProviderFactory getInstance() {
- return FACTORY;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java
deleted file mode 100644
index 8be63fd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.tarball;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-
-import java.io.IOException;
-
-public class TarballProviderService extends AbstractProviderService {
-
- protected TarballProviderService() {
- super(TarballProviderService.class.getSimpleName());
- }
-
- @Override
- public void processArtifact(ContainerLauncher launcher, Application
- application, RoleInstance roleInstance, SliderFileSystem fileSystem)
- throws IOException {
- Path artifact = new Path(roleInstance.providerRole.component
- .getArtifact().getId());
- if (!fileSystem.isFile(artifact)) {
- throw new IOException("Package doesn't exist as a resource: " +
- artifact.toString());
- }
- log.info("Adding resource {}", artifact.toString());
- LocalResourceType type = LocalResourceType.ARCHIVE;
- LocalResource packageResource = fileSystem.createAmResource(
- artifact, type);
- launcher.addLocalResource(APP_LIB_DIR, packageResource);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index f1b07f0..a03d4ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -23,27 +23,25 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.SliderProviderFactory;
-import org.apache.slider.server.appmaster.actions.ActionStartContainer;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import static org.apache.slider.common.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY;
+import static org.apache.hadoop.yarn.service.conf.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY;
/**
* A service for launching containers
@@ -59,15 +57,15 @@ public class RoleLaunchService
/**
* Queue submission API
*/
- private final QueueAccess actionQueue;
+ private QueueAccess actionQueue;
/**
* Filesystem to use for the launch
*/
- private final SliderFileSystem fs;
+ private SliderFileSystem fs;
- private Map<String, String> envVars;
+ private Map<String, String> envVars = new HashMap<>();
/**
* Construct an instance of the launcher
@@ -83,6 +81,11 @@ public class RoleLaunchService
this.envVars = envVars;
}
+ public RoleLaunchService(SliderFileSystem fs) {
+ super(ROLE_LAUNCH_SERVICE);
+ this.fs = fs;
+ }
+
@Override
public void init(Configuration conf) {
super.init(conf);
@@ -97,9 +100,13 @@ public class RoleLaunchService
*/
public void launchRole(ContainerAssignment assignment,
Application application, Credentials credentials) {
+ }
+
+ public void launchComponent(Application application,
+ ComponentInstance instance, Container container) {
RoleLaunchService.RoleLauncher launcher =
- new RoleLaunchService.RoleLauncher(assignment, application,
- credentials);
+ new RoleLaunchService.RoleLauncher(application, instance,
+ container);
execute(launcher);
}
@@ -107,67 +114,34 @@ public class RoleLaunchService
* Thread that runs on the AM to launch a container
*/
private class RoleLauncher implements Runnable {
-
- private final ContainerAssignment assignment;
// Allocated container
public final Container container;
public final Application application;
- public final ProviderRole role;
- private final Credentials credentials;
+ public ComponentInstance instance;
- public RoleLauncher(ContainerAssignment assignment,
+ public RoleLauncher(
Application application,
- Credentials credentials) {
- this.assignment = assignment;
- this.credentials = credentials;
- this.container = assignment.container;
- RoleStatus roleStatus = assignment.role;
- ProviderRole providerRole = roleStatus.getProviderRole();
- this.role = providerRole;
+ ComponentInstance instance, Container container) {
+ this.container = container;
this.application = application;
-
- }
-
- @Override
- public String toString() {
- return "RoleLauncher{" +
- "container=" + container.getId() +
- ", containerRole='" + role.name + '\'' +
- '}';
+ this.instance = instance;
}
@Override
public void run() {
try {
ContainerLauncher containerLauncher =
- new ContainerLauncher(getConfig(), fs, container, credentials);
- containerLauncher.setupUGI();
+ new ContainerLauncher(null, fs, container, null);
containerLauncher.putEnv(envVars);
- RoleInstance failedInstance = role.failedInstances.poll();
- RoleInstance instance;
- if (failedInstance != null) {
- instance = new RoleInstance(container, failedInstance);
- } else {
- instance = new RoleInstance(container, role);
- }
- String[] envDescription = containerLauncher.dumpEnvToString();
- String commandsAsString = containerLauncher.getCommandsAsString();
- log.info("Launching container {} for component instance = {}",
- container.getId(), instance.getCompInstanceName());
- log.info("Starting container with command: {}", commandsAsString);
- instance.command = commandsAsString;
- instance.role = role.name;
- instance.roleId = role.id;
- instance.environment = envDescription;
-
- ProviderService provider = SliderProviderFactory.getProviderService(
- role.component.getArtifact());
+ Component compSpec = instance.getCompSpec();
+ ProviderService provider = ProviderFactory.getProviderService(
+ compSpec.getArtifact());
provider.buildContainerLaunchContext(containerLauncher, application,
- container, role, fs, instance);
+ instance, fs);
- long delay = role.component.getConfiguration()
- .getPropertyLong(KEY_CONTAINER_LAUNCH_DELAY, 0);
+ long delay = compSpec.getConfiguration()
+ .getPropertyLong(KEY_CONTAINER_LAUNCH_DELAY, 0);
long maxDelay = getConfig()
.getLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);
@@ -177,14 +151,15 @@ public class RoleLaunchService
delay, maxDelay/1000);
delay = 0;
}
- log.info("Container launch delay for {} set to {} seconds", role.name,
- delay);
- actionQueue.schedule(
- new ActionStartContainer("starting " + role.name, container,
- containerLauncher.completeContainerLaunch(), instance, delay,
- TimeUnit.SECONDS));
+ if (delay > 0) {
+ Thread.sleep(delay * 1000);
+ }
+ instance.getComponent().getScheduler().getNmClient()
+ .startContainerAsync(container,
+ containerLauncher.completeContainerLaunch());
} catch (Exception e) {
- log.error("Exception thrown while trying to start " + role.name
+ log.error("Exception thrown while trying to start " + instance
+ .getCompInstanceName()
+ " container = " + container.getId() + " on host " + container
.getNodeId(), e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index e565849..06dde67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -76,6 +76,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.WebAppException;
import org.apache.hadoop.yarn.webapp.WebApps;
@@ -87,12 +89,12 @@ import org.apache.slider.api.proto.Messages;
import org.apache.slider.api.proto.SliderClusterAPI;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.SliderExitCodes;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.params.AbstractActionArgs;
-import org.apache.slider.common.params.SliderAMArgs;
-import org.apache.slider.common.params.SliderAMCreateAction;
-import org.apache.slider.common.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderAMCreateAction;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.PortScanner;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -109,8 +111,6 @@ import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.ProviderCompleted;
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.server.appmaster.actions.ActionHalt;
import org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance;
import org.apache.slider.server.appmaster.actions.ActionStopSlider;
@@ -142,8 +142,8 @@ import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
-import org.apache.slider.server.appmaster.timelineservice.SliderMetricsSink;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
import org.apache.slider.server.appmaster.web.SliderAMWebApp;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
@@ -157,7 +157,7 @@ import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.apache.slider.server.services.workflow.WorkflowRpcService;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
-import org.apache.slider.util.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -584,7 +584,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* @throws Throwable on a failure
*/
private int createAndRunCluster(String appName) throws Throwable {
- Path appDir = new Path((serviceArgs.getAppDefDir()));
+ Path appDir = new Path((serviceArgs.getAppDefPath()));
SliderFileSystem fs = getClusterFS();
fs.setAppDir(appDir);
application = ServiceApiUtil.loadApplication(fs, appName);
@@ -597,11 +597,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// initialize our providers
for (Component component : application.getComponents()) {
- SliderProviderFactory factory = SliderProviderFactory
+ ProviderFactory factory = ProviderFactory
.createSliderProviderFactory(component.getArtifact());
ProviderService providerService = factory.createServerProvider();
// init the provider BUT DO NOT START IT YET
- initAndAddService(providerService);
+// initAndAddService(providerService);
providers.add(providerService);
}
@@ -663,7 +663,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
serviceTimelinePublisher.start();
for (ProviderService providerService : providers) {
- providerService.setServiceTimelinePublisher(serviceTimelinePublisher);
}
appState.setServiceTimelinePublisher(serviceTimelinePublisher);
log.info("ServiceTimelinePublisher started.");
@@ -798,10 +797,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
liveContainers = amRegistrationData.getContainersFromPreviousAttempts();
DefaultMetricsSystem.initialize("SliderAppMaster");
if (timelineServiceEnabled) {
- DefaultMetricsSystem.instance().register("SliderMetricsSink",
+ DefaultMetricsSystem.instance().register("ServiceMetricsSink",
"For processing metrics to ATS",
- new SliderMetricsSink(serviceTimelinePublisher));
- log.info("SliderMetricsSink registered.");
+ new ServiceMetricsSink(serviceTimelinePublisher));
+ log.info("ServiceMetricsSink registered.");
}
//determine the location for the role history data
@@ -809,7 +808,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//build the instance
AppStateBindingInfo binding = new AppStateBindingInfo();
- binding.serviceConfig = serviceConf;
+ binding.serviceConfig = null;
binding.fs = fs.getFileSystem();
binding.historyPath = historyDir;
binding.liveContainers = liveContainers;
@@ -847,7 +846,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider access to the state, and AM
for (ProviderService providerService : providers) {
- providerService.setAMState(stateForProviders);
+// providerService.setAMState(stateForProviders);
}
// chaos monkey
@@ -1115,7 +1114,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
instanceName,
appAttemptID);
for (ProviderService providerService : providers) {
- providerService.bindToYarnRegistry(yarnRegistryOperations);
+// providerService.bindToYarnRegistry(yarnRegistryOperations);
}
// Yarn registry
@@ -1150,7 +1149,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
true);
}
if (timelineServiceEnabled) {
- serviceTimelinePublisher.serviceAttemptRegistered(appState);
+ serviceTimelinePublisher.serviceAttemptRegistered(application);
}
}
@@ -1185,15 +1184,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
new org.apache.slider.api.resource.Container();
container.setId(id.toString());
container.setLaunchTime(new Date());
- container.setState(org.apache.slider.api.resource.ContainerState.INIT);
+ container.setState(org.apache.slider.api.resource.ContainerState.RUNNING_BUT_UNREADY);
container.setBareHost(instance.host);
// TODO differentiate component name and component instance name ?
container.setComponentName(roleInstance.getCompInstanceName());
instance.providerRole.component.addContainer(container);
if (timelineServiceEnabled) {
- serviceTimelinePublisher.componentInstanceStarted(container,
- instance.providerRole.component.getName());
+ serviceTimelinePublisher.componentInstanceStarted(container, null);
}
return true;
}
@@ -1226,11 +1224,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
return;
}
String cid = RegistryPathUtils.encodeYarnID(containerId.toString());
- try {
- yarnRegistryOperations.deleteComponent(cid);
- } catch (IOException e) {
- log.warn("Failed to delete container {} : {}", containerId, e, e);
- }
+// try {
+// yarnRegistryOperations.deleteComponent(cid);
+// } catch (IOException e) {
+// log.warn("Failed to delete container {} : {}", containerId, e, e);
+// }
// remove component instance dir
try {
@@ -1879,7 +1877,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
throws IOException, SliderException {
// didn't start, so don't register
for (ProviderService providerService : providers) {
- providerService.start();
+// providerService.start();
}
// and send the started event ourselves
eventCallbackEvent(null);
@@ -1902,27 +1900,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
- /**
- * report container loss. If this isn't already known about, react
- *
- * @param containerId id of the container which has failed
- * @throws SliderException
- */
- public synchronized void providerLostContainer(
- ContainerId containerId)
- throws SliderException {
- log.info("containerLostContactWithProvider: container {} lost",
- containerId);
- RoleInstance activeContainer = appState.getOwnedContainer(containerId);
- if (activeContainer != null) {
- execute(appState.releaseContainer(containerId));
- // ask for more containers if needed
- log.info("Container released; triggering review");
- reviewRequestAndReleaseNodes("Loss of container");
- } else {
- log.info("Container not in active set - ignoring");
- }
- }
/**
* Async start container request
@@ -1985,30 +1962,29 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
LOG_YARN.error("Owned container not found for {}", containerId);
return;
}
- ProviderService providerService = SliderProviderFactory
+ ProviderService providerService = ProviderFactory
.getProviderService(cinfo.providerRole.component.getArtifact());
- if (providerService.processContainerStatus(containerId, containerStatus)) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
- LOG_YARN.info("Re-requesting status for role {}, {}",
- cinfo.role, containerId);
- //trigger another async container status
- nmClientAsync.getContainerStatusAsync(containerId,
- cinfo.container.getNodeId());
- } else if (timelineServiceEnabled) {
- RoleInstance instance = appState.getOwnedContainer(containerId);
- if (instance != null) {
- org.apache.slider.api.resource.Container container =
- instance.providerRole.component
- .getContainer(containerId.toString());
- if (container != null) {
- serviceTimelinePublisher.componentInstanceUpdated(container,
- instance.providerRole.component.getName());
- }
- }
- }
+// if (providerService.processContainerStatus(containerId, containerStatus)) {
+// try {
+// Thread.sleep(1000);
+// } catch (InterruptedException e) {
+// }
+// LOG_YARN.info("Re-requesting status for role {}, {}",
+// cinfo.role, containerId);
+// //trigger another async container status
+// nmClientAsync.getContainerStatusAsync(containerId,
+// cinfo.container.getNodeId());
+// } else if (timelineServiceEnabled) {
+// RoleInstance instance = appState.getOwnedContainer(containerId);
+// if (instance != null) {
+// org.apache.slider.api.resource.Container container =
+// instance.providerRole.component
+// .getContainer(containerId.toString());
+// if (container != null) {
+// serviceTimelinePublisher.componentInstanceUpdated(container);
+// }
+// }
+// }
}
@Override // NMClientAsync.CallbackHandler
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
deleted file mode 100644
index 41fe494..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Report container loss to the AM
- * {@link SliderAppMaster#providerLostContainer(ContainerId)}
- */
-public class ProviderReportedContainerLoss extends AsyncAction {
-
- private final ContainerId containerId;
-
- public ProviderReportedContainerLoss(ContainerId containerId) {
- this("lost container", 0, TimeUnit.MILLISECONDS, containerId);
- }
-
- public ProviderReportedContainerLoss(String name,
- long delay,
- TimeUnit timeUnit,
- ContainerId containerId) {
- super(name, delay, timeUnit);
- this.containerId = containerId;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- appMaster.providerLostContainer(containerId);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java
deleted file mode 100644
index cf607a0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.metrics;
-
-import com.codahale.metrics.Counter;
-import org.apache.hadoop.metrics2.MetricsCollector;
-import org.apache.hadoop.metrics2.MetricsInfo;
-import org.apache.hadoop.metrics2.MetricsSource;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.hadoop.metrics2.lib.Interns.info;
-
-@Metrics(context = "yarn-native-service")
-public class SliderMetrics implements MetricsSource {
-
- @Metric("containers requested")
- public MutableGaugeInt containersRequested;
-
- @Metric("anti-affinity containers pending")
- public MutableGaugeInt pendingAAContainers;
-
- @Metric("containers running")
- public MutableGaugeInt containersRunning;
-
- @Metric("containers desired")
- public MutableGaugeInt containersDesired;
-
- @Metric("containers completed")
- public MutableGaugeInt containersCompleted;
-
- @Metric("containers failed")
- public MutableGaugeInt containersFailed;
-
- @Metric("containers failed since last threshold")
- public MutableGaugeInt failedSinceLastThreshold;
-
- @Metric("containers preempted")
- public MutableGaugeInt containersPreempted;
-
- @Metric("containers exceeded limits")
- public MutableGaugeInt containersLimitsExceeded;
-
- @Metric("containers surplus")
- public MutableGaugeInt surplusContainers;
-
- @Metric("containers failed due to disk failure")
- public MutableGaugeInt containersDiskFailure;
-
- protected final MetricsRegistry registry;
-
- public SliderMetrics(MetricsInfo metricsInfo) {
- registry = new MetricsRegistry(metricsInfo);
- }
-
- @Override
- public void getMetrics(MetricsCollector collector, boolean all) {
- registry.snapshot(collector.addRecord(registry.info()), all);
- }
-
- public static SliderMetrics register(String name, String description) {
- SliderMetrics metrics = new SliderMetrics(info(name, description));
- DefaultMetricsSystem.instance().register(name, description, metrics);
- return metrics;
- }
-
- public void tag(String name, String description, String value) {
- registry.tag(name, description, value);
- }
-
- @Override public String toString() {
- return "SliderMetrics{"
- + "containersRequested=" + containersRequested.value()
- + ", pendingAAContainers=" + pendingAAContainers.value()
- + ", containersRunning=" + containersRunning.value()
- + ", containersDesired=" + containersDesired.value()
- + ", containersCompleted=" + containersCompleted.value()
- + ", containersFailed=" + containersFailed.value()
- + ", failedSinceLastThreshold=" + failedSinceLastThreshold.value()
- + ", containersPreempted=" + containersPreempted.value()
- + ", surplusContainers=" + surplusContainers.value() + '}';
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
index 1406fbe..022312c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
@@ -19,7 +19,7 @@
package org.apache.slider.server.appmaster.monkey;
import com.google.common.base.Preconditions;
-import org.apache.slider.common.SliderKeys;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
import org.apache.slider.server.appmaster.actions.ActionKillContainer;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.operations.RMOperationHandler;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
index dd4785d..2df9472 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
@@ -42,16 +42,13 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.common.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.ServiceNotReadyException;
import org.apache.slider.core.exceptions.SliderException;
-import static org.apache.slider.common.SliderXmlConfKeys.*;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java
index a40078a..37c0a70 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java
@@ -19,7 +19,7 @@ package org.apache.slider.server.appmaster.rpc;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.Service;
-import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
/**
* {@link PolicyProvider} for Slider protocols.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java
index 4fd4910..5b127b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.TokenInfo;
import org.apache.hadoop.security.token.TokenSelector;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector;
-import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
import java.lang.annotation.Annotation;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
index cc19eee..75eccd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import static org.apache.slider.core.main.LauncherExitCodes.EXIT_UNAUTHORIZED;
import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 52d67f8..ba923bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -53,8 +53,8 @@ import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.RoleStatistics;
-import org.apache.slider.common.SliderExitCodes;
-import org.apache.slider.common.SliderKeys;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.BadConfigException;
@@ -67,13 +67,13 @@ import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.management.MetricsConstants;
-import org.apache.slider.server.appmaster.metrics.SliderMetrics;
+import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
-import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
-import org.apache.slider.util.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -216,7 +216,7 @@ public class AppState {
private Resource minResource;
private Resource maxResource;
- private SliderMetrics appMetrics;
+ private ServiceMetrics appMetrics;
private ServiceTimelinePublisher serviceTimelinePublisher;
@@ -330,7 +330,7 @@ public class AppState {
// set the cluster specification (once its dependency the client properties
// is out the way
this.app = binding.application;
- appMetrics = SliderMetrics.register(app.getName(),
+ appMetrics = ServiceMetrics.register(app.getName(),
"Metrics for service");
appMetrics.tag("type", "Metrics type [component or service]", "service");
appMetrics.tag("appId", "Application id for service", app.getId());
@@ -1013,8 +1013,8 @@ public class AppState {
}
private void incCompletedContainers(RoleStatus role) {
- role.getComponentMetrics().containersCompleted.incr();
- appMetrics.containersCompleted.incr();
+ role.getComponentMetrics().containersSucceeded.incr();
+ appMetrics.containersSucceeded.incr();
}
@VisibleForTesting
@@ -1030,26 +1030,26 @@ public class AppState {
role.getComponentMetrics().containersDiskFailure.incr();
role.getComponentMetrics().containersFailed.incr();
break;
- case Failed:
- appMetrics.failedSinceLastThreshold.incr();
- appMetrics.containersFailed.incr();
- role.getComponentMetrics().failedSinceLastThreshold.incr();
- role.getComponentMetrics().containersFailed.incr();
- break;
- case Failed_limits_exceeded:
- appMetrics.containersLimitsExceeded.incr();
- appMetrics.failedSinceLastThreshold.incr();
- appMetrics.containersFailed.incr();
- role.getComponentMetrics().containersLimitsExceeded.incr();
- role.getComponentMetrics().failedSinceLastThreshold.incr();
- role.getComponentMetrics().containersFailed.incr();
- break;
- default:
- appMetrics.failedSinceLastThreshold.incr();
- appMetrics.containersFailed.incr();
- role.getComponentMetrics().failedSinceLastThreshold.incr();
- role.getComponentMetrics().containersFailed.incr();
- break;
+// case Failed:
+// appMetrics.failedSinceLastThreshold.incr();
+// appMetrics.containersFailed.incr();
+// role.getComponentMetrics().failedSinceLastThreshold.incr();
+// role.getComponentMetrics().containersFailed.incr();
+// break;
+// case Failed_limits_exceeded:
+// appMetrics.containersLimitsExceeded.incr();
+// appMetrics.failedSinceLastThreshold.incr();
+// appMetrics.containersFailed.incr();
+// role.getComponentMetrics().containersLimitsExceeded.incr();
+// role.getComponentMetrics().failedSinceLastThreshold.incr();
+// role.getComponentMetrics().containersFailed.incr();
+// break;
+// default:
+// appMetrics.failedSinceLastThreshold.incr();
+// appMetrics.containersFailed.incr();
+// role.getComponentMetrics().failedSinceLastThreshold.incr();
+// role.getComponentMetrics().containersFailed.incr();
+// break;
}
}
@@ -1308,7 +1308,7 @@ public class AppState {
log.info("decrementing role count for role {} to {}; completed={}",
roleStatus.getName(),
roleStatus.getComponentMetrics().containersRunning.value(),
- roleStatus.getComponentMetrics().containersCompleted.value());
+ roleStatus.getComponentMetrics().containersSucceeded.value());
result.outcome = ContainerOutcome.Completed;
roleHistory.onReleaseCompleted(container);
@@ -1671,7 +1671,7 @@ public class AppState {
if (instance.providerRole.probe == null) {
continue;
}
- boolean ready = instance.providerRole.probe.ping(instance).isSuccess();
+ boolean ready = instance.providerRole.probe.ping(null).isSuccess();
if (ready) {
if (instance.state != STATE_READY) {
instance.state = STATE_READY;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
index ac9b8eb..439a256 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
@@ -25,13 +25,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.CoreFileSystem;
import org.apache.slider.providers.ProviderRole;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
/**
* Binding information for application states; designed to be extensible
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index b6c3675..15333e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -31,7 +31,6 @@ import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.management.BoolMetric;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.management.Timestamp;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index 5619492..070f2f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.providers.ProviderRole;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 3d9a8f3..ec0ff25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -27,7 +27,7 @@ import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.management.BoolMetricPredicate;
-import org.apache.slider.server.appmaster.metrics.SliderMetrics;
+import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
import java.io.Serializable;
import java.util.Comparator;
@@ -54,7 +54,7 @@ public final class RoleStatus implements MetricSet {
/** resource requirements */
private Resource resourceRequirements;
- private SliderMetrics componentMetrics;
+ private ServiceMetrics componentMetrics;
/** any pending AA request */
private volatile OutstandingRequest outstandingAArequest = null;
@@ -67,12 +67,12 @@ public final class RoleStatus implements MetricSet {
this.name = providerRole.name;
this.key = providerRole.id;
componentMetrics =
- SliderMetrics.register(this.name, "Metrics for component " + this.name);
+ ServiceMetrics.register(this.name, "Metrics for component " + this.name);
componentMetrics
.tag("type", "Metrics type [component or service]", "component");
}
- public SliderMetrics getComponentMetrics() {
+ public ServiceMetrics getComponentMetrics() {
return this.componentMetrics;
}
@@ -235,14 +235,15 @@ public final class RoleStatus implements MetricSet {
}
public long getFailedRecently() {
- return componentMetrics.failedSinceLastThreshold.value();
+// return componentMetrics.failedSinceLastThreshold.value();
+ return 0;
}
public long resetFailedRecently() {
- long count =
- componentMetrics.failedSinceLastThreshold.value();
- componentMetrics.failedSinceLastThreshold.set(0);
- return count;
+// long count =
+// componentMetrics.failedSinceLastThreshold.value();
+// componentMetrics.failedSinceLastThreshold.set(0);
+ return 0;
}
public long getFailed() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
deleted file mode 100644
index 7504140..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.timelineservice;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.hadoop.metrics2.AbstractMetric;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.resource.Configuration;
-import org.apache.slider.api.resource.Container;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.appmaster.actions.ActionStopSlider;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A single service that publishes all the Timeline Entities.
- */
-public class ServiceTimelinePublisher extends CompositeService {
-
- // Number of bytes of config which can be published in one shot to ATSv2.
- public static final int ATS_CONFIG_PUBLISH_SIZE_BYTES = 10 * 1024;
-
- private TimelineV2Client timelineClient;
-
- private volatile boolean stopped = false;
-
- private static final Logger log =
- LoggerFactory.getLogger(ServiceTimelinePublisher.class);
-
- @Override
- protected void serviceStop() throws Exception {
- stopped = true;
- }
-
- public boolean isStopped() {
- return stopped;
- }
-
- public ServiceTimelinePublisher(TimelineV2Client client) {
- super(ServiceTimelinePublisher.class.getName());
- timelineClient = client;
- }
-
- public void serviceAttemptRegistered(AppState appState) {
- Application application = appState.getClusterStatus();
- long currentTimeMillis = application.getLaunchTime() == null
- ? System.currentTimeMillis() : application.getLaunchTime().getTime();
-
- TimelineEntity entity = createServiceAttemptEntity(application.getId());
- entity.setCreatedTime(currentTimeMillis);
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(SliderTimelineMetricsConstants.NAME, application.getName());
- entityInfos.put(SliderTimelineMetricsConstants.STATE,
- application.getState().toString());
- entityInfos.put(SliderTimelineMetricsConstants.LAUNCH_TIME,
- currentTimeMillis);
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent.setId(SliderTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString());
- startEvent.setTimestamp(currentTimeMillis);
- entity.addEvent(startEvent);
-
- // publish before configurations published
- putEntity(entity);
-
- // publish application specific configurations
- publishConfigurations(application.getConfiguration(), application.getId(),
- SliderTimelineEntityType.SERVICE_ATTEMPT.toString(), true);
-
- // publish component as separate entity.
- publishComponents(application.getComponents());
- }
-
- public void serviceAttemptUpdated(Application application) {
- TimelineEntity entity = createServiceAttemptEntity(application.getId());
- entity.addInfo(SliderTimelineMetricsConstants.QUICK_LINKS,
- application.getQuicklinks());
- putEntity(entity);
- }
-
- public void serviceAttemptUnregistered(AppState appState,
- ActionStopSlider stopAction) {
- long currentTimeMillis = System.currentTimeMillis();
-
- TimelineEntity entity =
- createServiceAttemptEntity(appState.getClusterStatus().getId());
-
- // add info
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(SliderTimelineMetricsConstants.EXIT_STATUS_CODE,
- stopAction.getExitCode());
- entityInfos.put(SliderTimelineMetricsConstants.STATE,
- stopAction.getFinalApplicationStatus().toString());
- if (stopAction.getMessage() != null) {
- entityInfos.put(SliderTimelineMetricsConstants.EXIT_REASON,
- stopAction.getMessage());
- }
- if (stopAction.getEx() != null) {
- entityInfos.put(SliderTimelineMetricsConstants.DIAGNOSTICS_INFO,
- stopAction.getEx().toString());
- }
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent
- .setId(SliderTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString());
- startEvent.setTimestamp(currentTimeMillis);
- entity.addEvent(startEvent);
-
- putEntity(entity);
- }
-
- public void componentInstanceStarted(Container container,
- String componentName) {
-
- TimelineEntity entity = createComponentInstanceEntity(container.getId());
- entity.setCreatedTime(container.getLaunchTime().getTime());
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(SliderTimelineMetricsConstants.BARE_HOST,
- container.getBareHost());
- entityInfos.put(SliderTimelineMetricsConstants.STATE,
- container.getState().toString());
- entityInfos.put(SliderTimelineMetricsConstants.LAUNCH_TIME,
- container.getLaunchTime().getTime());
- entityInfos.put(SliderTimelineMetricsConstants.COMPONENT_NAME,
- componentName);
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent
- .setId(SliderTimelineEvent.COMPONENT_INSTANCE_REGISTERED.toString());
- startEvent.setTimestamp(container.getLaunchTime().getTime());
- entity.addEvent(startEvent);
-
- putEntity(entity);
- }
-
- public void componentInstanceFinished(RoleInstance instance) {
- TimelineEntity entity = createComponentInstanceEntity(instance.id);
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(SliderTimelineMetricsConstants.EXIT_STATUS_CODE,
- instance.exitCode);
- entityInfos.put(SliderTimelineMetricsConstants.DIAGNOSTICS_INFO,
- instance.diagnostics);
- // TODO need to change the state based on enum value.
- entityInfos.put(SliderTimelineMetricsConstants.STATE, "FINISHED");
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent
- .setId(SliderTimelineEvent.COMPONENT_INSTANCE_UNREGISTERED.toString());
- startEvent.setTimestamp(System.currentTimeMillis());
- entity.addEvent(startEvent);
-
- putEntity(entity);
- }
-
- public void componentInstanceUpdated(Container container,
- String componentName) {
- TimelineEntity entity = createComponentInstanceEntity(container.getId());
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(SliderTimelineMetricsConstants.IP, container.getIp());
- entityInfos.put(SliderTimelineMetricsConstants.HOSTNAME,
- container.getHostname());
- entityInfos.put(SliderTimelineMetricsConstants.STATE,
- container.getState().toString());
- entity.addInfo(entityInfos);
-
- TimelineEvent updateEvent = new TimelineEvent();
- updateEvent
- .setId(SliderTimelineEvent.COMPONENT_INSTANCE_UPDATED.toString());
- updateEvent.setTimestamp(System.currentTimeMillis());
- entity.addEvent(updateEvent);
-
- putEntity(entity);
- }
-
- private void publishComponents(List<Component> components) {
- long currentTimeMillis = System.currentTimeMillis();
- for (Component component : components) {
- TimelineEntity entity = createComponentEntity(component.getName());
- entity.setCreatedTime(currentTimeMillis);
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(SliderTimelineMetricsConstants.ARTIFACT_ID,
- component.getArtifact().getId());
- entityInfos.put(SliderTimelineMetricsConstants.ARTIFACT_TYPE,
- component.getArtifact().getType().toString());
- if (component.getResource().getProfile() != null) {
- entityInfos.put(SliderTimelineMetricsConstants.RESOURCE_PROFILE,
- component.getResource().getProfile());
- }
- entityInfos.put(SliderTimelineMetricsConstants.RESOURCE_CPU,
- component.getResource().getCpus());
- entityInfos.put(SliderTimelineMetricsConstants.RESOURCE_MEMORY,
- component.getResource().getMemory());
-
- if (component.getLaunchCommand() != null) {
- entityInfos.put(SliderTimelineMetricsConstants.LAUNCH_COMMAND,
- component.getLaunchCommand());
- }
- entityInfos.put(SliderTimelineMetricsConstants.UNIQUE_COMPONENT_SUPPORT,
- component.getUniqueComponentSupport().toString());
- entityInfos.put(SliderTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER,
- component.getRunPrivilegedContainer().toString());
- if (component.getPlacementPolicy() != null) {
- entityInfos.put(SliderTimelineMetricsConstants.PLACEMENT_POLICY,
- component.getPlacementPolicy().getLabel());
- }
- entity.addInfo(entityInfos);
-
- putEntity(entity);
-
- // publish component specific configurations
- publishConfigurations(component.getConfiguration(), component.getName(),
- SliderTimelineEntityType.COMPONENT.toString(), false);
- }
- }
-
- private void publishConfigurations(Configuration configuration,
- String entityId, String entityType, boolean isServiceAttemptEntity) {
- if (isServiceAttemptEntity) {
- // publish slider-client.xml properties at service level
- publishConfigurations(SliderUtils.loadSliderClientXML().iterator(),
- entityId, entityType);
- }
- publishConfigurations(configuration.getProperties().entrySet().iterator(),
- entityId, entityType);
-
- publishConfigurations(configuration.getEnv().entrySet().iterator(),
- entityId, entityType);
-
- for (ConfigFile configFile : configuration.getFiles()) {
- publishConfigurations(configFile.getProps().entrySet().iterator(),
- entityId, entityType);
- }
- }
-
- private void publishConfigurations(Iterator<Entry<String, String>> iterator,
- String entityId, String entityType) {
- int configSize = 0;
- TimelineEntity entity = createTimelineEntity(entityId, entityType);
- while (iterator.hasNext()) {
- Entry<String, String> entry = iterator.next();
- int size = entry.getKey().length() + entry.getValue().length();
- configSize += size;
- // Configs are split into multiple entities if they exceed 100kb in size.
- if (configSize > ATS_CONFIG_PUBLISH_SIZE_BYTES) {
- if (entity.getConfigs().size() > 0) {
- putEntity(entity);
- entity = createTimelineEntity(entityId, entityType);
- }
- configSize = size;
- }
- entity.addConfig(entry.getKey(), entry.getValue());
- }
- if (configSize > 0) {
- putEntity(entity);
- }
- }
-
- /**
- * Called from SliderMetricsSink at regular interval of time.
- * @param metrics of service or components
- * @param entityId Id of entity
- * @param entityType Type of entity
- * @param timestamp
- */
- public void publishMetrics(Iterable<AbstractMetric> metrics, String entityId,
- String entityType, long timestamp) {
- TimelineEntity entity = createTimelineEntity(entityId, entityType);
- Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
- for (AbstractMetric metric : metrics) {
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setId(metric.name());
- timelineMetric.addValue(timestamp, metric.value());
- entityMetrics.add(timelineMetric);
- }
- entity.setMetrics(entityMetrics);
- putEntity(entity);
- }
-
- private TimelineEntity createServiceAttemptEntity(String serviceId) {
- TimelineEntity entity = createTimelineEntity(serviceId,
- SliderTimelineEntityType.SERVICE_ATTEMPT.toString());
- return entity;
- }
-
- private TimelineEntity createComponentInstanceEntity(String instanceId) {
- TimelineEntity entity = createTimelineEntity(instanceId,
- SliderTimelineEntityType.COMPONENT_INSTANCE.toString());
- return entity;
- }
-
- private TimelineEntity createComponentEntity(String componentId) {
- TimelineEntity entity = createTimelineEntity(componentId,
- SliderTimelineEntityType.COMPONENT.toString());
- return entity;
- }
-
- private TimelineEntity createTimelineEntity(String entityId,
- String entityType) {
- TimelineEntity entity = new TimelineEntity();
- entity.setId(entityId);
- entity.setType(entityType);
- return entity;
- }
-
- private void putEntity(TimelineEntity entity) {
- try {
- if (log.isDebugEnabled()) {
- log.debug("Publishing the entity " + entity + ", JSON-style content: "
- + TimelineUtils.dumpTimelineRecordtoJSON(entity));
- }
- if (timelineClient != null) {
- timelineClient.putEntitiesAsync(entity);
- } else {
- log.error("Seems like client has been removed before the entity "
- + "could be published for " + entity);
- }
- } catch (Exception e) {
- log.error("Error when publishing entity " + entity, e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java
deleted file mode 100644
index 869ae26..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.timelineservice;
-
-import org.apache.commons.configuration2.SubsetConfiguration;
-import org.apache.hadoop.metrics2.MetricsRecord;
-import org.apache.hadoop.metrics2.MetricsSink;
-import org.apache.hadoop.metrics2.MetricsTag;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Write the metrics to a ATSv2. Generally, this class is instantiated via
- * hadoop-metrics2 property files. Specifically, you would create this class by
- * adding the following to by This would actually be set as: <code>
- * [prefix].sink.[some instance name].class
- * =org.apache.slider.server.appmaster.timelineservice.SliderMetricsSink
- * </code>, where <tt>prefix</tt> is "atsv2": and <tt>some instance name</tt> is
- * just any unique name, so properties can be differentiated if there are
- * multiple sinks of the same type created
- */
-public class SliderMetricsSink implements MetricsSink {
-
- private static final Logger log =
- LoggerFactory.getLogger(SliderMetricsSink.class);
-
- private ServiceTimelinePublisher serviceTimelinePublisher;
-
- public SliderMetricsSink() {
-
- }
-
- public SliderMetricsSink(ServiceTimelinePublisher publisher) {
- serviceTimelinePublisher = publisher;
- }
-
- /**
- * Publishes service and component metrics to ATS.
- */
- @Override
- public void putMetrics(MetricsRecord record) {
- if (serviceTimelinePublisher.isStopped()) {
- log.warn("ServiceTimelinePublisher has stopped. "
- + "Not publishing any more metrics to ATS.");
- return;
- }
-
- boolean isServiceMetrics = false;
- boolean isComponentMetrics = false;
- String appId = null;
- for (MetricsTag tag : record.tags()) {
- if (tag.name().equals("type") && tag.value().equals("service")) {
- isServiceMetrics = true;
- } else if (tag.name().equals("type") && tag.value().equals("component")) {
- isComponentMetrics = true;
- break; // if component metrics, no more information required from tag so
- // break the loop
- } else if (tag.name().equals("appId")) {
- appId = tag.value();
- }
- }
-
- if (isServiceMetrics && appId != null) {
- if (log.isDebugEnabled()) {
- log.debug("Publishing service metrics. " + record);
- }
- serviceTimelinePublisher.publishMetrics(record.metrics(), appId,
- SliderTimelineEntityType.SERVICE_ATTEMPT.toString(),
- record.timestamp());
- } else if (isComponentMetrics) {
- if (log.isDebugEnabled()) {
- log.debug("Publishing Component metrics. " + record);
- }
- serviceTimelinePublisher.publishMetrics(record.metrics(), record.name(),
- SliderTimelineEntityType.COMPONENT.toString(), record.timestamp());
- }
- }
-
- @Override
- public void init(SubsetConfiguration conf) {
- }
-
- @Override
- public void flush() {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java
deleted file mode 100644
index 908754f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.timelineservice;
-
-/**
- * Slider entities that are published to ATS.
- */
-public enum SliderTimelineEntityType {
- /**
- * Used for publishing service entity information.
- */
- SERVICE_ATTEMPT,
-
- /**
- * Used for publishing component entity information.
- */
- COMPONENT,
-
- /**
- * Used for publishing component instance entity information.
- */
- COMPONENT_INSTANCE
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java
deleted file mode 100644
index 04f0219..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.timelineservice;
-
-/**
- * Events that are used to store in ATS.
- */
-public enum SliderTimelineEvent {
- SERVICE_ATTEMPT_REGISTERED,
-
- SERVICE_ATTEMPT_UNREGISTERED,
-
- COMPONENT_INSTANCE_REGISTERED,
-
- COMPONENT_INSTANCE_UNREGISTERED,
-
- COMPONENT_INSTANCE_UPDATED
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
deleted file mode 100644
index 58d77ce..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.timelineservice;
-
-/**
- * Constants which are stored as key in ATS
- */
-public final class SliderTimelineMetricsConstants {
-
- public static final String URI = "URI";
-
- public static final String NAME = "NAME";
-
- public static final String STATE = "STATE";
-
- public static final String EXIT_STATUS_CODE = "EXIT_STATUS_CODE";
-
- public static final String EXIT_REASON = "EXIT_REASON";
-
- public static final String DIAGNOSTICS_INFO = "DIAGNOSTICS_INFO";
-
- public static final String LAUNCH_TIME = "LAUNCH_TIME";
-
- public static final String QUICK_LINKS = "QUICK_LINKS";
-
- public static final String LAUNCH_COMMAND = "LAUNCH_COMMAND";
-
- public static final String TOTAL_CONTAINERS = "NUMBER_OF_CONTAINERS";
-
- public static final String RUNNING_CONTAINERS =
- "NUMBER_OF_RUNNING_CONTAINERS";
-
- /**
- * Artifacts constants.
- */
- public static final String ARTIFACT_ID = "ARTIFACT_ID";
-
- public static final String ARTIFACT_TYPE = "ARTIFACT_TYPE";
-
- public static final String ARTIFACT_URI = "ARTIFACT_URI";
-
- /**
- * Resource constants.
- */
- public static final String RESOURCE_CPU = "RESOURCE_CPU";
-
- public static final String RESOURCE_MEMORY = "RESOURCE_MEMORY";
-
- public static final String RESOURCE_PROFILE = "RESOURCE_PROFILE";
-
- /**
- * component instance constants.
- */
- public static final String IP = "IP";
-
- public static final String HOSTNAME = "HOSTNAME";
-
- public static final String BARE_HOST = "BARE_HOST";
-
- public static final String COMPONENT_NAME = "COMPONENT_NAME";
-
- /**
- * component constants.
- */
- public static final String DEPENDENCIES = "DEPENDENCIES";
-
- public static final String DESCRIPTION = "DESCRIPTION";
-
- public static final String UNIQUE_COMPONENT_SUPPORT =
- "UNIQUE_COMPONENT_SUPPORT";
-
- public static final String RUN_PRIVILEGED_CONTAINER =
- "RUN_PRIVILEGED_CONTAINER";
-
- public static final String PLACEMENT_POLICY = "PLACEMENT_POLICY";
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org