You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/07 21:10:16 UTC
[16/76] [abbrv] hadoop git commit: YARN-5461. Initial code ported
from slider-core module. (jianhe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
new file mode 100644
index 0000000..67d3647
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.slideram;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.server.appmaster.PublishedArtifacts;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
+
+/**
+ * Exists just to move some functionality out of AppMaster into a peer class
+ * of the actual service provider doing the real work
+ */
+public class SliderAMProviderService extends AbstractProviderService implements
+ ProviderCore,
+ AgentKeys,
+ SliderKeys {
+
+ public SliderAMProviderService() {
+ super("SliderAMProviderService");
+ }
+
+ @Override
+ public String getHumanName() {
+ return "Slider Application";
+ }
+
+ @Override
+ public Configuration loadProviderConfigurationInformation(File confDir) throws
+ BadCommandArgumentsException,
+ IOException {
+ return null;
+ }
+
+ @Override
+ public void buildContainerLaunchContext(ContainerLauncher containerLauncher,
+ AggregateConf instanceDefinition,
+ Container container,
+ ProviderRole role,
+ SliderFileSystem sliderFileSystem,
+ Path generatedConfPath,
+ MapOperations resourceComponent,
+ MapOperations appComponent,
+ Path containerTmpDirPath) throws IOException, SliderException {
+ }
+
+ @Override
+ public List<ProviderRole> getRoles() {
+ return new ArrayList<>(0);
+ }
+
+ @Override
+ public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ SliderException {
+
+ }
+
+ @Override
+ public void applyInitialRegistryDefinitions(URL amWebURI,
+ URL agentOpsURI,
+ URL agentStatusURI,
+ ServiceRecord serviceRecord)
+ throws IOException {
+ super.applyInitialRegistryDefinitions(amWebURI,
+ agentOpsURI,
+ agentStatusURI,
+ serviceRecord);
+ // now publish site.xml files
+ YarnConfiguration defaultYarnConfig = new YarnConfiguration();
+ amState.getPublishedSliderConfigurations().put(
+ PublishedArtifacts.COMPLETE_CONFIG,
+ new PublishedConfiguration(
+ "Complete slider application settings",
+ getConfig(), getConfig()));
+ amState.getPublishedSliderConfigurations().put(
+ PublishedArtifacts.YARN_SITE_CONFIG,
+ new PublishedConfiguration(
+ "YARN site settings",
+ ConfigHelper.loadFromResource("yarn-site.xml"),
+ defaultYarnConfig) );
+
+ amState.getPublishedSliderConfigurations().put(
+ PublishedArtifacts.CORE_SITE_CONFIG,
+ new PublishedConfiguration(
+ "Core site settings",
+ ConfigHelper.loadFromResource("core-site.xml"),
+ defaultYarnConfig) );
+ amState.getPublishedSliderConfigurations().put(
+ PublishedArtifacts.HDFS_SITE_CONFIG,
+ new PublishedConfiguration(
+ "HDFS site settings",
+ ConfigHelper.loadFromResource("hdfs-site.xml"),
+ new HdfsConfiguration(true)) );
+
+
+ try {
+
+ URL managementAPI = new URL(amWebURI, RELATIVE_PATH_MANAGEMENT);
+ URL registryREST = new URL(amWebURI, RELATIVE_PATH_REGISTRY);
+
+ URL publisherURL = new URL(amWebURI, RELATIVE_PATH_PUBLISHER);
+
+ // Set the configurations URL.
+
+ String configurationsURL = SliderUtils.appendToURL(
+ publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET);
+ String exportsURL = SliderUtils.appendToURL(
+ publisherURL.toExternalForm(), RestPaths.SLIDER_EXPORTS);
+
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.webEndpoint(
+ CustomRegistryConstants.WEB_UI, amWebURI.toURI()));
+
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.webEndpoint(
+ CustomRegistryConstants.AM_REST_BASE, amWebURI.toURI()));
+
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.MANAGEMENT_REST_API,
+ managementAPI.toURI()));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.PUBLISHER_REST_API,
+ publisherURL.toURI()));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.REGISTRY_REST_API,
+ registryREST.toURI()));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API,
+ new URI(configurationsURL)));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.PUBLISHER_EXPORTS_API,
+ new URI(exportsURL)));
+
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
new file mode 100644
index 0000000..288f25a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
+
+/**
+ * Interface of AM operations
+ */
+public interface AppMasterActionOperations extends RMOperationHandlerActions {
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
new file mode 100644
index 0000000..65b88cf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+
+/**
+ * Implement privileged connection to the CM
+ *
+ */
+public class PrivilegedConnectToCM implements PrivilegedAction<ContainerManagementProtocol> {
+ final SliderAppMaster appMaster;
+ final InetSocketAddress cmAddress;
+
+ public PrivilegedConnectToCM(SliderAppMaster appMaster,
+ InetSocketAddress cmAddress) {
+ this.appMaster = appMaster;
+ this.cmAddress = cmAddress;
+ }
+
+
+ @Override //PrivilegedAction
+ public ContainerManagementProtocol run() {
+ return ((ContainerManagementProtocol) appMaster.getProxy(
+ ContainerManagementProtocol.class,
+ cmAddress));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
new file mode 100644
index 0000000..5d52441
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.slider.server.appmaster.state.AbstractClusterServices;
+
+public class ProtobufClusterServices extends AbstractClusterServices {
+
+ public Resource newResource() {
+ return Records.newRecord(Resource.class);
+ }
+
+ @Override
+ public Resource newResource(int memory, int cores) {
+ return Resources.createResource(memory, cores);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
new file mode 100644
index 0000000..fdc386f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+/**
+ * This is the name of YARN artifacts that are published
+ */
+public interface PublishedArtifacts {
+
+ String COMPLETE_CONFIG = "complete-config";
+ String CORE_SITE_CONFIG = "core-site";
+ String HDFS_SITE_CONFIG = "hdfs-site";
+ String YARN_SITE_CONFIG = "yarn-site";
+ String LOG4J = "log4j";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
new file mode 100644
index 0000000..3cfe167
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.server.appmaster.actions.ActionStartContainer;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.state.ContainerAssignment;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.services.workflow.WorkflowExecutorService;
+import org.apache.slider.server.services.workflow.ServiceThreadFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A service for launching containers
+ */
+public class RoleLaunchService
+ extends WorkflowExecutorService<ExecutorService> {
+ protected static final Logger log =
+ LoggerFactory.getLogger(RoleLaunchService.class);
+
+ public static final String ROLE_LAUNCH_SERVICE = "RoleLaunchService";
+
+
+ /**
+ * Queue submission API
+ */
+ private final QueueAccess actionQueue;
+
+ /**
+ * Provider building up the command
+ */
+ private final ProviderService provider;
+
+ /**
+ * Filesystem to use for the launch
+ */
+ private final SliderFileSystem fs;
+
+ /**
+ * Path in the launch filesystem that refers to a configuration directory
+ * -the interpretation of it is left to the Provider
+ */
+ private final Path generatedConfDirPath;
+ /**
+ * Path in the launch filesystem that refers to a temp directory
+ * which will be cleaned up at (some) time in the future
+ */
+ private final Path launcherTmpDirPath;
+
+ private Map<String, String> envVars;
+
+ /**
+ * Construct an instance of the launcher
+ * @param queueAccess
+ * @param provider the provider
+ * @param fs filesystem
+ * @param generatedConfDirPath path in the FS for the generated dir
+ * @param envVars environment variables
+ * @param launcherTmpDirPath path for a temporary data in the launch process
+ */
+ public RoleLaunchService(QueueAccess queueAccess,
+ ProviderService provider,
+ SliderFileSystem fs,
+ Path generatedConfDirPath,
+ Map<String, String> envVars,
+ Path launcherTmpDirPath) {
+ super(ROLE_LAUNCH_SERVICE);
+ this.actionQueue = queueAccess;
+ this.fs = fs;
+ this.generatedConfDirPath = generatedConfDirPath;
+ this.launcherTmpDirPath = launcherTmpDirPath;
+ this.provider = provider;
+ this.envVars = envVars;
+ }
+
+ @Override
+ public void init(Configuration conf) {
+ super.init(conf);
+ setExecutor(Executors.newCachedThreadPool(
+ new ServiceThreadFactory(ROLE_LAUNCH_SERVICE, true)));
+ }
+
+ /**
+ * Start an asychronous launch operation
+ * @param assignment container assignment
+ * @param clusterSpec cluster spec to use for template
+ * @param credentials credentials to use
+ */
+ public void launchRole(ContainerAssignment assignment,
+ AggregateConf clusterSpec,
+ Credentials credentials) {
+ RoleStatus role = assignment.role;
+ String roleName = role.getName();
+ String roleGroup = role.getGroup();
+ // prelaunch safety check
+ Preconditions.checkArgument(provider.isSupportedRole(roleName));
+ RoleLaunchService.RoleLauncher launcher =
+ new RoleLaunchService.RoleLauncher(assignment,
+ clusterSpec,
+ clusterSpec.getResourceOperations().getOrAddComponent(roleGroup),
+ clusterSpec.getAppConfOperations().getOrAddComponent(roleGroup),
+ credentials);
+ execute(launcher);
+ }
+
+ /**
+ * Thread that runs on the AM to launch a container
+ */
+ private class RoleLauncher implements Runnable {
+
+ private final ContainerAssignment assignment;
+ // Allocated container
+ public final Container container;
+ private final MapOperations resourceComponent;
+ private final MapOperations appComponent;
+ private final AggregateConf instanceDefinition;
+ public final ProviderRole role;
+ private final Credentials credentials;
+ private Exception raisedException;
+
+ public RoleLauncher(ContainerAssignment assignment,
+ AggregateConf instanceDefinition,
+ MapOperations resourceComponent,
+ MapOperations appComponent,
+ Credentials credentials) {
+ this.assignment = assignment;
+ this.credentials = credentials;
+ this.container = assignment.container;
+ RoleStatus roleStatus = assignment.role;
+
+ assert resourceComponent != null;
+ assert appComponent != null;
+ ProviderRole providerRole = roleStatus.getProviderRole();
+ assert providerRole != null;
+ this.role = providerRole;
+ this.resourceComponent = resourceComponent;
+ this.appComponent = appComponent;
+ this.instanceDefinition = instanceDefinition;
+ }
+
+ public Exception getRaisedException() {
+ return raisedException;
+ }
+
+ @Override
+ public String toString() {
+ return "RoleLauncher{" +
+ "container=" + container.getId() +
+ ", containerRole='" + role.name + '\'' +
+ ", containerGroup='" + role.group + '\'' +
+ '}';
+ }
+
+ @Override
+ public void run() {
+ try {
+ ContainerLauncher containerLauncher =
+ new ContainerLauncher(getConfig(), fs, container, credentials);
+ containerLauncher.setupUGI();
+ containerLauncher.putEnv(envVars);
+
+ log.debug("Launching container {} into role {}",
+ container.getId(),
+ role.name);
+
+ //now build up the configuration data
+ Path containerTmpDirPath =
+ new Path(launcherTmpDirPath, container.getId().toString());
+ provider.buildContainerLaunchContext(containerLauncher,
+ instanceDefinition,
+ container,
+ role,
+ fs,
+ generatedConfDirPath,
+ resourceComponent,
+ appComponent,
+ containerTmpDirPath);
+
+ RoleInstance instance = new RoleInstance(container);
+ String[] envDescription = containerLauncher.dumpEnvToString();
+
+ String commandsAsString = containerLauncher.getCommandsAsString();
+ log.info("Starting container with command: {}",
+ commandsAsString);
+
+ instance.command = commandsAsString;
+ instance.role = role.name;
+ instance.group = role.group;
+ instance.roleId = role.id;
+ instance.appVersion = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(SliderKeys.APP_VERSION);
+ instance.environment = envDescription;
+ int delay = appComponent.getOptionInt(
+ AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0);
+ int maxDelay =
+ getConfig().getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);
+ if (delay > maxDelay/1000) {
+ log.warn("Container launch delay of {} exceeds the maximum allowed of"
+ + " {} seconds. Delay will not be utilized.",
+ delay, maxDelay/1000);
+ delay = 0;
+ }
+ log.info("Container launch delay for {} set to {} seconds",
+ role.name, delay);
+ actionQueue.schedule(new ActionStartContainer("starting " + role.name,
+ container,
+ containerLauncher.completeContainerLaunch(),
+ instance,
+ delay,
+ TimeUnit.SECONDS));
+ } catch (Exception e) {
+ log.error("Exception thrown while trying to start {}: {}",
+ role.name, e, e);
+ raisedException = e;
+ }
+ }
+
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org