You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2018/08/14 15:37:22 UTC
hadoop git commit: YARN-8160. Support upgrade of service that use
docker containers. Contributed by Chandni Singh
Repository: hadoop
Updated Branches:
refs/heads/trunk d1d129aa9 -> 4cba0741d
YARN-8160. Support upgrade of service that use docker containers.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cba0741
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cba0741
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cba0741
Branch: refs/heads/trunk
Commit: 4cba0741d58e0aeacde5b6541c9ef40a9f0e40c7
Parents: d1d129a
Author: Eric Yang <ey...@apache.org>
Authored: Tue Aug 14 11:36:26 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Tue Aug 14 11:36:26 2018 -0400
----------------------------------------------------------------------
.../yarn/service/component/Component.java | 3 +-
.../component/instance/ComponentInstance.java | 2 +-
.../containerlaunch/AbstractLauncher.java | 5 +
.../containerlaunch/ContainerLaunchService.java | 11 ++
.../provider/AbstractProviderService.java | 5 +-
.../defaultImpl/DefaultProviderService.java | 4 +-
.../provider/docker/DockerProviderService.java | 30 ++--
.../tarball/TarballProviderService.java | 7 +-
.../instance/TestComponentInstance.java | 4 +-
.../provider/TestAbstractProviderService.java | 138 +++++++++++++++++++
.../launcher/ContainerLaunch.java | 44 ++++--
11 files changed, 216 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index aaa23da..41a2fcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -635,7 +635,8 @@ public class Component implements EventHandler<ComponentEvent> {
version);
launchContext.setArtifact(compSpec.getArtifact())
.setConfiguration(compSpec.getConfiguration())
- .setLaunchCommand(compSpec.getLaunchCommand());
+ .setLaunchCommand(compSpec.getLaunchCommand())
+ .setRunPrivilegedContainer(compSpec.getRunPrivilegedContainer());
return launchContext;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 10128a0..11a6caa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -509,7 +509,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
return this.container.getNodeId();
}
- public org.apache.hadoop.yarn.service.api.records.Component getCompSpec() {
+ private org.apache.hadoop.yarn.service.api.records.Component getCompSpec() {
return component.getComponentSpec();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
index da5a8d6..c34cc77 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service.containerlaunch;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -254,4 +255,8 @@ public class AbstractLauncher {
this.runPrivilegedContainer = runPrivilegedContainer;
}
+ @VisibleForTesting
+ public String getDockerImage() {
+ return dockerImage;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
index f674e0d..3c856ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -139,6 +139,7 @@ public class ContainerLaunchService extends AbstractService{
private org.apache.hadoop.yarn.service.api.records.Configuration
configuration;
private String launchCommand;
+ private boolean runPrivilegedContainer;
public ComponentLaunchContext(String name, String serviceVersion) {
this.name = Preconditions.checkNotNull(name);
@@ -166,6 +167,10 @@ public class ContainerLaunchService extends AbstractService{
return launchCommand;
}
+ public boolean isRunPrivilegedContainer() {
+ return runPrivilegedContainer;
+ }
+
public ComponentLaunchContext setArtifact(Artifact artifact) {
this.artifact = artifact;
return this;
@@ -181,5 +186,11 @@ public class ContainerLaunchService extends AbstractService{
this.launchCommand = launchCommand;
return this;
}
+
+ public ComponentLaunchContext setRunPrivilegedContainer(
+ boolean runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ return this;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
index 3dfdadc..4394e62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -56,7 +56,8 @@ public abstract class AbstractProviderService implements ProviderService,
public abstract void processArtifact(AbstractLauncher launcher,
ComponentInstance compInstance, SliderFileSystem fileSystem,
- Service service)
+ Service service,
+ ContainerLaunchService.ComponentLaunchContext compLaunchCtx)
throws IOException;
public Map<String, String> buildContainerTokens(ComponentInstance instance,
@@ -140,7 +141,7 @@ public abstract class AbstractProviderService implements ProviderService,
SliderFileSystem fileSystem, Configuration yarnConf, Container container,
ContainerLaunchService.ComponentLaunchContext compLaunchContext)
throws IOException, SliderException {
- processArtifact(launcher, instance, fileSystem, service);
+ processArtifact(launcher, instance, fileSystem, service, compLaunchContext);
ServiceContext context =
instance.getComponent().getScheduler().getContext();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
index a3a0c1f..790fe20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.service.provider.defaultImpl;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
@@ -30,7 +31,8 @@ public class DefaultProviderService extends AbstractProviderService {
@Override
public void processArtifact(AbstractLauncher launcher,
ComponentInstance compInstance, SliderFileSystem fileSystem,
- Service service)
+ Service service,
+ ContainerLaunchService.ComponentLaunchContext compLaunchCtx)
throws IOException {
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
index 7844621..6027a66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
@@ -38,34 +37,38 @@ import java.util.Map;
public class DockerProviderService extends AbstractProviderService
implements DockerKeys {
+ @Override
public void processArtifact(AbstractLauncher launcher,
ComponentInstance compInstance, SliderFileSystem fileSystem,
- Service service) throws IOException{
+ Service service, ContainerLaunchService.ComponentLaunchContext
+ compLaunchCtx) throws IOException{
launcher.setYarnDockerMode(true);
- launcher.setDockerImage(compInstance.getCompSpec().getArtifact().getId());
- launcher.setDockerNetwork(compInstance.getCompSpec().getConfiguration()
+ launcher.setDockerImage(compLaunchCtx.getArtifact().getId());
+ launcher.setDockerNetwork(compLaunchCtx.getConfiguration()
.getProperty(DOCKER_NETWORK));
launcher.setDockerHostname(compInstance.getHostname());
launcher.setRunPrivilegedContainer(
- compInstance.getCompSpec().getRunPrivilegedContainer());
+ compLaunchCtx.isRunPrivilegedContainer());
}
/**
* Check if system is default to disable docker override or
* user requested a Docker container with ENTRY_POINT support.
*
- * @param component - YARN Service component
+ * @param compLaunchContext - launch context for the component.
* @return true if Docker launch command override is disabled
*/
- private boolean checkUseEntryPoint(Component component) {
+ private boolean checkUseEntryPoint(
+ ContainerLaunchService.ComponentLaunchContext compLaunchContext) {
boolean overrideDisable = false;
String overrideDisableKey = Environment.
YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE.
name();
- String overrideDisableValue = (component
- .getConfiguration().getEnv(overrideDisableKey) != null) ?
- component.getConfiguration().getEnv(overrideDisableKey) :
- System.getenv(overrideDisableKey);
+ String overrideDisableValue = (
+ compLaunchContext.getConfiguration().getEnv(overrideDisableKey)
+ != null) ?
+ compLaunchContext.getConfiguration().getEnv(
+ overrideDisableKey) : System.getenv(overrideDisableKey);
overrideDisable = Boolean.parseBoolean(overrideDisableValue);
return overrideDisable;
}
@@ -77,10 +80,9 @@ public class DockerProviderService extends AbstractProviderService
ContainerLaunchService.ComponentLaunchContext compLaunchContext,
Map<String, String> tokensForSubstitution)
throws IOException, SliderException {
- Component component = instance.getComponent().getComponentSpec();
- boolean useEntryPoint = checkUseEntryPoint(component);
+ boolean useEntryPoint = checkUseEntryPoint(compLaunchContext);
if (useEntryPoint) {
- String launchCommand = component.getLaunchCommand();
+ String launchCommand = compLaunchContext.getLaunchCommand();
if (!StringUtils.isEmpty(launchCommand)) {
launcher.addCommand(launchCommand);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
index 7f29d1f..87406f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
@@ -33,9 +34,9 @@ public class TarballProviderService extends AbstractProviderService {
@Override
public void processArtifact(AbstractLauncher launcher,
ComponentInstance instance, SliderFileSystem fileSystem,
- Service service)
- throws IOException {
- Path artifact = new Path(instance.getCompSpec().getArtifact().getId());
+ Service service, ContainerLaunchService.ComponentLaunchContext
+ compLaunchCtx) throws IOException {
+ Path artifact = new Path(compLaunchCtx.getArtifact().getId());
if (!fileSystem.isFile(artifact)) {
throw new IOException(
"Package doesn't exist as a resource: " + artifact);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
index bb480ba..f428904 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
@@ -101,8 +101,8 @@ public class TestComponentInstance {
instance.handle(new ComponentInstanceEvent(instance.getContainer().getId(),
ComponentInstanceEventType.BECOME_READY));
Assert.assertEquals("instance not ready", ContainerState.READY,
- instance.getCompSpec().getContainer(
- instance.getContainer().getId().toString()).getState());
+ component.getComponentSpec().getContainer(instance.getContainer()
+ .getId().toString()).getState());
}
private void upgradeComponent(Component component) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
new file mode 100644
index 0000000..81ccc7f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.service.MockRunningServiceContext;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.ServiceTestUtils;
+import org.apache.hadoop.yarn.service.TestServiceManager;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderService;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for {@link AbstractProviderService}
+ */
+public class TestAbstractProviderService {
+
+ private ServiceContext serviceContext;
+ private Service testService;
+ private AbstractLauncher launcher;
+
+ @Rule
+ public ServiceTestUtils.ServiceFSWatcher rule =
+ new ServiceTestUtils.ServiceFSWatcher();
+
+ @Before
+ public void setup() throws Exception {
+ testService = TestServiceManager.createBaseDef("testService");
+ serviceContext = new MockRunningServiceContext(rule, testService);
+ launcher = new AbstractLauncher(serviceContext);
+ rule.getFs().setAppDir(new Path("target/testAbstractProviderService"));
+ }
+
+ @After
+ public void teardown() throws Exception {
+ FileUtils.deleteQuietly(
+ new File(rule.getFs().getAppDir().toUri().getPath()));
+ }
+
+ @Test
+ public void testBuildContainerLaunchCommand() throws Exception {
+ AbstractProviderService providerService = new DockerProviderService();
+ Component component = serviceContext.scheduler.getAllComponents().entrySet()
+ .iterator().next().getValue();
+ ContainerLaunchService.ComponentLaunchContext clc =
+ createEntryPointCLCFor(testService, component);
+
+ ComponentInstance instance = component.getAllComponentInstances().iterator()
+ .next();
+ Container container = mock(Container.class);
+ providerService.buildContainerLaunchCommand(launcher, testService, instance,
+ rule.getFs(), serviceContext.scheduler.getConfig(), container, clc,
+ null);
+
+ Assert.assertEquals("commands", Lists.newArrayList(clc.getLaunchCommand()),
+ launcher.getCommands());
+ }
+
+ @Test
+ public void testBuildContainerLaunchContext() throws Exception {
+ AbstractProviderService providerService = new DockerProviderService();
+ Component component = serviceContext.scheduler.getAllComponents().entrySet()
+ .iterator().next().getValue();
+ ContainerLaunchService.ComponentLaunchContext clc =
+ createEntryPointCLCFor(testService, component);
+
+ ComponentInstance instance = component.getAllComponentInstances().iterator()
+ .next();
+ Container container = mock(Container.class);
+ ContainerId containerId = ContainerId.newContainerId(
+ ApplicationAttemptId.newInstance(ApplicationId.newInstance(
+ System.currentTimeMillis(), 1), 1), 1L);
+ when(container.getId()).thenReturn(containerId);
+ providerService.buildContainerLaunchContext(launcher, testService, instance,
+ rule.getFs(), serviceContext.scheduler.getConfig(), container, clc);
+
+ Assert.assertEquals("artifact", clc.getArtifact().getId(),
+ launcher.getDockerImage());
+ }
+
+ private static ContainerLaunchService.ComponentLaunchContext
+ createEntryPointCLCFor(Service service, Component component) {
+ String launchCmd = "sleep,9000";
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.DOCKER);
+ artifact.setId("example");
+ Map<String, String> env = new HashMap<>();
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL", "true");
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE", "true");
+ component.getComponentSpec().getConfiguration().setEnv(env);
+
+ return new ContainerLaunchService.ComponentLaunchContext(
+ component.getName(),
+ service.getVersion())
+ .setArtifact(artifact)
+ .setConfiguration(component.getComponentSpec().getConfiguration())
+ .setLaunchCommand(launchCmd);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cba0741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 23ad408..2195232 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -43,6 +43,8 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -136,6 +138,8 @@ public class ContainerLaunch implements Callable<Integer> {
protected final LocalDirsHandlerService dirsHandler;
+ private final Lock containerExecLock = new ReentrantLock();
+
public ContainerLaunch(Context context, Configuration configuration,
Dispatcher dispatcher, ContainerExecutor exec, Application app,
Container container, LocalDirsHandlerService dirsHandler,
@@ -486,7 +490,12 @@ public class ContainerLaunch implements Callable<Integer> {
throws IOException, ConfigurationException {
int launchPrep = prepareForLaunch(ctx);
if (launchPrep == 0) {
- return exec.launchContainer(ctx);
+ containerExecLock.lock();
+ try {
+ return exec.launchContainer(ctx);
+ } finally {
+ containerExecLock.unlock();
+ }
}
return launchPrep;
}
@@ -495,7 +504,12 @@ public class ContainerLaunch implements Callable<Integer> {
throws IOException, ConfigurationException {
int launchPrep = prepareForLaunch(ctx);
if (launchPrep == 0) {
- return exec.relaunchContainer(ctx);
+ containerExecLock.lock();
+ try {
+ return exec.relaunchContainer(ctx);
+ } finally {
+ containerExecLock.unlock();
+ }
}
return launchPrep;
}
@@ -803,18 +817,22 @@ public class ContainerLaunch implements Callable<Integer> {
lfs.delete(pidFilePath.suffix(EXIT_CODE_FILE_SUFFIX), false);
}
}
-
- // Reap the container
- boolean result = exec.reapContainer(
- new ContainerReapContext.Builder()
- .setContainer(container)
- .setUser(container.getUser())
- .build());
- if (!result) {
- throw new IOException("Reap container failed for container "
- + containerIdStr);
+ containerExecLock.lock();
+ try {
+ // Reap the container
+ boolean result = exec.reapContainer(
+ new ContainerReapContext.Builder()
+ .setContainer(container)
+ .setUser(container.getUser())
+ .build());
+ if (!result) {
+ throw new IOException("Reap container failed for container "
+ + containerIdStr);
+ }
+ cleanupContainerFiles(getContainerWorkDir());
+ } finally {
+ containerExecLock.unlock();
}
- cleanupContainerFiles(getContainerWorkDir());
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org