You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by eb...@apache.org on 2019/03/11 17:55:31 UTC
[hadoop] branch trunk updated: YARN-8805. Automatically convert the
launch command to the exec form when using entrypoint support
This is an automated email from the ASF dual-hosted git repository.
ebadger pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new fa7a0b2 YARN-8805. Automatically convert the launch command to the exec form when using entrypoint support
fa7a0b2 is described below
commit fa7a0b269a8613627819c41ab0e9c02a55e278b3
Author: Eric Badger <eb...@verizonmedia.com>
AuthorDate: Mon Mar 11 12:43:14 2019 -0500
YARN-8805. Automatically convert the launch command to the exec form when using entrypoint support
---
.../yarn/service/provider/ProviderUtils.java | 17 ++++++++++++++
.../provider/docker/DockerProviderService.java | 6 +++++
.../provider/TestAbstractProviderService.java | 27 ++++++++++++++++++----
.../yarn/service/provider/TestProviderUtils.java | 9 ++++++++
4 files changed, 55 insertions(+), 4 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 88883f7..ea1fb0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -47,9 +47,12 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.COMPONENT_ID;
@@ -149,6 +152,20 @@ public class ProviderUtils implements YarnServiceConstants {
return content;
}
+ public static String replaceSpacesWithDelimiter(String content,
+ String delimiter) {
+ List<String> parts = new ArrayList<String>();
+ Matcher m = Pattern.compile("([^\"]\\S*|\".+?\")\\s*").matcher(content);
+ while (m.find()) {
+ String part = m.group(1);
+ if(part.startsWith("\"") && part.endsWith("\"")) {
+ part = part.replaceAll("^\"|\"$", "");
+ }
+ parts.add(part);
+ }
+ return String.join(delimiter, parts);
+ }
+
// configs will be substituted by corresponding env in tokenMap
public static void substituteMapWithTokens(Map<String, String> configs,
Map<String, String> tokenMap) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
index 6027a66..9b4138e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
@@ -84,6 +85,11 @@ public class DockerProviderService extends AbstractProviderService
if (useEntryPoint) {
String launchCommand = compLaunchContext.getLaunchCommand();
if (!StringUtils.isEmpty(launchCommand)) {
+ if(launchCommand.contains(" ")) {
+ // convert space delimiter command to exec format
+ launchCommand = ProviderUtils
+ .replaceSpacesWithDelimiter(launchCommand, ",");
+ }
launcher.addCommand(launchCommand);
}
} else {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
index 81ccc7f..f0525aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
@@ -82,7 +82,7 @@ public class TestAbstractProviderService {
Component component = serviceContext.scheduler.getAllComponents().entrySet()
.iterator().next().getValue();
ContainerLaunchService.ComponentLaunchContext clc =
- createEntryPointCLCFor(testService, component);
+ createEntryPointCLCFor(testService, component, "sleep,9000");
ComponentInstance instance = component.getAllComponentInstances().iterator()
.next();
@@ -96,12 +96,31 @@ public class TestAbstractProviderService {
}
@Test
+ public void testBuildContainerLaunchCommandWithSpace() throws Exception {
+ AbstractProviderService providerService = new DockerProviderService();
+ Component component = serviceContext.scheduler.getAllComponents().entrySet()
+ .iterator().next().getValue();
+ ContainerLaunchService.ComponentLaunchContext clc =
+ createEntryPointCLCFor(testService, component, "ls -l \" space\"");
+
+ ComponentInstance instance = component.getAllComponentInstances().iterator()
+ .next();
+ Container container = mock(Container.class);
+ providerService.buildContainerLaunchCommand(launcher, testService, instance,
+ rule.getFs(), serviceContext.scheduler.getConfig(), container, clc,
+ null);
+
+ Assert.assertEquals("commands don't match.",
+ Lists.newArrayList("ls,-l, space"), launcher.getCommands());
+ }
+
+ @Test
public void testBuildContainerLaunchContext() throws Exception {
AbstractProviderService providerService = new DockerProviderService();
Component component = serviceContext.scheduler.getAllComponents().entrySet()
.iterator().next().getValue();
ContainerLaunchService.ComponentLaunchContext clc =
- createEntryPointCLCFor(testService, component);
+ createEntryPointCLCFor(testService, component, "sleep,9000");
ComponentInstance instance = component.getAllComponentInstances().iterator()
.next();
@@ -118,8 +137,8 @@ public class TestAbstractProviderService {
}
private static ContainerLaunchService.ComponentLaunchContext
- createEntryPointCLCFor(Service service, Component component) {
- String launchCmd = "sleep,9000";
+ createEntryPointCLCFor(Service service, Component component,
+ String launchCmd) {
Artifact artifact = new Artifact();
artifact.setType(Artifact.TypeEnum.DOCKER);
artifact.setId("example");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
index 0f7f375..84c3b6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
@@ -168,4 +168,13 @@ public class TestProviderUtils {
Assert.assertEquals(resolved.getResolvedRsrcPaths().get("destFile1"),
"destFile1");
}
+
+ @Test
+ public void testReplaceSpacesWithDelimiter() {
+ String command = "ls -l \" space\"";
+ String expected = "ls,-l, space";
+ String actual = ProviderUtils.replaceSpacesWithDelimiter(command, ",");
+ Assert.assertEquals("replaceSpaceWithDelimiter produces unexpected result.",
+ expected, actual);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org