You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/05/02 21:54:22 UTC
[26/50] [abbrv] hadoop git commit: YARN-6472. Improve Java sandbox
regex (gphillips via rkanter)
YARN-6472. Improve Java sandbox regex (gphillips via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b05bddb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b05bddb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b05bddb
Branch: refs/heads/HDFS-10467
Commit: 7b05bddb7695f9e9d0070e035034582d9f57c45a
Parents: 55b59b8
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Apr 28 11:01:50 2017 -0700
Committer: Inigo <in...@apache.org>
Committed: Tue May 2 14:52:05 2017 -0700
----------------------------------------------------------------------
.../DelegatingLinuxContainerRuntime.java | 9 +++++----
.../JavaSandboxLinuxContainerRuntime.java | 13 ++++++++-----
.../TestJavaSandboxLinuxContainerRuntime.java | 20 +++++++++++++++++---
3 files changed, 30 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b05bddb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index 896ffd0..90b13a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -70,11 +70,12 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
private LinuxContainerRuntime pickContainerRuntime(
Map<String, String> environment){
LinuxContainerRuntime runtime;
-
- if (DockerLinuxContainerRuntime.isDockerContainerRequested(environment)){
+ //Sandbox checked first to ensure DockerRuntime doesn't circumvent controls
+ if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()){
+ runtime = javaSandboxLinuxContainerRuntime;
+ } else if (DockerLinuxContainerRuntime
+ .isDockerContainerRequested(environment)){
runtime = dockerLinuxContainerRuntime;
- } else if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()) {
- runtime = javaSandboxLinuxContainerRuntime;
} else {
runtime = defaultLinuxContainerRuntime;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b05bddb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
index 6dc627b..1e5bf57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
@@ -141,8 +141,6 @@ public class JavaSandboxLinuxContainerRuntime
this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
- initializePolicyDir();
-
super.initialize(conf);
}
@@ -223,6 +221,7 @@ public class JavaSandboxLinuxContainerRuntime
OutputStream policyOutputStream = null;
try {
String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
+ initializePolicyDir();
Path policyFilePath = Files.createFile(
Paths.get(policyFileDir.toString(),
@@ -368,8 +367,12 @@ public class JavaSandboxLinuxContainerRuntime
static final String STRIP_POLICY_FLAG = POLICY_APPEND_FLAG + "[^ ]+";
static final String CONTAINS_JAVA_CMD = "\\$" + JAVA_HOME + JAVA_CMD + ".*";
- static final String CHAINED_COMMAND_REGEX =
- "^.*(&&.+$)|(\\|\\|.+$).*$"; //Matches any occurrences of '||' or '&&'
+ static final String MULTI_COMMAND_REGEX =
+ "(?s).*(" + //command read as single line
+ "(&[^>]|&&)|(\\|{1,2})|(\\|&)|" + //Matches '&','&&','|','||' and '|&'
+ "(`[^`]+`)|(\\$\\([^)]+\\))|" + //Matches occurrences of $() or ``
+ "(;)" + //Matches end of statement ';'
+ ").*";
static final String CLEAN_CMD_REGEX =
"(" + SECURITY_FLAG + ")|" +
"(" + STRIP_POLICY_FLAG + ")";
@@ -459,7 +462,7 @@ public class JavaSandboxLinuxContainerRuntime
String command = commands.get(i);
if(validateJavaHome(env.get(JAVA_HOME.name()))
&& command.matches(CONTAINS_JAVA_CMD)
- && !command.matches(CHAINED_COMMAND_REGEX)){
+ && !command.matches(MULTI_COMMAND_REGEX)){
command = command.replaceAll(CLEAN_CMD_REGEX, "");
String securityString = JVM_SECURITY_CMD + policyPath + " ";
if(LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b05bddb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
index e482c8d..e10d0dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
@@ -47,7 +47,8 @@ import java.util.List;
import java.util.Map;
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
-import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CHAINED_COMMAND_REGEX;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.LOG;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.MULTI_COMMAND_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
@@ -293,8 +294,21 @@ public class TestJavaSandboxLinuxContainerRuntime {
@Test
public void testChainedCmdRegex(){
- Assert.assertTrue("cmd1 && cmd2 || cmd3".matches(CHAINED_COMMAND_REGEX));
- Assert.assertFalse("cmd1 &> logfile".matches(CHAINED_COMMAND_REGEX));
+ String[] multiCmds = {
+ "cmd1 && cmd2",
+ "cmd1 || cmd2",
+ "cmd1 `cmd2`",
+ "cmd1 $(cmd2)",
+ "cmd1; \\\n cmd2",
+ "cmd1; cmd2",
+ "cmd1|&cmd2",
+ "cmd1|cmd2",
+ "cmd1&cmd2"
+ };
+
+ Arrays.stream(multiCmds)
+ .forEach(cmd -> Assert.assertTrue(cmd.matches(MULTI_COMMAND_REGEX)));
+ Assert.assertFalse("cmd1 &> logfile".matches(MULTI_COMMAND_REGEX));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org