You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/04/07 01:59:30 UTC
[34/50] [abbrv] hadoop git commit: YARN-6403. Invalid local resource
request can raise NPE and make NM exit. Contributed by Tao Yang
YARN-6403. Invalid local resource request can raise NPE and make NM exit. Contributed by Tao Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93932094
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93932094
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93932094
Branch: refs/heads/HDFS-10467
Commit: 939320941ad870adacc26fe096b278cf63529312
Parents: ca33197
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Wed Apr 5 12:30:45 2017 -0500
Committer: Inigo <in...@apache.org>
Committed: Thu Apr 6 18:58:22 2017 -0700
----------------------------------------------------------------------
.../impl/pb/ContainerLaunchContextPBImpl.java | 13 ++++++
.../TestApplicationClientProtocolRecords.java | 29 +++++++++++++
.../containermanager/ContainerManagerImpl.java | 10 +++++
.../TestContainerManagerWithLCE.java | 11 +++++
.../containermanager/TestContainerManager.java | 45 ++++++++++++++++++++
5 files changed, 108 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93932094/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
index 1f76c34..f07a9d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
@@ -208,11 +208,24 @@ extends ContainerLaunchContext {
final Map<String, LocalResource> localResources) {
if (localResources == null)
return;
+ checkLocalResources(localResources);
initLocalResources();
this.localResources.clear();
this.localResources.putAll(localResources);
}
+ private void checkLocalResources(Map<String, LocalResource> localResources) {
+ for (Map.Entry<String, LocalResource> rsrcEntry : localResources
+ .entrySet()) {
+ if (rsrcEntry.getValue() == null
+ || rsrcEntry.getValue().getResource() == null) {
+ throw new NullPointerException(
+ "Null resource URL for local resource " + rsrcEntry.getKey() + " : "
+ + rsrcEntry.getValue());
+ }
+ }
+ }
+
private void addLocalResourcesToProto() {
maybeInitBuilder();
builder.clearLocalResources();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93932094/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
index 0294ad1..8773d11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
@@ -30,6 +30,10 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.junit.Assert;
import org.junit.Test;
@@ -66,4 +70,29 @@ public class TestApplicationClientProtocolRecords {
clcProto.getEnvironment().get("testCLCPBImplNullEnv"));
}
+
+ /*
+ * This test validates the scenario in which the client sets a null value for
+ * local resource URL.
+ */
+ @Test
+ public void testCLCPBImplNullResourceURL() throws IOException {
+ RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+ try {
+ LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
+ rsrc_alpha.setResource(null);
+ rsrc_alpha.setSize(-1);
+ rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
+ rsrc_alpha.setType(LocalResourceType.FILE);
+ rsrc_alpha.setTimestamp(System.currentTimeMillis());
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put("null_url_resource", rsrc_alpha);
+ ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ containerLaunchContext.setLocalResources(localResources);
+ Assert.fail("Setting an invalid local resource should be an error!");
+ } catch (NullPointerException e) {
+ Assert.assertTrue(e.getMessage().contains("Null resource URL for local resource"));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93932094/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index f3fe8cc..85dc5fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -996,6 +997,15 @@ public class ContainerManagerImpl extends CompositeService implements
ContainerLaunchContext launchContext = request.getContainerLaunchContext();
+ // Sanity check for local resources
+ for (Map.Entry<String, LocalResource> rsrc : launchContext
+ .getLocalResources().entrySet()) {
+ if (rsrc.getValue() == null || rsrc.getValue().getResource() == null) {
+ throw new YarnException(
+ "Null resource URL for local resource " + rsrc.getKey() + " : " + rsrc.getValue());
+ }
+ }
+
Credentials credentials =
YarnServerSecurityUtils.parseCredentials(launchContext);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93932094/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
index 8221827..5dc4717 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -364,6 +364,17 @@ public class TestContainerManagerWithLCE extends TestContainerManager {
super.testContainerRestart();
}
+ @Override
+ public void testStartContainerFailureWithInvalidLocalResource() throws Exception {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ LOG.info("Running testStartContainerFailureWithInvalidLocalResource");
+ super.testStartContainerFailureWithInvalidLocalResource();
+ }
+
private boolean shouldRunTest() {
return System
.getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH) != null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93932094/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 66686a5..6fead7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -1859,4 +1859,49 @@ public class TestContainerManager extends BaseContainerManagerTest {
Assert.assertEquals(signal, signalContext.getSignal());
}
}
+
+ @Test
+ public void testStartContainerFailureWithInvalidLocalResource()
+ throws Exception {
+ containerManager.start();
+ LocalResource rsrc_alpha =
+ recordFactory.newRecordInstance(LocalResource.class);
+ rsrc_alpha.setResource(null);
+ rsrc_alpha.setSize(-1);
+ rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
+ rsrc_alpha.setType(LocalResourceType.FILE);
+ rsrc_alpha.setTimestamp(System.currentTimeMillis());
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put("invalid_resource", rsrc_alpha);
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ContainerLaunchContext spyContainerLaunchContext =
+ Mockito.spy(containerLaunchContext);
+ Mockito.when(spyContainerLaunchContext.getLocalResources())
+ .thenReturn(localResources);
+
+ ContainerId cId = createContainerId(0);
+ String user = "start_container_fail";
+ Token containerToken =
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
+ user, context.getContainerTokenSecretManager());
+ StartContainerRequest request = StartContainerRequest
+ .newInstance(spyContainerLaunchContext, containerToken);
+
+ // start containers
+ List<StartContainerRequest> startRequest =
+ new ArrayList<StartContainerRequest>();
+ startRequest.add(request);
+ StartContainersRequest requestList =
+ StartContainersRequest.newInstance(startRequest);
+
+ StartContainersResponse response =
+ containerManager.startContainers(requestList);
+ Assert.assertTrue(response.getFailedRequests().size() == 1);
+ Assert.assertTrue(response.getSuccessfullyStartedContainers().size() == 0);
+ Assert.assertTrue(response.getFailedRequests().containsKey(cId));
+ Assert.assertTrue(response.getFailedRequests().get(cId).getMessage()
+ .contains("Null resource URL for local resource"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org