You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2019/07/10 09:15:41 UTC
[hadoop] branch branch-3.2 updated: YARN-9557. Application fails in
diskchecker when ReadWriteDiskValidator is configured. Contributed by Bilwa
S T.
This is an automated email from the ASF dual-hosted git repository.
bibinchundatt pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 5f8395f YARN-9557. Application fails in diskchecker when ReadWriteDiskValidator is configured. Contributed by Bilwa S T.
5f8395f is described below
commit 5f8395f393e0759215c56927dae1297dfdb0b955
Author: bibinchundatt <bi...@apache.org>
AuthorDate: Wed Jul 10 10:34:39 2019 +0530
YARN-9557. Application fails in diskchecker when ReadWriteDiskValidator is configured. Contributed by Bilwa S T.
---
.../localizer/ContainerLocalizer.java | 11 +++++++--
.../localizer/TestContainerLocalizer.java | 28 ++++++++++++++++++++++
2 files changed, 37 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index c034369..2879cc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -130,13 +130,19 @@ public class ContainerLocalizer {
this.localDirs = localDirs;
this.localizerId = localizerId;
this.recordFactory = recordFactory;
- this.conf = new YarnConfiguration();
+ this.conf = initConfiguration();
this.diskValidator = DiskValidatorFactory.getInstance(
YarnConfiguration.DEFAULT_DISK_VALIDATOR);
this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
this.pendingResources = new HashMap<LocalResource,Future<Path>>();
}
+ @VisibleForTesting
+ @Private
+ Configuration initConfiguration() {
+ return new YarnConfiguration();
+ }
+
@Private
@VisibleForTesting
public LocalizationProtocol getProxy(final InetSocketAddress nmAddr) {
@@ -246,7 +252,8 @@ public class ContainerLocalizer {
if (rsrc.getVisibility() == LocalResourceVisibility.PRIVATE) {
createParentDirs(destDirPath);
}
- diskValidator.checkStatus(new File(destDirPath.toUri().getRawPath()));
+ diskValidator
+ .checkStatus(new File(destDirPath.getParent().toUri().getRawPath()));
return new FSDownloadWrapper(lfs, ugi, conf, destDirPath, rsrc);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
index 17eee82..5bb84bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
@@ -68,12 +68,14 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -240,6 +242,32 @@ public class TestContainerLocalizer {
}
@Test
+ public void testDiskCheckFailure() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.DISK_VALIDATOR, "read-write");
+ FileContext lfs = FileContext.getLocalFSFileContext(conf);
+ Path fileCacheDir = lfs.makeQualified(new Path(basedir, "filecache"));
+ lfs.mkdir(fileCacheDir, FsPermission.getDefault(), true);
+ RecordFactory recordFactory = mock(RecordFactory.class);
+ ContainerLocalizer localizer = new ContainerLocalizer(lfs,
+ UserGroupInformation.getCurrentUser().getUserName(), "application_01",
+ "container_01", new ArrayList<>(), recordFactory) {
+ @Override
+ Configuration initConfiguration() {
+ return conf;
+ }
+ };
+ LocalResource rsrc = mock(LocalResource.class);
+ Path destDirPath = new Path(fileCacheDir, "11");
+ try {
+ localizer.download(destDirPath, rsrc,
+ UserGroupInformation.getCurrentUser());
+ } catch (DiskErrorException ex) {
+ fail(ex.getCause().toString());
+ }
+ }
+
+ @Test
@SuppressWarnings("unchecked")
public void testLocalizerTokenIsGettingRemoved() throws Exception {
ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org