You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2015/11/25 00:48:09 UTC
hadoop git commit: YARN-4365. FileSystemNodeLabelStore should check
for root dir existence on startup. Contributed by Kuhu Shukla (cherry picked
from commit f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 9e54433c6 -> 3a2f5d632
YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla
(cherry picked from commit f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a2f5d63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a2f5d63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a2f5d63
Branch: refs/heads/branch-2
Commit: 3a2f5d6329e80884b795899f0c42c0d61d729e15
Parents: 9e54433
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Nov 24 23:47:10 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Nov 24 23:47:54 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 ++++
.../nodelabels/FileSystemNodeLabelsStore.java | 6 ++--
.../TestFileSystemNodeLabelsStore.java | 29 ++++++++++++++++++++
3 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a2f5d63/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fee03c5..fe1d3f0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1048,6 +1048,9 @@ Release 2.7.3 - UNRELEASED
YARN-4344. NMs reconnecting with changed capabilities can lead to wrong
cluster resource calculations (Varun Vasudev via jlowe)
+ YARN-4365. FileSystemNodeLabelStore should check for root dir existence on
+ startup (Kuhu Shukla via jlowe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -1914,6 +1917,9 @@ Release 2.6.3 - UNRELEASED
YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container
log files from full disks. (zhihai xu via jlowe)
+ YARN-4365. FileSystemNodeLabelStore should check for root dir existence on
+ startup (Kuhu Shukla via jlowe)
+
Release 2.6.2 - 2015-10-28
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a2f5d63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
index 20dc67c..c9727a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
@@ -88,7 +88,9 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore {
setFileSystem(conf);
// mkdir of root dir path
- fs.mkdirs(fsWorkingPath);
+ if (!fs.exists(fsWorkingPath)) {
+ fs.mkdirs(fsWorkingPath);
+ }
}
@Override
@@ -96,7 +98,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore {
IOUtils.cleanup(LOG, fs, editlogOs);
}
- private void setFileSystem(Configuration conf) throws IOException {
+ void setFileSystem(Configuration conf) throws IOException {
Configuration confCopy = new Configuration(conf);
confCopy.setBoolean("dfs.client.retry.policy.enabled", true);
String retryPolicy =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a2f5d63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
index 4b052c9..4929f95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.InlineDispatcher;
@@ -33,6 +35,7 @@ import org.junit.Before;
import org.junit.Test;
import com.google.common.collect.ImmutableMap;
+import org.mockito.Mockito;
public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
MockNodeLabelManager mgr = null;
@@ -317,4 +320,30 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
Arrays.asList("p2", "p4", "p6", "p7", "p8", "p9")));
mgr.stop();
}
+
+ @Test
+ public void testRootMkdirOnInitStore() throws Exception {
+ final FileSystem mockFs = Mockito.mock(FileSystem.class);
+ FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore(mgr) {
+ void setFileSystem(Configuration conf) throws IOException {
+ fs = mockFs;
+ }
+ };
+ mockStore.fs = mockFs;
+ verifyMkdirsCount(mockStore, true, 0);
+ verifyMkdirsCount(mockStore, false, 1);
+ verifyMkdirsCount(mockStore, true, 1);
+ verifyMkdirsCount(mockStore, false, 2);
+ }
+
+ private void verifyMkdirsCount(FileSystemNodeLabelsStore store,
+ boolean existsRetVal, int expectedNumOfCalls)
+ throws Exception {
+ Mockito.when(store.fs.exists(Mockito.any(
+ Path.class))).thenReturn(existsRetVal);
+ store.init(conf);
+ Mockito.verify(store.fs,Mockito.times(
+ expectedNumOfCalls)).mkdirs(Mockito.any(Path
+ .class));
+ }
}