You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2018/07/25 01:31:55 UTC
[12/50] hadoop git commit: YARN-7300. DiskValidator is not used in
LocalDirAllocator. (Szilard Nemeth via Haibo Chen)
YARN-7300. DiskValidator is not used in LocalDirAllocator. (Szilard Nemeth via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6873dfd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6873dfd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6873dfd
Branch: refs/heads/HADOOP-15461
Commit: e6873dfde057e63ce5efa91f3061db3ee1b2e236
Parents: f354f47
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Jul 19 16:27:11 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Thu Jul 19 16:27:11 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/LocalDirAllocator.java | 28 +++++++++++++++-----
.../nodemanager/LocalDirsHandlerService.java | 27 ++++++++++++++-----
2 files changed, 42 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6873dfd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
index 1c216f4..a4b158a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
@@ -24,8 +24,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.util.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -78,11 +76,25 @@ public class LocalDirAllocator {
/** Used when size of file to be allocated is unknown. */
public static final int SIZE_UNKNOWN = -1;
+ private final DiskValidator diskValidator;
+
/**Create an allocator object
* @param contextCfgItemName
*/
public LocalDirAllocator(String contextCfgItemName) {
this.contextCfgItemName = contextCfgItemName;
+ try {
+ this.diskValidator = DiskValidatorFactory.getInstance(
+ BasicDiskValidator.NAME);
+ } catch (DiskErrorException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public LocalDirAllocator(String contextCfgItemName,
+ DiskValidator diskValidator) {
+ this.contextCfgItemName = contextCfgItemName;
+ this.diskValidator = diskValidator;
}
/** This method must be used to obtain the dir allocation context for a
@@ -96,7 +108,8 @@ public class LocalDirAllocator {
AllocatorPerContext l = contexts.get(contextCfgItemName);
if (l == null) {
contexts.put(contextCfgItemName,
- (l = new AllocatorPerContext(contextCfgItemName)));
+ (l = new AllocatorPerContext(contextCfgItemName,
+ diskValidator)));
}
return l;
}
@@ -255,6 +268,7 @@ public class LocalDirAllocator {
// NOTE: the context must be accessed via a local reference as it
// may be updated at any time to reference a different context
private AtomicReference<Context> currentContext;
+ private final DiskValidator diskValidator;
private static class Context {
private AtomicInteger dirNumLastAccessed = new AtomicInteger(0);
@@ -280,9 +294,11 @@ public class LocalDirAllocator {
}
}
- public AllocatorPerContext(String contextCfgItemName) {
+ public AllocatorPerContext(String contextCfgItemName,
+ DiskValidator diskValidator) {
this.contextCfgItemName = contextCfgItemName;
this.currentContext = new AtomicReference<Context>(new Context());
+ this.diskValidator = diskValidator;
}
/** This method gets called everytime before any read/write to make sure
@@ -312,7 +328,7 @@ public class LocalDirAllocator {
? new File(ctx.localFS.makeQualified(tmpDir).toUri())
: new File(dirStrings[i]);
- DiskChecker.checkDir(tmpFile);
+ diskValidator.checkStatus(tmpFile);
dirs.add(new Path(tmpFile.getPath()));
dfList.add(new DF(tmpFile, 30000));
} catch (DiskErrorException de) {
@@ -348,7 +364,7 @@ public class LocalDirAllocator {
//check whether we are able to create a directory here. If the disk
//happens to be RDONLY we will fail
try {
- DiskChecker.checkDir(new File(file.getParent().toUri().getPath()));
+ diskValidator.checkStatus(new File(file.getParent().toUri().getPath()));
return file;
} catch (DiskErrorException d) {
LOG.warn("Disk Error Exception: ", d);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6873dfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index 621cabc..6eabd0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -27,6 +27,9 @@ import java.util.List;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+import org.apache.hadoop.util.DiskValidator;
+import org.apache.hadoop.util.DiskValidatorFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -155,13 +158,23 @@ public class LocalDirsHandlerService extends AbstractService {
String local = conf.get(YarnConfiguration.NM_LOCAL_DIRS);
conf.set(NM_GOOD_LOCAL_DIRS,
(local != null) ? local : "");
- localDirsAllocator = new LocalDirAllocator(
- NM_GOOD_LOCAL_DIRS);
- String log = conf.get(YarnConfiguration.NM_LOG_DIRS);
- conf.set(NM_GOOD_LOG_DIRS,
- (log != null) ? log : "");
- logDirsAllocator = new LocalDirAllocator(
- NM_GOOD_LOG_DIRS);
+ String diskValidatorName = conf.get(YarnConfiguration.DISK_VALIDATOR,
+ YarnConfiguration.DEFAULT_DISK_VALIDATOR);
+ try {
+ DiskValidator diskValidator =
+ DiskValidatorFactory.getInstance(diskValidatorName);
+ localDirsAllocator = new LocalDirAllocator(
+ NM_GOOD_LOCAL_DIRS, diskValidator);
+ String log = conf.get(YarnConfiguration.NM_LOG_DIRS);
+ conf.set(NM_GOOD_LOG_DIRS,
+ (log != null) ? log : "");
+ logDirsAllocator = new LocalDirAllocator(
+ NM_GOOD_LOG_DIRS, diskValidator);
+ } catch (DiskErrorException e) {
+ throw new YarnRuntimeException(
+ "Failed to create DiskValidator of type " + diskValidatorName + "!",
+ e);
+ }
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org