You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/05/04 00:05:52 UTC
[05/50] hadoop git commit: MAPREDUCE-6680. JHS UserLogDir scan
algorithm sometime could skip directory with update in CloudFS (Azure
FileSystem, S3, etc. Contributed by Junping Du
MAPREDUCE-6680. JHS UserLogDir scan algorithm sometime could skip directory with update in CloudFS (Azure FileSystem, S3, etc. Contributed by Junping Du
(cherry picked from commit 1e48eefe5800975ea0c4295c9911ae3f572ed37d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b008dc2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b008dc2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b008dc2
Branch: refs/heads/branch-2.8
Commit: 0b008dc21c3c8c9dbb211d0d176db5936d853485
Parents: 7fb1cfc59
Author: Jian He <ji...@apache.org>
Authored: Wed Apr 20 19:02:10 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Wed Apr 20 19:03:19 2016 -0700
----------------------------------------------------------------------
.../mapreduce/v2/hs/HistoryFileManager.java | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b008dc2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 01fb9d5..0b22d13 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -306,10 +306,21 @@ public class HistoryFileManager extends AbstractService {
*/
private class UserLogDir {
long modTime = 0;
-
+ private long scanTime = 0;
+
public synchronized void scanIfNeeded(FileStatus fs) {
long newModTime = fs.getModificationTime();
- if (modTime != newModTime) {
+ // MAPREDUCE-6680: In some Cloud FileSystem, like Azure FS or S3, file's
+ // modification time is truncated into seconds. In that case,
+ // modTime == newModTime doesn't means no file update in the directory,
+ // so we need to have additional check.
+ // Note: modTime (X second Y millisecond) could be casted to X second or
+ // X+1 second.
+ if (modTime != newModTime
+ || (scanTime/1000) == (modTime/1000)
+ || (scanTime/1000 + 1) == (modTime/1000)) {
+ // reset scanTime before scanning happens
+ scanTime = System.currentTimeMillis();
Path p = fs.getPath();
try {
scanIntermediateDirectory(p);
@@ -323,10 +334,12 @@ public class HistoryFileManager extends AbstractService {
if (LOG.isDebugEnabled()) {
LOG.debug("Scan not needed of " + fs.getPath());
}
+ // reset scanTime
+ scanTime = System.currentTimeMillis();
}
}
}
-
+
public class HistoryFileInfo {
private Path historyFile;
private Path confFile;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org