You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/09/29 18:14:01 UTC
[39/50] [abbrv] hadoop git commit: MAPREDUCE-6718. add progress log
to JHS during startup (haibochen via rkanter)
MAPREDUCE-6718. add progress log to JHS during startup (haibochen via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d6778d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d6778d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d6778d8
Branch: refs/heads/HDFS-7240
Commit: 0d6778d800ff16366911e3b064f3af6162dee2e4
Parents: bcb2528
Author: Robert Kanter <rk...@apache.org>
Authored: Wed Sep 28 15:41:40 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Sep 28 15:41:40 2016 -0700
----------------------------------------------------------------------
.../mapreduce/v2/hs/HistoryFileManager.java | 21 ++++++++++++++++++++
1 file changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6778d8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 12e7a27..0f09df2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -302,6 +302,10 @@ public class HistoryFileManager extends AbstractService {
public boolean isFull() {
return cache.size() >= maxSize;
}
+
+ public int size() {
+ return cache.size();
+ }
}
/**
@@ -612,6 +616,9 @@ public class HistoryFileManager extends AbstractService {
while (!done &&
((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3 attempts, 30sec
+ if (done) {
+ break;
+ }
try {
Thread.sleep(intervalCheckMillis);
} catch (InterruptedException ex) {
@@ -760,15 +767,29 @@ public class HistoryFileManager extends AbstractService {
List<FileStatus> timestampedDirList = findTimestampedDirectories();
// Sort first just so insertion is in a consistent order
Collections.sort(timestampedDirList);
+ LOG.info("Found " + timestampedDirList.size() + " directories to load");
for (FileStatus fs : timestampedDirList) {
// TODO Could verify the correct format for these directories.
addDirectoryToSerialNumberIndex(fs.getPath());
}
+ final double maxCacheSize = (double) jobListCache.maxSize;
+ int prevCacheSize = jobListCache.size();
for (int i= timestampedDirList.size() - 1;
i >= 0 && !jobListCache.isFull(); i--) {
FileStatus fs = timestampedDirList.get(i);
addDirectoryToJobListCache(fs.getPath());
+
+ int currCacheSize = jobListCache.size();
+ if((currCacheSize - prevCacheSize)/maxCacheSize >= 0.05) {
+ LOG.info(currCacheSize * 100.0 / maxCacheSize +
+ "% of cache is loaded.");
+ }
+ prevCacheSize = currCacheSize;
}
+ final double loadedPercent = maxCacheSize == 0.0 ?
+ 100 : prevCacheSize * 100.0 / maxCacheSize;
+ LOG.info("Existing job initialization finished. " +
+ loadedPercent + "% of cache is occupied.");
}
private void removeDirectoryFromSerialNumberIndex(Path serialDirPath) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org