You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by jl...@apache.org on 2013/06/03 16:53:01 UTC
svn commit: r1489016 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/j...
Author: jlowe
Date: Mon Jun 3 14:53:01 2013
New Revision: 1489016
URL: http://svn.apache.org/r1489016
Log:
svn merge -c 1489012 FIXES: MAPREDUCE-5268. Improve history server startup performance. Contributed by Karthik Kambatla
Added:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobIdHistoryFileInfoMap.java
- copied unchanged from r1489012, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobIdHistoryFileInfoMap.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobListCache.java
- copied unchanged from r1489012, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobListCache.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1489016&r1=1489015&r2=1489016&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon Jun 3 14:53:01 2013
@@ -10,6 +10,9 @@ Release 0.23.9 - UNRELEASED
OPTIMIZATIONS
+ MAPREDUCE-5268. Improve history server startup performance (Karthik
+ Kambatla via jlowe)
+
BUG FIXES
Release 0.23.8 - UNRELEASED
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java?rev=1489016&r1=1489015&r2=1489016&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java Mon Jun 3 14:53:01 2013
@@ -26,6 +26,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.NavigableSet;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
@@ -36,6 +37,7 @@ import java.util.concurrent.LinkedBlocki
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -131,19 +133,73 @@ public class HistoryFileManager extends
}
}
- static class JobListCache {
+ /**
+ * Wrapper around {@link ConcurrentSkipListMap} that maintains size along
+ * side for O(1) size() implementation for use in JobListCache.
+ *
+ * Note: The size is not updated atomically with changes additions/removals.
+ * This race can lead to size() returning an incorrect size at times.
+ */
+ static class JobIdHistoryFileInfoMap {
private ConcurrentSkipListMap<JobId, HistoryFileInfo> cache;
+ private AtomicInteger mapSize;
+
+ JobIdHistoryFileInfoMap() {
+ cache = new ConcurrentSkipListMap<JobId, HistoryFileInfo>();
+ mapSize = new AtomicInteger();
+ }
+
+ public HistoryFileInfo putIfAbsent(JobId key, HistoryFileInfo value) {
+ HistoryFileInfo ret = cache.putIfAbsent(key, value);
+ if (ret == null) {
+ mapSize.incrementAndGet();
+ }
+ return ret;
+ }
+
+ public HistoryFileInfo remove(JobId key) {
+ HistoryFileInfo ret = cache.remove(key);
+ if (ret != null) {
+ mapSize.decrementAndGet();
+ }
+ return ret;
+ }
+
+ /**
+ * Returns the recorded size of the internal map. Note that this could be out
+ * of sync with the actual size of the map
+ * @return "recorded" size
+ */
+ public int size() {
+ return mapSize.get();
+ }
+
+ public HistoryFileInfo get(JobId key) {
+ return cache.get(key);
+ }
+
+ public NavigableSet<JobId> navigableKeySet() {
+ return cache.navigableKeySet();
+ }
+
+ public Collection<HistoryFileInfo> values() {
+ return cache.values();
+ }
+ }
+
+ static class JobListCache {
+ private JobIdHistoryFileInfoMap cache;
private int maxSize;
private long maxAge;
public JobListCache(int maxSize, long maxAge) {
this.maxSize = maxSize;
this.maxAge = maxAge;
- this.cache = new ConcurrentSkipListMap<JobId, HistoryFileInfo>();
+ this.cache = new JobIdHistoryFileInfoMap();
}
public HistoryFileInfo addIfAbsent(HistoryFileInfo fileInfo) {
- JobId jobId = fileInfo.getJobIndexInfo().getJobId();
+ JobId jobId = fileInfo.getJobId();
if (LOG.isDebugEnabled()) {
LOG.debug("Adding " + jobId + " to job list cache with "
+ fileInfo.getJobIndexInfo());