You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/10/21 01:39:17 UTC
svn commit: r1400532 - in /hadoop/common/branches/branch-1-win: ./ bin/
src/core/org/apache/hadoop/security/token/
src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/mapred/
src/mapred/org/apache/hadoop/mapred/
src/test/org/apache/hadoop/hdfs/server/...
Author: suresh
Date: Sat Oct 20 23:39:16 2012
New Revision: 1400532
URL: http://svn.apache.org/viewvc?rev=1400532&view=rev
Log:
Merging changes r1361046:r1389199 that correspond to changes introduced in release 1.0.4
Modified:
hadoop/common/branches/branch-1-win/ (props changed)
hadoop/common/branches/branch-1-win/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-1-win/bin/hadoop-config.sh
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/token/SecretManager.java
hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/branch-1-win/src/mapred/ (props changed)
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java
Propchange: hadoop/common/branches/branch-1-win/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-1.0:r1361047-1389199
Modified: hadoop/common/branches/branch-1-win/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.txt?rev=1400532&r1=1400531&r2=1400532&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.txt Sat Oct 20 23:39:16 2012
@@ -27,6 +27,12 @@ Release 1.1.1 - Unreleased
HADOOP-8823. ant package target should not depend on cn-docs. (szetszwo)
+ HADOOP-7154. Should set MALLOC_ARENA_MAX in hadoop-env.sh
+ (todd via mattf)
+
+ MAPREDUCE-4399. Change the Jetty response buffer size to improve
+ shuffle performance. (Luke Lu via suresh)
+
BUG FIXES
Release 1.1.0 - 2012.09.28
@@ -188,6 +194,9 @@ Release 1.1.0 - 2012.09.28
HDFS-3163. TestHDFSCLI.testAll fails if user name is not all lowercase.
(Brandon Li via suresh)
+ HDFS-3652. FSEditLog failure removes the wrong edit stream when storage
+ dirs have same name. (todd)
+
Release 1.0.3 - 2012.05.07
NEW FEATURES
Propchange: hadoop/common/branches/branch-1-win/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-1.0/CHANGES.txt:r1361047-1389199
Merged /hadoop/common/branches/branch-1.1/CHANGES.txt:r1365208
Modified: hadoop/common/branches/branch-1-win/bin/hadoop-config.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/bin/hadoop-config.sh?rev=1400532&r1=1400531&r2=1400532&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/bin/hadoop-config.sh (original)
+++ hadoop/common/branches/branch-1-win/bin/hadoop-config.sh Sat Oct 20 23:39:16 2012
@@ -75,6 +75,11 @@ if [ "$HADOOP_HOME_WARN_SUPPRESS" = "" ]
echo 1>&2
fi
+# Newer versions of glibc use an arena memory allocator that causes virtual
+# memory usage to explode. This interacts badly with the many threads that
+# we use in Hadoop. Tune the variable down to prevent vmem explosion.
+export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
+
export HADOOP_HOME=${HADOOP_PREFIX}
export HADOOP_HOME_WARN_SUPPRESS=1
Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/token/SecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/token/SecretManager.java?rev=1400532&r1=1400531&r2=1400532&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/token/SecretManager.java (original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/token/SecretManager.java Sat Oct 20 23:39:16 2012
@@ -75,7 +75,7 @@ public abstract class SecretManager<T ex
/**
* The length of the random keys to use.
*/
- private static final int KEY_LENGTH = 20;
+ private static final int KEY_LENGTH = 64;
/**
* A thread local store for the Macs.
Modified: hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1400532&r1=1400531&r2=1400532&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Sat Oct 20 23:39:16 2012
@@ -459,7 +459,8 @@ public class FSEditLog {
}
for (int idx = 0; idx < editStreams.size(); idx++) {
File parentDir = getStorageDirForStream(idx);
- if (parentDir.getName().equals(sd.getRoot().getName())) {
+ if (parentDir.getAbsolutePath().equals(
+ sd.getRoot().getAbsolutePath())) {
EditLogOutputStream s = editStreams.remove(idx);
try {
s.close();
@@ -467,6 +468,7 @@ public class FSEditLog {
} catch (IOException e) {
LOG.warn("Failed to close the stream or unlock storage dir");
}
+ idx--;
}
}
exitIfNoStreams();
Propchange: hadoop/common/branches/branch-1-win/src/mapred/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-1.0/src/mapred:r1361047-1389199
Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1400532&r1=1400531&r2=1400532&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Sat Oct 20 23:39:16 2012
@@ -3808,7 +3808,8 @@ public class TaskTracker implements MRCo
public static class MapOutputServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
private static final int MAX_BYTES_TO_READ = 64 * 1024;
-
+ // work around jetty internal buffering issues
+ private static final int RESPONSE_BUFFER_SIZE = MAX_BYTES_TO_READ + 16;
private static LRUCache<String, Path> fileCache = new LRUCache<String, Path>(FILE_CACHE_SIZE);
private static LRUCache<String, Path> fileIndexCache = new LRUCache<String, Path>(FILE_CACHE_SIZE);
@@ -3910,10 +3911,9 @@ public class TaskTracker implements MRCo
//set the custom "for-reduce-task" http header to the reduce task number
//for which this map output is being transferred
response.setHeader(FOR_REDUCE_TASK, Integer.toString(reduce));
-
- //use the same buffersize as used for reading the data from disk
- response.setBufferSize(MAX_BYTES_TO_READ);
-
+
+ response.setBufferSize(RESPONSE_BUFFER_SIZE);
+
/**
* Read the data from the sigle map-output file and
* send it to the reducer.
Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java?rev=1400532&r1=1400531&r2=1400532&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java (original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java Sat Oct 20 23:39:16 2012
@@ -64,9 +64,11 @@ public class TestStorageDirectoryFailure
String baseDir = System.getProperty("test.build.data", "/tmp");
File dfsDir = new File(baseDir, "dfs");
nameDirs = new ArrayList<String>();
- nameDirs.add(new File(dfsDir, "name1").getPath());
- nameDirs.add(new File(dfsDir, "name2").getPath());
- nameDirs.add(new File(dfsDir, "name3").getPath());
+ // Have all the name dirs with the same filename: important for regression
+ // testing HDFS-3652.
+ nameDirs.add(new File(new File(dfsDir, "name1"), "nn").getPath());
+ nameDirs.add(new File(new File(dfsDir, "name2"), "nn").getPath());
+ nameDirs.add(new File(new File(dfsDir, "name3"), "nn").getPath());
conf.set("dfs.name.dir", StringUtils.join(nameDirs, ","));
conf.set("dfs.data.dir", new File(dfsDir, "data").getPath());