You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2011/04/26 05:29:59 UTC
svn commit: r1096648 - in /hadoop/hdfs/branches/branch-0.22: ./
src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/
src/java/org/apache/hadoop/hdfs/
src/java/org/apache/hadoop/hdfs/server/datanode/ src/test/hdfs/
src/test/hdfs/org/apache/hadoop/hdfs/ src...
Author: eli
Date: Tue Apr 26 03:29:58 2011
New Revision: 1096648
URL: http://svn.apache.org/viewvc?rev=1096648&view=rev
Log:
HDFS-1861. svn merge -c 1096647 from trunk
Modified:
hadoop/hdfs/branches/branch-0.22/ (props changed)
hadoop/hdfs/branches/branch-0.22/CHANGES.txt
hadoop/hdfs/branches/branch-0.22/build.xml (props changed)
hadoop/hdfs/branches/branch-0.22/src/c++/libhdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/contrib/hdfsproxy/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (props changed)
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java
hadoop/hdfs/branches/branch-0.22/src/webapps/datanode/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/secondary/ (props changed)
Propchange: hadoop/hdfs/branches/branch-0.22/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 03:29:58 2011
@@ -2,4 +2,4 @@
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-265:796829-820463
/hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645
+/hadoop/hdfs/trunk:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645,1096647
Modified: hadoop/hdfs/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/CHANGES.txt?rev=1096648&r1=1096647&r2=1096648&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.22/CHANGES.txt Tue Apr 26 03:29:58 2011
@@ -222,6 +222,9 @@ Release 0.22.0 - Unreleased
HDFS-1582. Remove auto-generated native build files. (rvs via eli)
+ HDFS-1861. Rename dfs.datanode.max.xcievers and bump its default value.
+ (eli)
+
OPTIMIZATIONS
HDFS-1140. Speedup INode.getPathComponents. (Dmytro Molkov via shv)
Propchange: hadoop/hdfs/branches/branch-0.22/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 03:29:58 2011
@@ -3,4 +3,4 @@
/hadoop/core/trunk/build.xml:779102
/hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
/hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645
+/hadoop/hdfs/trunk/build.xml:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645,1096647
Propchange: hadoop/hdfs/branches/branch-0.22/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 03:29:58 2011
@@ -1,4 +1,4 @@
/hadoop/common/trunk/src/c++/libhdfs:1096645
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/trunk/src/c++/libhdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645
+/hadoop/hdfs/trunk/src/c++/libhdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645,1096647
Propchange: hadoop/hdfs/branches/branch-0.22/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 03:29:58 2011
@@ -3,4 +3,4 @@
/hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645,1096647
Propchange: hadoop/hdfs/branches/branch-0.22/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 03:29:58 2011
@@ -3,4 +3,4 @@
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645
+/hadoop/hdfs/trunk/src/java:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645,1096647
Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1096648&r1=1096647&r2=1096648&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Apr 26 03:29:58 2011
@@ -153,8 +153,8 @@ public class DFSConfigKeys extends Commo
public static final int DFS_DATANODE_HANDLER_COUNT_DEFAULT = 3;
public static final String DFS_DATANODE_HTTP_ADDRESS_KEY = "dfs.datanode.http.address";
public static final String DFS_DATANODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:50075";
- public static final String DFS_DATANODE_MAX_XCIEVERS_KEY = "dfs.datanode.max.xcievers";
- public static final int DFS_DATANODE_MAX_XCIEVERS_DEFAULT = 256;
+ public static final String DFS_DATANODE_MAX_RECEIVER_THREADS_KEY = "dfs.datanode.max.transfer.threads";
+ public static final int DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT = 4096;
public static final String DFS_DATANODE_NUMBLOCKS_KEY = "dfs.datanode.numblocks";
public static final int DFS_DATANODE_NUMBLOCKS_DEFAULT = 64;
public static final String DFS_DATANODE_SCAN_PERIOD_HOURS_KEY = "dfs.datanode.scan.period.hours";
Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/HdfsConfiguration.java?rev=1096648&r1=1096647&r2=1096648&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/HdfsConfiguration.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/HdfsConfiguration.java Tue Apr 26 03:29:58 2011
@@ -84,5 +84,6 @@ public class HdfsConfiguration extends C
deprecate("dfs.permissions.supergroup", DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY);
deprecate("dfs.write.packet.size", DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY);
deprecate("dfs.block.size", DFSConfigKeys.DFS_BLOCK_SIZE_KEY);
+ deprecate("dfs.datanode.max.xcievers", DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY);
}
}
Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java?rev=1096648&r1=1096647&r2=1096648&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java Tue Apr 26 03:29:58 2011
@@ -56,8 +56,8 @@ class DataXceiverServer implements Runna
* Enforcing the limit is required in order to avoid data-node
* running out of memory.
*/
- static final int MAX_XCEIVER_COUNT = 256;
- int maxXceiverCount = MAX_XCEIVER_COUNT;
+ int maxXceiverCount =
+ DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT;
/** A manager to make sure that cluster balancing does not
* take too much resources.
@@ -115,9 +115,10 @@ class DataXceiverServer implements Runna
this.ss = ss;
this.datanode = datanode;
- this.maxXceiverCount = conf.getInt("dfs.datanode.max.xcievers",
- MAX_XCEIVER_COUNT);
-
+ this.maxXceiverCount =
+ conf.getInt(DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY,
+ DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT);
+
this.estimateBlockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
//set up parameter for cluster balancing
Propchange: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 03:29:58 2011
@@ -5,4 +5,4 @@
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645,1096647
Propchange: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 03:29:58 2011
@@ -3,4 +3,4 @@
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645
+/hadoop/hdfs/trunk/src/test/hdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645,1096647
Modified: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1096648&r1=1096647&r2=1096648&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java Tue Apr 26 03:29:58 2011
@@ -380,7 +380,7 @@ public class TestDFSClientRetries extend
int bufferSize = 4096;
Configuration conf = new HdfsConfiguration();
- conf.setInt("dfs.datanode.max.xcievers",xcievers);
+ conf.setInt(DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, xcievers);
conf.setInt("dfs.client.max.block.acquire.failures", retries);
conf.setInt("dfs.client.retry.window.base", timeWin);
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 03:29:58 2011
@@ -3,4 +3,4 @@
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645
+/hadoop/hdfs/trunk/src/webapps/datanode:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645,1096647
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 03:29:58 2011
@@ -3,4 +3,4 @@
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645
+/hadoop/hdfs/trunk/src/webapps/hdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645,1096647
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 03:29:58 2011
@@ -3,4 +3,4 @@
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645
+/hadoop/hdfs/trunk/src/webapps/secondary:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308,1043314,1044102,1044218,1050109,1050125,1051134,1051334,1051603,1052169,1053203,1054807,1054858,1059508,1065860,1068725,1070025,1071023,1095235,1095245,1095830,1096645,1096647