You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by at...@apache.org on 2011/11/30 19:27:19 UTC
svn commit: r1208644 - in
/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common:
CHANGES.txt src/main/docs/ src/main/java/
src/main/packages/hadoop-setup-conf.sh
src/main/packages/templates/conf/hdfs-site.xml src/test/core/
Author: atm
Date: Wed Nov 30 18:27:04 2011
New Revision: 1208644
URL: http://svn.apache.org/viewvc?rev=1208644&view=rev
Log:
Merge trunk into HA branch.
Modified:
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1208644&r1=1208643&r2=1208644&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt Wed Nov 30 18:27:04 2011
@@ -126,6 +126,9 @@ Release 0.23.1 - Unreleased
HADOOP-7424. Log an error if the topology script doesn't handle multiple args.
(Uma Maheswara Rao G via eli)
+ HADOOP-7804. Enable hadoop config generator to set configurations to enable
+ short circuit read. (Arpit Gupta via jitendra)
+
OPTIMIZATIONS
BUG FIXES
@@ -141,6 +144,9 @@ Release 0.23.1 - Unreleased
HADOOP-7859. TestViewFsHdfs.testgetFileLinkStatus is failing an assert. (eli)
+ HADOOP-7864. Building mvn site with Maven < 3.0.2 causes OOM errors.
+ (Andrew Bayer via eli)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 30 18:27:04 2011
@@ -1,5 +1,5 @@
/hadoop/common/branches/yahoo-merge/CHANGES.txt:1079157,1079163-1079164,1079167
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1208001
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1208622
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 30 18:27:04 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1208001
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1208622
/hadoop/core/branches/branch-0.19/src/docs:713112
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 30 18:27:04 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1208001
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1208622
/hadoop/core/branches/branch-0.19/core/src/java:713112
/hadoop/core/trunk/src/core:776175-785643,785929-786278
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh?rev=1208644&r1=1208643&r2=1208644&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh Wed Nov 30 18:27:04 2011
@@ -67,6 +67,10 @@ usage: $0 <parameters>
This value should be <= mapred.cluster.max.map.memory.mb
--mapreduce-reduce-memory-mb=memory Virtual memory, of a single reduce slot for a job. Defaults to -1
This value should be <= mapred.cluster.max.reduce.memory.mb
+ --dfs-datanode-dir-perm=700 Set the permission for the datanode data directories. Defaults to 700
+ --dfs-block-local-path-access-user=user User for which you want to enable shortcircuit read.
+ --dfs-client-read-shortcircuit=true/false Enable shortcircuit read for the client. Will default to true if the shortcircuit user is set.
+ --dfs-client-read-shortcircuit-skip-checksum=false/true Disable checking of checksum when shortcircuit read is taking place. Defaults to false.
"
exit 1
}
@@ -124,7 +128,7 @@ function addPropertyToXMLConf
local finalVal=$5
#create the property text, make sure the / are escaped
- propText="<property>\n<name>$property<\/name>\n<value>$propValue<\/value>"
+ propText="<property>\n<name>$property<\/name>\n<value>$propValue<\/value>\n"
#if description is not empty add it
if [ ! -z $desc ]
then
@@ -147,6 +151,28 @@ function addPropertyToXMLConf
}
##########################################
+# Function to setup up the short circuit read settings
+#########################################
+function setupShortCircuitRead
+{
+ local conf_file="${HADOOP_CONF_DIR}/hdfs-site.xml"
+ #if the shortcircuit user is not set then return
+ if [ -z $DFS_BLOCK_LOCAL_PATH_ACCESS_USER ]
+ then
+ return
+ fi
+
+ #set the defaults if values not present
+ DFS_CLIENT_READ_SHORTCIRCUIT=${DFS_CLIENT_READ_SHORTCIRCUIT:-false}
+ DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM=${DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM:-false}
+
+ #add the user to the conf file
+ addPropertyToXMLConf "$conf_file" "dfs.block.local-path-access.user" "$DFS_BLOCK_LOCAL_PATH_ACCESS_USER"
+ addPropertyToXMLConf "$conf_file" "dfs.client.read.shortcircuit" "$DFS_CLIENT_READ_SHORTCIRCUIT"
+ addPropertyToXMLConf "$conf_file" "dfs.client.read.shortcircuit.skip.checksum" "$DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM"
+}
+
+##########################################
# Function to setup up the proxy user settings
#########################################
function setupProxyUsers
@@ -217,6 +243,10 @@ OPTS=$(getopt \
-l 'mapreduce-jobtracker-maxreducememory-mb:' \
-l 'mapreduce-map-memory-mb:' \
-l 'mapreduce-reduce-memory-mb:' \
+ -l 'dfs-datanode-dir-perm:' \
+ -l 'dfs-block-local-path-access-user:' \
+ -l 'dfs-client-read-shortcircuit:' \
+ -l 'dfs-client-read-shortcircuit-skip-checksum:' \
-o 'h' \
-- "$@")
@@ -376,6 +406,22 @@ while true ; do
MAPREDUCE_REDUCE_MEMORY_MB=$2; shift 2
AUTOMATED=1
;;
+ --dfs-datanode-dir-perm)
+ DFS_DATANODE_DIR_PERM=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --dfs-block-local-path-access-user)
+ DFS_BLOCK_LOCAL_PATH_ACCESS_USER=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --dfs-client-read-shortcircuit)
+ DFS_CLIENT_READ_SHORTCIRCUIT=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --dfs-client-read-shortcircuit-skip-checksum)
+ DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM=$2; shift 2
+ AUTOMATED=1
+ ;;
--)
shift ; break
;;
@@ -421,6 +467,8 @@ DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:
KERBEROS_REALM=${KERBEROS_REALM:-KERBEROS.EXAMPLE.COM}
SECURITY_TYPE=${SECURITY_TYPE:-simple}
KINIT=${KINIT:-/usr/kerberos/bin/kinit}
+#deault the data dir perm to 700
+DFS_DATANODE_DIR_PERM=${DFS_DATANODE_DIR_PERM:-700}
if [ "${SECURITY_TYPE}" = "kerberos" ]; then
TASK_CONTROLLER="org.apache.hadoop.mapred.LinuxTaskController"
HADOOP_DN_ADDR="0.0.0.0:1019"
@@ -561,7 +609,10 @@ if [ "${AUTOSETUP}" == "1" -o "${AUTOSET
#setup up the proxy users
setupProxyUsers
-
+
+ #setup short circuit read
+ setupShortCircuitRead
+
#set the owner of the hadoop dir to root
chown root ${HADOOP_PREFIX}
chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
@@ -611,6 +662,9 @@ else
#setup up the proxy users
setupProxyUsers
+ #setup short circuit read
+ setupShortCircuitRead
+
chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
#set taskcontroller
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml?rev=1208644&r1=1208643&r2=1208644&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml Wed Nov 30 18:27:04 2011
@@ -202,7 +202,7 @@
<property>
<name>dfs.datanode.data.dir.perm</name>
- <value>700</value>
+ <value>${DFS_DATANODE_DIR_PERM}</value>
<description>The permissions that should be there on dfs.data.dir
directories. The datanode will not come up if the permissions are
different on existing dfs.data.dir directories. If the directories
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 30 18:27:04 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1208001
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1208622
/hadoop/core/branches/branch-0.19/core/src/test/core:713112
/hadoop/core/trunk/src/test/core:776175-785643,785929-786278