You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/02/13 03:01:35 UTC
hadoop git commit: HADOOP-11565. Add --slaves shell option (aw)
Repository: hadoop
Updated Branches:
refs/heads/trunk 6a49e58cb -> 93b941c63
HADOOP-11565. Add --slaves shell option (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93b941c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93b941c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93b941c6
Branch: refs/heads/trunk
Commit: 93b941c637e814f08466383055ec333e41e83ec4
Parents: 6a49e58
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Feb 12 18:01:28 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Feb 12 18:01:28 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../hadoop-common/src/main/bin/hadoop | 5 +++++
.../hadoop-common/src/main/bin/hadoop-config.sh | 9 +++++++--
.../hadoop-common/src/main/bin/hadoop-daemons.sh | 16 ++++++++++++++--
.../src/main/bin/hadoop-functions.sh | 14 ++++++++++++++
.../hadoop-hdfs/src/main/bin/hdfs | 5 +++++
hadoop-mapreduce-project/bin/mapred | 5 +++++
hadoop-yarn-project/hadoop-yarn/bin/yarn | 5 +++++
.../hadoop-yarn/bin/yarn-daemons.sh | 19 +++++++++++++++++--
9 files changed, 74 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93b941c6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c11e340..0d8c02f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -29,6 +29,8 @@ Trunk (Unreleased)
HADOOP-11554. Expose HadoopKerberosName as a hadoop subcommand (aw)
+ HADOOP-11565. Add --slaves shell option (aw)
+
IMPROVEMENTS
HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93b941c6/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 42e4e83..6003927 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -179,6 +179,11 @@ esac
hadoop_verify_user "${COMMAND}"
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+ hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
+ exit $?
+fi
+
# Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93b941c6/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
index f0cabbe..58b871e 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
@@ -1,4 +1,4 @@
-
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -91,7 +91,7 @@ hadoop_bootstrap
# save these off in case our caller needs them
# shellcheck disable=SC2034
-HADOOP_USER_PARAMS="$@"
+HADOOP_USER_PARAMS=("$@")
HADOOP_DAEMON_MODE="default"
@@ -152,6 +152,11 @@ while [[ -z "${_hadoop_common_done}" ]]; do
HADOOP_LOGLEVEL="$1"
shift
;;
+ --slaves)
+ shift
+ # shellcheck disable=SC2034
+ HADOOP_SLAVE_MODE=true
+ ;;
*)
_hadoop_common_done=true
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93b941c6/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
index 20d7e4a..9e4e6b0 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
@@ -56,5 +56,17 @@ else
hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs"
fi
-hadoop_connect_to_hosts "$hdfsscript" \
- --config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@"
+hadoop_error "WARNING: Use of this script to ${daemonmode} HDFS daemons is deprecated."
+hadoop_error "WARNING: Attempting to execute replacement \"hdfs --slaves --daemon ${daemonmode}\" instead."
+
+#
+# Original input was usually:
+# hadoop-daemons.sh (shell options) (start|stop) (datanode|...) (daemon options)
+# we're going to turn this into
+# hdfs --slaves --daemon (start|stop) (rest of options)
+#
+argv=(${HADOOP_USER_PARAMS[@]/start})
+argv=(${argv[@]/stop})
+argv=(${argv[@]/status})
+
+${hdfsscript} --slaves --daemon "${daemonmode}" "${argv[@]}"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93b941c6/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 142ac22..cec6b2c 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -405,6 +405,20 @@ function hadoop_connect_to_hosts
fi
}
+function hadoop_common_slave_mode_execute
+{
+ #
+ # input should be the command line as given by the user
+ # in the form of an array
+ #
+ local argv=("$@")
+
+ # if --slaves is still on the command line, remove it
+ # to prevent loops
+ argv=(${argv[@]/--slaves})
+ hadoop_connect_to_hosts -- "${argv[@]}"
+}
+
function hadoop_validate_classname
{
local class=$1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93b941c6/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 5f8ff18..830ca36 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -247,6 +247,11 @@ esac
hadoop_verify_user "${COMMAND}"
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+ hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
+ exit $?
+fi
+
if [[ -n "${secure_service}" ]]; then
HADOOP_SECURE_USER="${secure_user}"
hadoop_verify_secure_prereq
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93b941c6/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index 6d0c781..d199128 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -135,6 +135,11 @@ esac
hadoop_verify_user "${COMMAND}"
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+ hadoop_common_slave_mode_execute "${HADOOP_MAPRED_HOME}/bin/mapred" "${HADOOP_USER_PARAMS[@]}"
+ exit $?
+fi
+
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93b941c6/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index b3fe1f3..f1a06a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -186,6 +186,11 @@ esac
hadoop_verify_user "${COMMAND}"
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+ hadoop_common_slave_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}"
+ exit $?
+fi
+
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93b941c6/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
index 17d3db6..c6963d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
@@ -41,8 +41,23 @@ else
exit 1
fi
+yarnscript="${HADOOP_YARN_HOME}/bin/yarn"
+
daemonmode=$1
shift
-hadoop_connect_to_hosts "${HADOOP_YARN_HOME}/bin/yarn" \
- --config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@"
+hadoop_error "WARNING: Use of this script to ${daemonmode} YARN daemons is deprecated."
+hadoop_error "WARNING: Attempting to execute replacement \"yarn --slaves --daemon ${daemonmode}\" instead."
+
+#
+# Original input was usually:
+# yarn-daemons.sh (shell options) (start|stop) nodemanager (daemon options)
+# we're going to turn this into
+# yarn --slaves --daemon (start|stop) (rest of options)
+#
+argv=(${HADOOP_USER_PARAMS[@]/start})
+argv=(${argv[@]/stop})
+argv=(${argv[@]/status})
+
+${yarnscript} --slaves --daemon "${daemonmode}" "${argv[@]}"
+