You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/05/06 21:11:28 UTC
[10/16] hadoop git commit: HADOOP-12936. modify hadoop-tools to take
advantage of dynamic subcommands
HADOOP-12936. modify hadoop-tools to take advantage of dynamic subcommands
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b7c1d88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b7c1d88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b7c1d88
Branch: refs/heads/HADOOP-12930
Commit: 1b7c1d887d966aae0b611b81ec56718cdf8872b2
Parents: 4247c06
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed May 4 07:21:36 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri May 6 14:10:19 2016 -0700
----------------------------------------------------------------------
.../main/resources/assemblies/hadoop-tools.xml | 32 ++++++++++++++
.../hadoop-common/src/main/bin/hadoop | 45 +++++++-------------
hadoop-mapreduce-project/bin/mapred | 21 ---------
.../main/shellprofile.d/hadoop-archive-logs.sh | 33 ++++++++++++++
.../src/main/shellprofile.d/hadoop-archives.sh | 41 ++++++++++++++++++
.../src/main/shellprofile.d/hadoop-distcp.sh | 43 +++++++++++++++++++
.../src/main/shellprofile.d/hadoop-extras.sh | 33 ++++++++++++++
7 files changed, 197 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index f8ba48e..c5ea6ad 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -24,6 +24,38 @@
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
+ <directory>../hadoop-archive-logs/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../hadoop-archives/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../hadoop-distcp/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../hadoop-extras/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
<directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory>
<includes>
<include>*.hh</include>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 453ac83..ce695cb 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -30,14 +30,11 @@ function hadoop_usage
hadoop_add_option "hosts filename" "list of hosts to use in slave mode"
hadoop_add_option "slaves" "turn on slave mode"
- hadoop_add_subcommand "archive" "create a Hadoop archive"
hadoop_add_subcommand "checknative" "check native Hadoop and compression libraries availability"
hadoop_add_subcommand "classpath" "prints the class path needed to get the Hadoop jar and the required libraries"
hadoop_add_subcommand "conftest" "validate configuration XML files"
hadoop_add_subcommand "credential" "interact with credential providers"
hadoop_add_subcommand "daemonlog" "get/set the log level for each daemon"
- hadoop_add_subcommand "distch" "distributed metadata changer"
- hadoop_add_subcommand "distcp" "copy file or directories recursively"
hadoop_add_subcommand "dtutil" "operations related to delegation tokens"
hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
hadoop_add_subcommand "fs" "run a generic filesystem user client"
@@ -102,35 +99,23 @@ function hadoopcmd_case
exit 1
fi
;;
- archive)
- HADOOP_CLASS=org.apache.hadoop.tools.HadoopArchives
- hadoop_add_to_classpath_tools hadoop-archives
- ;;
checknative)
- HADOOP_CLASS=org.apache.hadoop.util.NativeLibraryChecker
+ HADOOP_CLASSNAME=org.apache.hadoop.util.NativeLibraryChecker
;;
classpath)
- hadoop_do_classpath_subcommand HADOOP_CLASS "$@"
+ hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
;;
conftest)
- HADOOP_CLASS=org.apache.hadoop.util.ConfTest
+ HADOOP_CLASSNAME=org.apache.hadoop.util.ConfTest
;;
credential)
- HADOOP_CLASS=org.apache.hadoop.security.alias.CredentialShell
+ HADOOP_CLASSNAME=org.apache.hadoop.security.alias.CredentialShell
;;
daemonlog)
- HADOOP_CLASS=org.apache.hadoop.log.LogLevel
- ;;
- distch)
- HADOOP_CLASS=org.apache.hadoop.tools.DistCh
- hadoop_add_to_classpath_tools hadoop-extras
- ;;
- distcp)
- HADOOP_CLASS=org.apache.hadoop.tools.DistCp
- hadoop_add_to_classpath_tools hadoop-distcp
+ HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel
;;
dtutil)
- HADOOP_CLASS=org.apache.hadoop.security.token.DtUtilShell
+ HADOOP_CLASSNAME=org.apache.hadoop.security.token.DtUtilShell
;;
envvars)
echo "JAVA_HOME='${JAVA_HOME}'"
@@ -145,7 +130,7 @@ function hadoopcmd_case
exit 0
;;
fs)
- HADOOP_CLASS=org.apache.hadoop.fs.FsShell
+ HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
;;
gridmix)
CLASS=org.apache.hadoop.mapred.gridmix.Gridmix
@@ -156,7 +141,7 @@ function hadoopcmd_case
if [[ -n "${YARN_OPTS}" ]] || [[ -n "${YARN_CLIENT_OPTS}" ]]; then
hadoop_error "WARNING: Use \"yarn jar\" to launch YARN applications."
fi
- HADOOP_CLASS=org.apache.hadoop.util.RunJar
+ HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
;;
jnipath)
hadoop_finalize
@@ -164,20 +149,20 @@ function hadoopcmd_case
exit 0
;;
kerbname)
- HADOOP_CLASS=org.apache.hadoop.security.HadoopKerberosName
+ HADOOP_CLASSNAME=org.apache.hadoop.security.HadoopKerberosName
;;
key)
- HADOOP_CLASS=org.apache.hadoop.crypto.key.KeyShell
+ HADOOP_CLASSNAME=org.apache.hadoop.crypto.key.KeyShell
;;
trace)
- HADOOP_CLASS=org.apache.hadoop.tracing.TraceAdmin
+ HADOOP_CLASSNAME=org.apache.hadoop.tracing.TraceAdmin
;;
version)
- HADOOP_CLASS=org.apache.hadoop.util.VersionInfo
+ HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
;;
*)
- HADOOP_CLASS="${subcmd}"
- if ! hadoop_validate_classname "${HADOOP_CLASS}"; then
+ HADOOP_CLASSNAME="${subcmd}"
+ if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
hadoop_exit_with_usage 1
fi
;;
@@ -229,5 +214,5 @@ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
hadoop_finalize
-hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASS}" "$@"
+hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index 69b00b3..84f03e4 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -24,10 +24,7 @@ HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
## @replaceable no
function hadoop_usage
{
- hadoop_add_subcommand "archive" "create a hadoop archive"
- hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives"
hadoop_add_subcommand "classpath" "prints the class path needed for running mapreduce subcommands"
- hadoop_add_subcommand "distcp" "copy file or directories recursively"
hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
hadoop_add_subcommand "historyserver" "run job history servers as a standalone daemon"
hadoop_add_subcommand "hsadmin" "job history server admin interface"
@@ -55,27 +52,9 @@ function mapredcmd_case
hadoop_error "You may find similar functionality with the \"yarn\" shell command."
hadoop_exit_with_usage 1
;;
- archive)
- HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchives
- hadoop_add_to_classpath_tools hadoop-archives
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- archive-logs)
- HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs
- hadoop_add_to_classpath_tools hadoop-archive-logs
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
classpath)
hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
;;
- distcp)
- HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
- hadoop_add_to_classpath_tools hadoop-distcp
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
envvars)
echo "JAVA_HOME='${JAVA_HOME}'"
echo "HADOOP_MAPRED_HOME='${HADOOP_MAPRED_HOME}'"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh b/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
new file mode 100755
index 0000000..d37411e
--- /dev/null
+++ b/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+ hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives"
+fi
+
+## @description archive-logs command for mapred
+## @audience public
+## @stability stable
+## @replaceable no
+function mapred_subcommand_archive-logs
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs
+ hadoop_add_to_classpath_tools hadoop-archive-logs
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh b/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh
new file mode 100755
index 0000000..b85ff25
--- /dev/null
+++ b/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop
+ || "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+ hadoop_add_subcommand "archive" "create a Hadoop archive"
+fi
+
+## @description archive command for hadoop (and mapred)
+## @audience public
+## @stability stable
+## @replaceable no
+function hadoop_subcommand_archive
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchives
+ hadoop_add_to_classpath_tools hadoop-archives
+}
+
+## @description archive-logs command for mapred (calls hadoop version)
+## @audience public
+## @stability stable
+## @replaceable no
+function mapred_subcommand_archive
+{
+ hadoop_subcommand_archive
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh b/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
new file mode 100755
index 0000000..6cb13fc
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop
+ || "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+ hadoop_add_subcommand "distcp" "copy file or directories recursively"
+fi
+
+## @description distcp command for hadoop
+## @audience public
+## @stability stable
+## @replaceable no
+function hadoop_subcommand_distcp
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
+ hadoop_add_to_classpath_tools hadoop-distcp
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+## @description distcp command for mapred (calls hadoop)
+## @audience public
+## @stability stable
+## @replaceable no
+function mapred_subcommand_distcp
+{
+ hadoop_subcommand_distcp
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh b/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
new file mode 100755
index 0000000..e599334
--- /dev/null
+++ b/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+ hadoop_add_subcommand "distch" "distributed metadata changer"
+fi
+
+## @description distch command for hadoop
+## @audience public
+## @stability stable
+## @replaceable no
+function hadoop_subcommand_distch
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCh
+ hadoop_add_to_classpath_tools hadoop-extras
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org