You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/03/26 23:10:03 UTC
hadoop git commit: HADOOP-11553. Foramlize the shell API (aw)
Repository: hadoop
Updated Branches:
refs/heads/trunk 61df1b27a -> b30ca8ce0
HADOOP-11553. Foramlize the shell API (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b30ca8ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b30ca8ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b30ca8ce
Branch: refs/heads/trunk
Commit: b30ca8ce0e0d435327e179f0877bd58fa3896793
Parents: 61df1b2
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Mar 26 15:09:51 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Mar 26 15:09:51 2015 -0700
----------------------------------------------------------------------
dev-support/shelldocs.py | 250 +++++++++++
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
hadoop-common-project/hadoop-common/pom.xml | 57 ++-
.../src/main/bin/hadoop-functions.sh | 411 ++++++++++++++++---
.../src/site/markdown/UnixShellGuide.md | 98 +++++
hadoop-project/src/site/site.xml | 4 +-
6 files changed, 765 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30ca8ce/dev-support/shelldocs.py
----------------------------------------------------------------------
diff --git a/dev-support/shelldocs.py b/dev-support/shelldocs.py
new file mode 100755
index 0000000..2547450
--- /dev/null
+++ b/dev-support/shelldocs.py
@@ -0,0 +1,250 @@
+#!/usr/bin/python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import sys
+import string
+from optparse import OptionParser
+
+def docstrip(key,string):
+ string=re.sub("^## @%s " % key ,"",string)
+ string=string.lstrip()
+ string=string.rstrip()
+ return string
+
+def toc(list):
+ tocout=[]
+ header=()
+ for i in list:
+ if header != i.getinter():
+ header=i.getinter()
+ line=" * %s\n" % (i.headerbuild())
+ tocout.append(line)
+ line=" * [%s](#%s)\n" % (i.getname().replace("_","\_"),i.getname())
+ tocout.append(line)
+ return tocout
+
+class ShellFunction:
+ def __init__(self):
+ self.reset()
+
+ def __cmp__(self,other):
+ if (self.audience == other.audience):
+ if (self.stability == other.stability):
+ if (self.replaceb == other.replaceb):
+ return(cmp(self.name,other.name))
+ else:
+ if (self.replaceb == "Yes"):
+ return -1
+ else:
+ return 1
+ else:
+ if (self.stability == "Stable"):
+ return -1
+ else:
+ return 1
+ else:
+ if (self.audience == "Public"):
+ return -1
+ else:
+ return 1
+
+ def reset(self):
+ self.name=None
+ self.audience=None
+ self.stability=None
+ self.replaceb=None
+ self.returnt=None
+ self.desc=None
+ self.params=None
+
+ def setname(self,text):
+ definition=text.split();
+ self.name=definition[1]
+
+ def getname(self):
+ if (self.name is None):
+ return "None"
+ else:
+ return self.name
+
+ def setaudience(self,text):
+ self.audience=docstrip("audience",text)
+ self.audience=self.audience.capitalize()
+
+ def getaudience(self):
+ if (self.audience is None):
+ return "None"
+ else:
+ return self.audience
+
+ def setstability(self,text):
+ self.stability=docstrip("stability",text)
+ self.stability=self.stability.capitalize()
+
+ def getstability(self):
+ if (self.stability is None):
+ return "None"
+ else:
+ return self.stability
+
+ def setreplace(self,text):
+ self.replaceb=docstrip("replaceable",text)
+ self.replaceb=self.replaceb.capitalize()
+
+ def getreplace(self):
+ if (self.replaceb is None):
+ return "None"
+ else:
+ return self.replaceb
+
+ def getinter(self):
+ return( (self.getaudience(), self.getstability(), self.getreplace()))
+
+ def addreturn(self,text):
+ if (self.returnt is None):
+ self.returnt = []
+ self.returnt.append(docstrip("return",text))
+
+ def getreturn(self):
+ if (self.returnt is None):
+ return "Nothing"
+ else:
+ return "\n\n".join(self.returnt)
+
+ def adddesc(self,text):
+ if (self.desc is None):
+ self.desc = []
+ self.desc.append(docstrip("description",text))
+
+ def getdesc(self):
+ if (self.desc is None):
+ return "None"
+ else:
+ return " ".join(self.desc)
+
+ def addparam(self,text):
+ if (self.params is None):
+ self.params = []
+ self.params.append(docstrip("param",text))
+
+ def getparams(self):
+ if (self.params is None):
+ return ""
+ else:
+ return " ".join(self.params)
+
+ def getusage(self):
+ line="%s %s" % (self.name, self.getparams())
+ return line
+
+ def headerbuild(self):
+ if self.getreplace() == "Yes":
+ replacetext="Replaceable"
+ else:
+ replacetext="Not Replaceable"
+ line="%s/%s/%s" % (self.getaudience(), self.getstability(), replacetext)
+ return(line)
+
+ def getdocpage(self):
+ line="### `%s`\n\n"\
+ "* Synopsis\n\n"\
+ "```\n%s\n"\
+ "```\n\n" \
+ "* Description\n\n" \
+ "%s\n\n" \
+ "* Returns\n\n" \
+ "%s\n\n" \
+ "| Classification | Level |\n" \
+ "| :--- | :--- |\n" \
+ "| Audience | %s |\n" \
+ "| Stability | %s |\n" \
+ "| Replaceable | %s |\n\n" \
+ % (self.getname(),
+ self.getusage(),
+ self.getdesc(),
+ self.getreturn(),
+ self.getaudience(),
+ self.getstability(),
+ self.getreplace())
+ return line
+
+ def __str__(self):
+ line="{%s %s %s %s}" \
+ % (self.getname(),
+ self.getaudience(),
+ self.getstability(),
+ self.getreplace())
+ return line
+
+def main():
+ parser=OptionParser(usage="usage: %prog --skipprnorep --output OUTFILE --input INFILE [--input INFILE ...]")
+ parser.add_option("-o","--output", dest="outfile",
+ action="store", type="string",
+ help="file to create", metavar="OUTFILE")
+ parser.add_option("-i","--input", dest="infile",
+ action="append", type="string",
+ help="file to read", metavar="INFILE")
+ parser.add_option("--skipprnorep", dest="skipprnorep",
+ action="store_true", help="Skip Private & Not Replaceable")
+
+ (options, args)=parser.parse_args()
+
+ allfuncs=[]
+ for filename in options.infile:
+ with open(filename,"r") as shellcode:
+ funcdef=ShellFunction()
+ for line in shellcode:
+ if line.startswith('## @description'):
+ funcdef.adddesc(line)
+ elif line.startswith('## @audience'):
+ funcdef.setaudience(line)
+ elif line.startswith('## @stability'):
+ funcdef.setstability(line)
+ elif line.startswith('## @replaceable'):
+ funcdef.setreplace(line)
+ elif line.startswith('## @param'):
+ funcdef.addparam(line)
+ elif line.startswith('## @return'):
+ funcdef.addreturn(line)
+ elif line.startswith('function'):
+ funcdef.setname(line)
+ if options.skipprnorep:
+ if funcdef.getaudience() == "Private" and \
+ funcdef.getreplace() == "No":
+ pass
+ else:
+ allfuncs.append(funcdef)
+ funcdef=ShellFunction()
+
+ allfuncs=sorted(allfuncs)
+
+ outfile=open(options.outfile, "w")
+ for line in toc(allfuncs):
+ outfile.write(line)
+
+ outfile.write("\n------\n\n")
+
+ header=[]
+ for funcs in allfuncs:
+ if header != funcs.getinter():
+ header=funcs.getinter()
+ line="## %s\n" % (funcs.headerbuild())
+ outfile.write(line)
+ outfile.write(funcs.getdocpage())
+ outfile.close()
+
+if __name__ == "__main__":
+ main()
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30ca8ce/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 5f43236..dbe9e55 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -19,6 +19,8 @@ Trunk (Unreleased)
HADOOP-11657. Align the output of `hadoop fs -du` to be more Unix-like.
(aajisaka)
+ HADOOP-11553. Foramlize the shell API (aw)
+
NEW FEATURES
HADOOP-6590. Add a username check for hadoop sub-commands (John Smith via aw)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30ca8ce/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 3ae78f4..706f5b5 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -471,10 +471,10 @@
<goal>run</goal>
</goals>
<configuration>
- <tasks>
+ <target>
<copy file="src/main/resources/core-default.xml" todir="src/site/resources"/>
<copy file="src/main/xsl/configuration.xsl" todir="src/site/resources"/>
- </tasks>
+ </target>
</configuration>
</execution>
</executions>
@@ -509,6 +509,53 @@
</excludes>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shelldocs</id>
+ <phase>pre-site</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>python</executable>
+ <workingDirectory>src/site/markdown</workingDirectory>
+ <arguments>
+ <argument>${basedir}/../../dev-support/shelldocs.py</argument>
+ <argument>--skipprnorep</argument>
+ <argument>--output</argument>
+ <argument>${basedir}/src/site/markdown/UnixShellAPI.md</argument>
+ <argument>--input</argument>
+ <argument>${basedir}/src/main/bin/hadoop-functions.sh</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <configuration>
+ <filesets>
+ <fileset>
+ <directory>src/site/markdown</directory>
+ <includes>
+ <include>UnixShellAPI.md</include>
+ </includes>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>src/site/resources</directory>
+ <includes>
+ <include>configuration.xsl</include>
+ <include>core-default.xml</include>
+ </includes>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
</plugins>
</build>
@@ -550,7 +597,7 @@
<family>unix</family>
<message>native build only supported on Mac or Unix</message>
</requireOS>
- </rules>
+ </rules>
<fail>true</fail>
</configuration>
</execution>
@@ -670,7 +717,7 @@
<family>windows</family>
<message>native-win build only supported on Windows</message>
</requireOS>
- </rules>
+ </rules>
<fail>true</fail>
</configuration>
</execution>
@@ -786,7 +833,7 @@
<family>mac</family>
<family>unix</family>
</requireOS>
- </rules>
+ </rules>
<fail>true</fail>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30ca8ce/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 616e706..85f8200 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -14,13 +14,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+## @description Print a message to stderr
+## @audience public
+## @stability stable
+## @replaceable no
+## @param string
function hadoop_error
{
- # NOTE: This function is not user replaceable.
-
echo "$*" 1>&2
}
+## @description Print a message to stderr if --debug is turned on
+## @audience public
+## @stability stable
+## @replaceable no
+## @param string
function hadoop_debug
{
if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
@@ -28,10 +36,14 @@ function hadoop_debug
fi
}
+## @description Replace `oldvar` with `newvar` if `oldvar` exists.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param oldvar
+## @param newvar
function hadoop_deprecate_envvar
{
- #
- # Deprecate $1 with $2
local oldvar=$1
local newvar=$2
local oldval=${!oldvar}
@@ -50,10 +62,12 @@ function hadoop_deprecate_envvar
fi
}
+## @description Bootstraps the Hadoop shell environment
+## @audience private
+## @stability evolving
+## @replaceable no
function hadoop_bootstrap
{
- # NOTE: This function is not user replaceable.
-
# the root of the Hadoop installation
# See HADOOP-6255 for the expected directory structure layout
@@ -94,14 +108,14 @@ function hadoop_bootstrap
hadoop_debug "Initial HADOOP_OPTS=${HADOOP_OPTS}"
}
+## @description Locate Hadoop's configuration directory
+## @audience private
+## @stability evolving
+## @replaceable no
function hadoop_find_confdir
{
- # NOTE: This function is not user replaceable.
-
local conf_dir
- # Look for the basic hadoop configuration area.
- #
- #
+
# An attempt at compatibility with some Hadoop 1.x
# installs.
if [[ -e "${HADOOP_PREFIX}/conf/hadoop-env.sh" ]]; then
@@ -114,6 +128,11 @@ function hadoop_find_confdir
hadoop_debug "HADOOP_CONF_DIR=${HADOOP_CONF_DIR}"
}
+## @description Validate ${HADOOP_CONF_DIR}
+## @audience public
+## @stability stable
+## @replaceable yes
+## @return will exit on failure conditions
function hadoop_verify_confdir
{
# Check only log4j.properties by default.
@@ -123,10 +142,12 @@ function hadoop_verify_confdir
fi
}
+## @description Import the hadoop-env.sh settings
+## @audience private
+## @stability evolving
+## @replaceable no
function hadoop_exec_hadoopenv
{
- # NOTE: This function is not user replaceable.
-
if [[ -z "${HADOOP_ENV_PROCESSED}" ]]; then
if [[ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]]; then
export HADOOP_ENV_PROCESSED=true
@@ -135,26 +156,35 @@ function hadoop_exec_hadoopenv
fi
}
+## @description Import the replaced functions
+## @audience private
+## @stability evolving
+## @replaceable no
function hadoop_exec_userfuncs
{
- # NOTE: This function is not user replaceable.
-
if [[ -e "${HADOOP_CONF_DIR}/hadoop-user-functions.sh" ]]; then
. "${HADOOP_CONF_DIR}/hadoop-user-functions.sh"
fi
}
+## @description Read the user's settings. This provides for users to
+## @description override and/or append hadoop-env.sh. It is not meant
+## @description as a complete system override.
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_exec_hadooprc
{
- # Read the user's settings. This provides for users to override
- # and/or append hadoop-env.sh. It is not meant as a complete system override.
-
if [[ -f "${HOME}/.hadooprc" ]]; then
hadoop_debug "Applying the user's .hadooprc"
. "${HOME}/.hadooprc"
fi
}
+## @description Import shellprofile.d content
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_import_shellprofiles
{
local i
@@ -180,6 +210,10 @@ function hadoop_import_shellprofiles
done
}
+## @description Initialize the registered shell profiles
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_shellprofiles_init
{
local i
@@ -194,6 +228,10 @@ function hadoop_shellprofiles_init
done
}
+## @description Apply the shell profile classpath additions
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_shellprofiles_classpath
{
local i
@@ -208,6 +246,10 @@ function hadoop_shellprofiles_classpath
done
}
+## @description Apply the shell profile native library additions
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_shellprofiles_nativelib
{
local i
@@ -222,6 +264,10 @@ function hadoop_shellprofiles_nativelib
done
}
+## @description Apply the shell profile final configuration
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_shellprofiles_finalize
{
local i
@@ -236,6 +282,11 @@ function hadoop_shellprofiles_finalize
done
}
+## @description Initialize the Hadoop shell environment, now that
+## @description user settings have been imported
+## @audience private
+## @stability evolving
+## @replaceable no
function hadoop_basic_init
{
# Some of these are also set in hadoop-env.sh.
@@ -290,10 +341,15 @@ function hadoop_basic_init
HADOOP_SSH_PARALLEL=${HADOOP_SSH_PARALLEL:-10}
}
-function hadoop_populate_slaves_file()
+## @description Set the slave support information to the contents
+## @description of `filename`
+## @audience public
+## @stability stable
+## @replaceable no
+## @param filename
+## @return will exit if file does not exist
+function hadoop_populate_slaves_file
{
- # NOTE: This function is not user replaceable.
-
local slavesfile=$1
shift
if [[ -f "${slavesfile}" ]]; then
@@ -308,10 +364,17 @@ function hadoop_populate_slaves_file()
fi
}
+## @description Rotates the given `file` until `number` of
+## @description files exist.
+## @audience public
+## @stability stable
+## @replaceable no
+## @param filename
+## @param [number]
+## @return $? will contain last mv's return value
function hadoop_rotate_log
{
#
- # log rotation (mainly used for .out files)
# Users are likely to replace this one for something
# that gzips or uses dates or who knows what.
#
@@ -334,6 +397,13 @@ function hadoop_rotate_log
fi
}
+## @description Via ssh, log into `hostname` and run `command`
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param hostname
+## @param command
+## @param [...]
function hadoop_actual_ssh
{
# we are passing this function to xargs
@@ -345,6 +415,13 @@ function hadoop_actual_ssh
ssh ${HADOOP_SSH_OPTS} ${slave} $"${@// /\\ }" 2>&1 | sed "s/^/$slave: /"
}
+## @description Connect to ${HADOOP_SLAVES} or ${HADOOP_SLAVE_NAMES}
+## @description and execute command.
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param command
+## @param [...]
function hadoop_connect_to_hosts
{
# shellcheck disable=SC2124
@@ -405,6 +482,11 @@ function hadoop_connect_to_hosts
fi
}
+## @description Utility routine to handle --slaves mode
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param commandarray
function hadoop_common_slave_mode_execute
{
#
@@ -431,6 +513,14 @@ function hadoop_common_slave_mode_execute
hadoop_connect_to_hosts -- "${argv[@]}"
}
+## @description Verify that a shell command was passed a valid
+## @description class name
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param classname
+## @return 0 = success
+## @return 1 = failure w/user message
function hadoop_validate_classname
{
local class=$1
@@ -445,6 +535,14 @@ function hadoop_validate_classname
return 0
}
+## @description Append the `appendstring` if `checkstring` is not
+## @description present in the given `envvar`
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param envvar
+## @param checkstring
+## @param appendstring
function hadoop_add_param
{
#
@@ -466,21 +564,30 @@ function hadoop_add_param
fi
}
+## @description Register the given `shellprofile` to the Hadoop
+## @description shell subsystem
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param shellprofile
function hadoop_add_profile
{
# shellcheck disable=SC2086
hadoop_add_param HADOOP_SHELL_PROFILES $1 $1
}
+## @description Add a file system object (directory, file,
+## @description wildcard, ...) to the classpath. Optionally provide
+## @description a hint as to where in the classpath it should go.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param object
+## @param [before|after]
+## @return 0 = success (added or duplicate)
+## @return 1 = failure (doesn't exist or some other reason)
function hadoop_add_classpath
{
- # two params:
- # $1 = directory, file, wildcard, whatever to add
- # $2 = before or after, which determines where in the
- # classpath this object should go. default is after
- # return 0 = success (added or duplicate)
- # return 1 = failure (doesn't exist, whatever)
-
# However, with classpath (& JLP), we can do dedupe
# along with some sanity checking (e.g., missing directories)
# since we have a better idea of what is legal
@@ -517,15 +624,23 @@ function hadoop_add_classpath
return 0
}
+## @description Add a file system object (directory, file,
+## @description wildcard, ...) to the colonpath. Optionally provide
+## @description a hint as to where in the colonpath it should go.
+## @description Prior to adding, objects are checked for duplication
+## @description and check for existence. Many other functions use
+## @description this function as their base implementation
+## @description including `hadoop_add_javalibpath` and `hadoop_add_ldlibpath`.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param envvar
+## @param object
+## @param [before|after]
+## @return 0 = success (added or duplicate)
+## @return 1 = failure (doesn't exist or some other reason)
function hadoop_add_colonpath
{
- # two params:
- # $1 = directory, file, wildcard, whatever to add
- # $2 = before or after, which determines where in the
- # classpath this object should go
- # return 0 = success
- # return 1 = failure (duplicate)
-
# this is CLASSPATH, JLP, etc but with dedupe but no
# other checking
if [[ -d "${2}" ]] && [[ ":${!1}:" != *":$2:"* ]]; then
@@ -548,12 +663,34 @@ function hadoop_add_colonpath
return 1
}
+## @description Add a file system object (directory, file,
+## @description wildcard, ...) to the Java JNI path. Optionally
+## @description provide a hint as to where in the Java JNI path
+## @description it should go.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param object
+## @param [before|after]
+## @return 0 = success (added or duplicate)
+## @return 1 = failure (doesn't exist or some other reason)
function hadoop_add_javalibpath
{
# specialized function for a common use case
hadoop_add_colonpath JAVA_LIBRARY_PATH "$1" "$2"
}
+## @description Add a file system object (directory, file,
+## @description wildcard, ...) to the LD_LIBRARY_PATH. Optionally
+## @description provide a hint as to where in the LD_LIBRARY_PATH
+## @description it should go.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param object
+## @param [before|after]
+## @return 0 = success (added or duplicate)
+## @return 1 = failure (doesn't exist or some other reason)
function hadoop_add_ldlibpath
{
# specialized function for a common use case
@@ -563,6 +700,11 @@ function hadoop_add_ldlibpath
export LD_LIBRARY_PATH
}
+## @description Add the common/core Hadoop components to the
+## @description environment
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_add_common_to_classpath
{
#
@@ -582,6 +724,11 @@ function hadoop_add_common_to_classpath
hadoop_add_classpath "${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR}"'/*'
}
+## @description Add the user's custom classpath settings to the
+## @description environment
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_add_to_classpath_userpath
{
# Add the user-specified HADOOP_CLASSPATH to the
@@ -619,13 +766,15 @@ function hadoop_add_to_classpath_userpath
fi
}
+## @description Routine to configure any OS-specific settings.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @return may exit on failure conditions
function hadoop_os_tricks
{
local bindv6only
- # Some OSes have special needs. Here's some out of the box examples for OS X,
- # Linux and Windows on Cygwin.
- # Vendors, replace this with your special sauce.
HADOOP_IS_CYGWIN=false
case ${HADOOP_OS_TYPE} in
Darwin)
@@ -664,6 +813,11 @@ function hadoop_os_tricks
esac
}
+## @description Configure/verify ${JAVA_HOME}
+## @audience public
+## @stability stable
+## @replaceable yes
+## @return may exit on failure conditions
function hadoop_java_setup
{
# Bail if we did not detect it
@@ -685,6 +839,10 @@ function hadoop_java_setup
fi
}
+## @description Finish Java JNI paths prior to execution
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_finalize_libpaths
{
if [[ -n "${JAVA_LIBRARY_PATH}" ]]; then
@@ -695,6 +853,10 @@ function hadoop_finalize_libpaths
fi
}
+## @description Finish Java heap parameters prior to execution
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_finalize_hadoop_heap
{
if [[ -n "${HADOOP_HEAPSIZE_MAX}" ]]; then
@@ -720,9 +882,15 @@ function hadoop_finalize_hadoop_heap
fi
}
-# Accepts a variable name. If running on Cygwin, sets the variable value to the
-# equivalent translated Windows path by running the cygpath utility. If the
-# second argument is true, then the variable is treated as a path list.
+## @description Converts the contents of the variable name
+## @description `varnameref` into the equivalent Windows path.
+## @description If the second parameter is true, then `varnameref`
+## @description is treated as though it was a path list.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param varnameref
+## @param [true]
function hadoop_translate_cygwin_path
{
if [[ "${HADOOP_IS_CYGWIN}" = "true" ]]; then
@@ -736,9 +904,11 @@ function hadoop_translate_cygwin_path
fi
}
-#
-# fill in any last minute options that might not have been defined yet
-#
+## @description Finish configuring Hadoop specific system properties
+## @description prior to executing Java
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_finalize_hadoop_opts
{
hadoop_translate_cygwin_path HADOOP_LOG_DIR
@@ -754,6 +924,10 @@ function hadoop_finalize_hadoop_opts
hadoop_add_param HADOOP_OPTS hadoop.security.logger "-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER}"
}
+## @description Finish Java classpath prior to execution
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_finalize_classpath
{
hadoop_add_classpath "${HADOOP_CONF_DIR}" before
@@ -764,6 +938,10 @@ function hadoop_finalize_classpath
hadoop_translate_cygwin_path CLASSPATH true
}
+## @description Finish Catalina configuration prior to execution
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_finalize_catalina_opts
{
@@ -783,9 +961,14 @@ function hadoop_finalize_catalina_opts
hadoop_add_param CATALINA_OPTS "${prefix}.ssl.keystore.file" "-D${prefix}.ssl.keystore.file=${HADOOP_CATALINA_SSL_KEYSTORE_FILE}"
}
+## @description Finish all the remaining environment settings prior
+## @description to executing Java. This is a wrapper that calls
+## @description the other `finalize` routines.
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_finalize
{
-
hadoop_shellprofiles_finalize
hadoop_finalize_classpath
@@ -801,10 +984,15 @@ function hadoop_finalize
hadoop_translate_cygwin_path HADOOP_MAPRED_HOME
}
+## @description Print usage information and exit with the passed
+## @description `exitcode`
+## @audience public
+## @stability stable
+## @replaceable no
+## @param exitcode
+## @return This function will always exit.
function hadoop_exit_with_usage
{
- # NOTE: This function is not user replaceable.
-
local exitcode=$1
if [[ -z $exitcode ]]; then
exitcode=1
@@ -819,6 +1007,12 @@ function hadoop_exit_with_usage
exit $exitcode
}
+## @description Verify that prerequisites have been met prior to
+## @description excuting a privileged program.
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @return This routine may exit.
function hadoop_verify_secure_prereq
{
# if you are on an OS like Illumos that has functional roles
@@ -834,6 +1028,9 @@ function hadoop_verify_secure_prereq
fi
}
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_setup_secure_service
{
# need a more complicated setup? replace me!
@@ -842,6 +1039,9 @@ function hadoop_setup_secure_service
HADOOP_LOG_DIR=${HADOOP_SECURE_LOG_DIR}
}
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_verify_piddir
{
if [[ -z "${HADOOP_PID_DIR}" ]]; then
@@ -864,6 +1064,9 @@ function hadoop_verify_piddir
rm "${HADOOP_PID_DIR}/$$" >/dev/null 2>&1
}
+## @audience private
+## @stability evolving
+## @replaceable yes
function hadoop_verify_logdir
{
if [[ -z "${HADOOP_LOG_DIR}" ]]; then
@@ -886,7 +1089,14 @@ function hadoop_verify_logdir
rm "${HADOOP_LOG_DIR}/$$" >/dev/null 2>&1
}
-function hadoop_status_daemon()
+## @description Determine the status of the daemon referenced
+## @description by `pidfile`
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param pidfile
+## @return (mostly) LSB 4.1.0 compatible status
+function hadoop_status_daemon
{
#
# LSB 4.1.0 compatible status command (1)
@@ -919,6 +1129,14 @@ function hadoop_status_daemon()
return 3
}
+## @description Execute the Java `class`, passing along any `options`.
+## @description Additionally, set the Java property -Dproc_`command`.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param command
+## @param class
+## @param [options]
function hadoop_java_exec
{
# run a java command. this is used for
@@ -936,6 +1154,14 @@ function hadoop_java_exec
exec "${JAVA}" "-Dproc_${command}" ${HADOOP_OPTS} "${class}" "$@"
}
+## @description Start a non-privileged daemon in the foreground.
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param command
+## @param class
+## @param pidfile
+## @param [options]
function hadoop_start_daemon
{
# this is our non-privileged daemon starter
@@ -961,10 +1187,17 @@ function hadoop_start_daemon
exec "${JAVA}" "-Dproc_${command}" ${HADOOP_OPTS} "${class}" "$@"
}
+## @description Start a non-privileged daemon in the background.
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param command
+## @param class
+## @param pidfile
+## @param outfile
+## @param [options]
function hadoop_start_daemon_wrapper
{
- # this is our non-privileged daemon start
- # that fires up a daemon in the *background*
local daemonname=$1
local class=$2
local pidfile=$3
@@ -1019,6 +1252,17 @@ function hadoop_start_daemon_wrapper
return 0
}
+## @description Start a privileged daemon in the foreground.
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param command
+## @param class
+## @param daemonpidfile
+## @param daemonoutfile
+## @param daemonerrfile
+## @param wrapperpidfile
+## @param [options]
function hadoop_start_secure_daemon
{
# this is used to launch a secure daemon in the *foreground*
@@ -1075,6 +1319,18 @@ function hadoop_start_secure_daemon
"${class}" "$@"
}
+## @description Start a privileged daemon in the background.
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param command
+## @param class
+## @param daemonpidfile
+## @param daemonoutfile
+## @param wrapperpidfile
+## @param warpperoutfile
+## @param daemonerrfile
+## @param [options]
function hadoop_start_secure_daemon_wrapper
{
# this wraps hadoop_start_secure_daemon to take care
@@ -1155,6 +1411,13 @@ function hadoop_start_secure_daemon_wrapper
return 0
}
+## @description Stop the non-privileged `command` daemon with that
+## @description that is running at `pidfile`.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param command
+## @param pidfile
function hadoop_stop_daemon
{
local cmd=$1
@@ -1180,6 +1443,15 @@ function hadoop_stop_daemon
fi
}
+## @description Stop the privileged `command` daemon with that
+## @description that is running at `daemonpidfile` and launched with
+## @description the wrapper at `wrapperpidfile`.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param command
+## @param daemonpidfile
+## @param wrapperpidfile
function hadoop_stop_secure_daemon
{
local command=$1
@@ -1194,6 +1466,16 @@ function hadoop_stop_secure_daemon
return ${ret}
}
+## @description Manage a non-privileged daemon.
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param [start|stop|status|default]
+## @param command
+## @param class
+## @param daemonpidfile
+## @param daemonoutfile
+## @param [options]
function hadoop_daemon_handler
{
local daemonmode=$1
@@ -1238,6 +1520,19 @@ function hadoop_daemon_handler
esac
}
+## @description Manage a privileged daemon.
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param [start|stop|status|default]
+## @param command
+## @param class
+## @param daemonpidfile
+## @param daemonoutfile
+## @param wrapperpidfile
+## @param wrapperoutfile
+## @param wrappererrfile
+## @param [options]
function hadoop_secure_daemon_handler
{
local daemonmode=$1
@@ -1290,6 +1585,13 @@ function hadoop_secure_daemon_handler
esac
}
+## @description Verify that ${USER} is allowed to execute the
+## @description given subcommand.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param subcommand
+## @return will exit on failure conditions
function hadoop_verify_user
{
local command=$1
@@ -1303,6 +1605,13 @@ function hadoop_verify_user
fi
}
+## @description Perform the 'hadoop classpath', etc subcommand with the given
+## @description parameters
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param [parameters]
+## @return will print & exit with no params
function hadoop_do_classpath_subcommand
{
if [[ "$#" -gt 1 ]]; then
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30ca8ce/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
new file mode 100644
index 0000000..006fca5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
@@ -0,0 +1,98 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Unix Shell Guide
+
+Much of Hadoop's functionality is controlled via [the shell](CommandsManual.html). There are several ways to modify the default behavior of how these commands execute.
+
+## Important End-User Environment Variables
+
+Hadoop has many environment variables that control various aspects of the software. (See `hadoop-env.sh` and related files.) Some of these environment variables are dedicated to helping end users manage their runtime.
+
+### `HADOOP_CLIENT_OPTS`
+
+This environment variable is used for almost all end-user operations. It can be used to set any Java options as well as any Hadoop options via a system property definition. For example:
+
+```bash
+HADOOP_CLIENT_OPTS="-Xmx1g -Dhadoop.socks.server=localhost:4000" hadoop fs -ls /tmp
+```
+
+will increase the memory and send this command via a SOCKS proxy server.
+
+### `HADOOP_USER_CLASSPATH`
+
+The Hadoop scripts have the capability to inject more content into the classpath of the running command by setting this environment variable. It should be a colon delimited list of directories, files, or wildcard locations.
+
+```bash
+HADOOP_USER_CLASSPATH=${HOME}/lib/myjars/*.jar hadoop classpath
+```
+
+A user can provides hints to the location of the paths via the `HADOOP_USER_CLASSPATH_FIRST` variable. Setting this to any value will tell the system to try and push these paths near the front.
+
+### Auto-setting of Variables
+
+If a user has a common set of settings, they can be put into the `${HOME}/.hadooprc` file. This file is always read to initialize and override any variables that the user may want to customize. It uses bash syntax, similar to the `.bashrc` file:
+
+For example:
+
+```bash
+#
+# my custom Hadoop settings!
+#
+
+HADOOP_USER_CLASSPATH=${HOME}/hadoopjars/*
+HADOOP_USER_CLASSPATH_FIRST=yes
+HADOOP_CLIENT_OPTS="-Xmx1g"
+```
+
+The `.hadooprc` file can also be used to extend functionality and teach Hadoop new tricks. For example, to run hadoop commands accessing the server referenced in the environment variable `${HADOOP_SERVER}`, the following in the `.hadooprc` will do just that:
+
+```bash
+
+if [[ -n ${HADOOP_SERVER} ]]; then
+ HADOOP_CONF_DIR=/etc/hadoop.${HADOOP_SERVER}
+fi
+```
+
+## Administrator Environment
+
+There are many environment variables that impact how the system operates. By far, the most important are the series of `_OPTS` variables that control how daemons work. These variables should contain all of the relevant settings for those daemons.
+
+More, detailed information is contained in `hadoop-env.sh` and the other env.sh files.
+
+Advanced administrators may wish to supplement or do some platform-specific fixes to the existing scripts. In some systems, this means copying the errant script or creating a custom build with these changes. Hadoop provides the capabilities to do function overrides so that the existing code base may be changed in place without all of that work. Replacing functions is covered later under the Shell API documentation.
+
+## Developer and Advanced Administrator Environment
+
+### Shell Profiles
+
+Apache Hadoop allows for third parties to easily add new features through a variety of pluggable interfaces. This includes a shell code subsystem that makes it easy to inject the necessary content into the base installation.
+
+Core to this functionality is the concept of a shell profile. Shell profiles are shell snippets that can do things such as add jars to the classpath, configure Java system properties and more.
+
+Shell profiles may be installed in either `${HADOOP_CONF_DIR}/shellprofile.d` or `${HADOOP_PREFIX}/libexec/shellprofile.d`. Shell profiles in the `libexec` directory are part of the base installation and cannot be overriden by the user. Shell profiles in the configuration directory may be ignored if the end user changes the configuration directory at runtime.
+
+An example of a shell profile is in the libexec directory.
+
+## Shell API
+
+Hadoop's shell code has a [function library](./UnixShellAPI.html) that is open for administrators and developers to use to assist in their configuration and advanced feature management. These APIs follow the standard [Hadoop Interface Classification](./InterfaceClassification.html), with one addition: Replaceable.
+
+The shell code allows for core functions to be overridden. However, not all functions can be or are safe to be replaced. If a function is not safe to replace, it will have an attribute of Replaceable: No. If a function is safe to replace, it will have the attribute of Replaceable: Yes.
+
+In order to replace a function, create a file called `hadoop-user-functions.sh` in the `${HADOOP_CONF_DIR}` directory. Simply define the new, replacement function in this file and the system will pick it up automatically. There may be as many replacement functions as needed in this file. Examples of function replacement are in the `hadoop-user-functions.sh.examples` file.
+
+
+Functions that are marked Public and Stable are safe to use in shell profiles as-is. Other functions may change in a minor release.
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30ca8ce/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index a90c41d..2e098ef 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -68,6 +68,7 @@
<item name="HTTP Authentication" href="hadoop-project-dist/hadoop-common/HttpAuthentication.html"/>
<item name="Hadoop KMS" href="hadoop-kms/index.html"/>
<item name="Tracing" href="hadoop-project-dist/hadoop-common/Tracing.html"/>
+ <item name="Unix Shell Guide" href="hadoop-project-dist/hadoop-common/UnixShellGuide.html"/>
</menu>
<menu name="HDFS" inherit="top">
@@ -159,7 +160,8 @@
<menu name="Reference" inherit="top">
<item name="Release Notes" href="hadoop-project-dist/hadoop-common/releasenotes.html"/>
- <item name="API docs" href="api/index.html"/>
+ <item name="Java API docs" href="api/index.html"/>
+ <item name="Unix Shell API" href="hadoop-project-dist/hadoop-common/UnixShellAPI.html"/>
<item name="Common CHANGES.txt" href="hadoop-project-dist/hadoop-common/CHANGES.txt"/>
<item name="HDFS CHANGES.txt" href="hadoop-project-dist/hadoop-hdfs/CHANGES.txt"/>
<item name="MapReduce CHANGES.txt" href="hadoop-project-dist/hadoop-mapreduce/CHANGES.txt"/>