You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2024/12/22 18:44:48 UTC
(accumulo) branch main updated (f3aa11a85c -> 90304d382f)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from f3aa11a85c Merge branch '3.1'
add 9e146f95a4 Backport and improve the cluster scripts to 2.1 (#5174)
new 8fc9ff9daf Merge branch '2.1' into 3.1
new 90304d382f Merge branch '3.1'
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
assemble/bin/accumulo-cluster | 331 ++++++++++++++++++++++++------------------
assemble/bin/accumulo-service | 12 +-
assemble/conf/accumulo-env.sh | 2 +-
3 files changed, 200 insertions(+), 145 deletions(-)
(accumulo) 01/01: Merge branch '3.1'
Posted by ct...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 90304d382f274e22605b8de04fc7d5a258f595bf
Merge: f3aa11a85c 8fc9ff9daf
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Sun Dec 22 13:44:28 2024 -0500
Merge branch '3.1'
assemble/bin/accumulo-cluster | 331 ++++++++++++++++++++++++------------------
assemble/bin/accumulo-service | 12 +-
assemble/conf/accumulo-env.sh | 2 +-
3 files changed, 200 insertions(+), 145 deletions(-)
diff --cc assemble/bin/accumulo-cluster
index a87ad4938a,8367fb2a1b..cc6b04654e
--- a/assemble/bin/accumulo-cluster
+++ b/assemble/bin/accumulo-cluster
@@@ -26,35 -26,53 +26,42 @@@
# and addresses to override the default lookups
#
- function print_usage {
+ function print_usage() {
cat <<EOF
- Usage: accumulo-cluster <command> (<argument> ...) [<option> ...]
-
- Options:
- --dry-run Prints information and commands, but does not execute them
-
- Commands:
- create-config Creates cluster config
- start-here Deprecated. Alias for "start --local --all"
- stop-here Deprecated. Alias for "stop --local --all"
- restart [--local] [--all | [--manager] [--gc] [--monitor] [--tservers[=group]] [--sservers[=group]] [--compactors[=group]]
- Restarts the Accumulo cluster
- start [--local] [--all | [--manager] [--gc] [--monitor] [--tservers[=group]] [--sservers[=group]] [--compactors[=group]]
- Starts Accumulo cluster services
- stop [--local] [--all | [--manager] [--gc] [--monitor] [--tservers[=group]] [--sservers[=group]] [--compactors[=group]]
- Stops Accumulo cluster services
- kill [--local] [--all | [--manager] [--gc] [--monitor] [--tservers[=group]] [--sservers[=group]] [--compactors[=group]]
- Kills Accumulo cluster services
-
- Examples:
-
- accumulo-cluster start # start all servers
- accumulo-cluster start --dry-run # print debug information and commands to be executed
- accumulo-cluster start --local # start all local services
- accumulo-cluster start --local --manager # start local manager services
- accumulo-cluster start --tservers # start all tservers
- accumulo-cluster start --tservers=group1 # start all group1 tservers
- accumulo-cluster start --local --manager --tservers # Start the local manager and local tservers
+ $(cyan Usage): $(green accumulo-cluster) <$(yellow command)> [<$(yellow option)> ...]
+
+ $(cyan Options):
+ $(cyan General options):
+ $(green --dry-run) Prints information and commands, but does not execute them
+ $(green --local) Operates on matching local services only (no SSH)
+
+ $(cyan 'Service type selection options (if none are used, all service types are selected)'):
+ $(green --manager) Selects the manager service (oversees cluster operations)
+ $(green --gc) Selects the gc service (cleans up unused files)
+ $(green --monitor) Selects the monitor web service (shows cluster information)
- $(green --compaction-coordinator) Selects the compaction-coordinator service (external compactions)
- $(green --tservers) Selects the tablet server services (read/write operations on tablets)
++ $(green --tservers[$(yellow '=group')]) Selects the tablet server services (read/write operations on tablets)
+ $(green --sservers)[$(yellow '=group')] Selects the scan server services (read-only eventually consistent scans)
+ $(green --compactors)[$(yellow '=group')] Selects the compactor services (external compactions)
- $(green --no-tservers) $(yellow Deprecated). Selects service types except tservers (for backwards compatibility)
+ $(yellow NOTE): some server types support an optional $(yellow group) name. If it is not provided or is empty, then
+ all groups are considered. Declare multiple groups as a single space-separated parameter. Later
+ options overwrite earlier ones, so $(yellow '--sservers="g1 g2" --sserver') operates on all sserver groups.
+
+ $(cyan Commands):
+ $(green create-config) Creates cluster config (ignores service selection options)
+ $(green start) Starts Accumulo cluster services
+ $(green stop) Stops Accumulo cluster services
+ $(green restart) Restarts Accumulo cluster services
+ $(green kill) Kills Accumulo cluster services
+
- $(cyan Deprecated commands):
- $(green start-non-tservers) $(yellow Deprecated). Alias for "start --no-tservers"
- $(green start-servers) $(yellow Deprecated). Alias for "start"
- $(green stop-servers) $(yellow Deprecated). Alias for "stop"
- $(green start-tservers) $(yellow Deprecated). Alias for "start --tservers"
- $(green stop-tservers) $(yellow Deprecated). Alias for "stop --tservers"
- $(green start-here) $(yellow Deprecated). Alias for "start --local"
- $(green stop-here) $(yellow Deprecated). Alias for "stop --local"
-
+ $(cyan Examples):
+ $(purple 'accumulo-cluster start') $(blue '# start all servers')
+ $(purple 'accumulo-cluster start --dry-run') $(blue '# print debug information and commands to be executed')
+ $(purple 'accumulo-cluster start --local') $(blue '# start all local services')
+ $(purple 'accumulo-cluster start --local --manager') $(blue '# start local manager services')
+ $(purple 'accumulo-cluster start --tservers') $(blue '# start all tservers')
+ $(purple 'accumulo-cluster start --sservers=group1') $(blue '# start all group1 sservers')
+ $(purple 'accumulo-cluster start --sservers="group1 group2"') $(blue '# start all group1 and group2 sservers')
+ $(purple 'accumulo-cluster start --local --manager --tservers') $(blue '# Start the local manager and local tservers')
EOF
}
@@@ -73,7 -127,18 +115,18 @@@ function parse_args()
ARG_COMPACTOR=0
ARG_COMPACTOR_GROUP=""
- PARSE_OUTPUT=$(getopt -o "" --long "dry-run,all,local,manager,gc,monitor,tservers::,sservers::,compactors::" -n 'accumulo-cluster' -- "$@")
+ # the correct version of getopt will always exit with exit code 4 when provided the `-T` flag
+ getopt -T &>/dev/null
+ if (($? != 4)); then
+ echo "$(red ERROR): Missing $(yellow util-linux) (or equivalent) version of $(green getopt). Unable to continue."
+ exit 2
+ fi
+
- if ! PARSE_OUTPUT=$(getopt -o "" --long "dry-run,local,manager,gc,monitor,compaction-coordinator,no-tservers,tservers,sservers::,compactors::" -n 'accumulo-cluster' -- "$@"); then
++ if ! PARSE_OUTPUT=$(getopt -o "" --long "dry-run,local,manager,gc,monitor,tservers::,sservers::,compactors::" -n 'accumulo-cluster' -- "$@"); then
+ print_usage
+ exit 1
+ fi
+
eval set -- "$PARSE_OUTPUT"
while true; do
@@@ -103,27 -168,39 +156,24 @@@
ARG_MONITOR=1
shift 1
;;
- --compaction-coordinator)
- ARG_ALL=0
- ARG_COORDINATOR=1
- shift 1
- ;;
--tservers)
+ ARG_ALL=0
ARG_TSERVER=1
- if [[ -n $2 ]]; then
- ARG_TSERVER_GROUP=$2
- fi
- shift 1
++ ARG_TSERVER_GROUP=$2
+ shift 2
;;
--sservers)
+ ARG_ALL=0
ARG_SSERVER=1
- if [[ -n $2 ]]; then
- ARG_SSERVER_GROUP=$2
- fi
+ ARG_SSERVER_GROUP=$2
shift 2
;;
--compactors)
+ ARG_ALL=0
ARG_COMPACTOR=1
- if [[ -n $2 ]]; then
- ARG_COMPACTOR_GROUP=$2
- fi
+ ARG_COMPACTOR_GROUP=$2
shift 2
;;
- --no-tservers)
- echo "$(yellow WARN): The $(yellow "--no-tservers") option is deprecated. Please specify the servers you wish to manage instead"
- ARG_ALL=0
- ARG_MANAGER=1
- ARG_GC=1
- ARG_MONITOR=1
- ARG_COORDINATOR=1
- ARG_SSERVER=1
- ARG_COMPACTOR=1
- shift 1
- ;;
--)
shift
break
@@@ -136,40 -213,34 +186,33 @@@
esac
done
- # All and any of the others are mutually exclusive
- if [[ $ARG_ALL == 1 && ($ARG_MANAGER == 1 ||
- $ARG_GC == 1 || $ARG_MONITOR == 1 || $ARG_TSERVER == 1 ||
- $ARG_SSERVER == 1 || $ARG_COMPACTOR == 1) ]]; then
- echo "--all cannot be used with other options"
- print_usage
- exit 1
- fi
-
- # Handle the case where no args were passed
- if [[ $ARG_MANAGER == 0 && $ARG_GC == 0 &&
- $ARG_MONITOR == 0 && $ARG_TSERVER == 0 &&
- $ARG_SSERVER == 0 && $ARG_COMPACTOR == 0 ]]; then
- ARG_ALL=1
+ if [[ $# -eq 0 ]]; then
+ invalid_args "<command> cannot be empty"
+ elif [[ $# -ne 1 ]]; then
+ # use getopt to display the remaining non-option parameters quoted for readability
+ local remaining
+ remaining="$(getopt -o "" -- "$@")"
+ invalid_args "Only one <$(yellow command)> is allowed, but found:$(yellow "${remaining#*--}")"
fi
+ ARG_CMD=$1
+ debug "ARG_CMD=$ARG_CMD"
+ debug "ARG_ALL=$ARG_ALL"
+ debug "ARG_LOCAL=$ARG_LOCAL"
+ debug "ARG_MANAGER=$ARG_MANAGER"
+ debug "ARG_GC=$ARG_GC"
+ debug "ARG_MONITOR=$ARG_MONITOR"
- debug "ARG_COORDINATOR=$ARG_COORDINATOR"
+ debug "ARG_TSERVER=$ARG_TSERVER"
+ debug "ARG_TSERVER_GROUP=$ARG_TSERVER_GROUP"
+ debug "ARG_SSERVER=$ARG_SSERVER"
+ debug "ARG_SSERVER_GROUP=$ARG_SSERVER_GROUP"
+ debug "ARG_COMPACTOR=$ARG_COMPACTOR"
+ debug "ARG_COMPACTOR_GROUP=$ARG_COMPACTOR_GROUP"
- if isDebug; then
- echo "DEBUG=$DEBUG"
- echo "ARG_ALL=$ARG_ALL"
- echo "ARG_LOCAL=$ARG_LOCAL"
- echo "ARG_MANAGER=$ARG_MANAGER"
- echo "ARG_GC=$ARG_GC"
- echo "ARG_MONITOR=$ARG_MONITOR"
- echo "ARG_TSERVER=$ARG_TSERVER"
- echo "ARG_TSERVER_GROUP=$ARG_TSERVER_GROUP"
- echo "ARG_SSERVER=$ARG_SSERVER"
- echo "ARG_SSERVER_GROUP=$ARG_SSERVER_GROUP"
- echo "ARG_COMPACTOR=$ARG_COMPACTOR"
- echo "ARG_COMPACTOR_GROUP=$ARG_COMPACTOR_GROUP"
- fi
}
- function invalid_args {
- echo -e "Invalid arguments: $1\n"
+ function invalid_args() {
+ echo "$(red ERROR): $(yellow invalid arguments): $*"
+ echo
print_usage 1>&2
exit 1
}
@@@ -241,8 -335,45 +307,8 @@@ function parse_config()
exit 1
fi
- # This version doesn't support configurable tserver groups, so
- # use a default group to allow code reuse with newer branches
- # Shellcheck thinks these aren't used, but they are referenced
- # indirectly by group name.
- TSERVER_GROUPS="default"
- #shellcheck disable=SC2034
- TSERVERS_PER_HOST_default=${NUM_TSERVERS:-1}
- #shellcheck disable=SC2034
- TSERVER_HOSTS_default=$TSERVER_HOSTS
-
- # Rename variables from this version's config parser to ones that
- # this script expects, to support code reuse with newer branches
- COMPACTOR_GROUPS=$COMPACTION_QUEUES
- local var_name
- for group in $SSERVER_GROUPS; do
- var_name="NUM_SSERVERS_$group"
- if [[ -n ${!var_name} ]]; then
- declare "SSERVERS_PER_HOST_$group"="${!var_name}"
- else
- declare "SSERVERS_PER_HOST_$group"="${NUM_SSERVERS:-1}"
- fi
- done
- for group in $COMPACTOR_GROUPS; do
- var_name="NUM_COMPACTORS_$group"
- if [[ -n ${!var_name} ]]; then
- declare "COMPACTORS_PER_HOST_$group"="${!var_name}"
- else
- declare "COMPACTORS_PER_HOST_$group"="${NUM_COMPACTORS:-1}"
- fi
- done
-
- # This version requires a compaction coordinator for compactors
- if [[ -n $COMPACTOR_GROUPS && -z $COORDINATOR_HOSTS ]]; then
- echo "$(yellow WARN): External compactor group(s) configured, but no coordinator configured"
- fi
-
if [[ -z $COMPACTOR_GROUPS ]]; then
- echo "WARN: No compactor groups configured"
- # no compactor groups are required for this version
- debug "No compactor groups configured"
++ echo "$(yellow WARN): No compactor groups configured"
else
for group in $COMPACTOR_GROUPS; do
G="COMPACTOR_HOSTS_$group"
@@@ -392,8 -528,8 +463,8 @@@ function control_services()
hosts="TSERVER_HOSTS_$group"
for tserver in ${!hosts}; do
if canRunOnHost "$tserver"; then
- isDebug || echo -n "."
+ echo -n "."
- execute_command "$operation" "$tserver" tserver "$group"
+ execute_command "$operation" "$tserver" tserver "$group" "-o" "tserver.group=$group"
if ((++count % 72 == 0)); then
echo
wait
@@@ -470,7 -619,7 +544,7 @@@
if [[ $ARG_LOCAL == 0 && $ARG_ALL == 1 && ($operation == "stop" || $operation == "kill") ]]; then
if ! isDebug; then
echo "Cleaning all server entries in ZooKeeper"
- $accumulo_cmd org.apache.accumulo.server.util.ZooZap -manager -tservers -compactors -sservers
- "$accumulo_cmd" org.apache.accumulo.server.util.ZooZap -manager -tservers -compaction-coordinators -compactors -sservers
++ "$accumulo_cmd" org.apache.accumulo.server.util.ZooZap -manager -tservers -compactors -sservers
fi
fi
@@@ -561,21 -718,52 +629,8 @@@ EO
parse_config
control_services kill
;;
-- start-here)
-- parse_config
-- ARG_ALL=1
-- ARG_LOCAL=1
-- control_services start
-- ;;
-- stop-here)
-- parse_config
-- ARG_ALL=1
-- ARG_LOCAL=1
-- control_services stop
-- control_services kill
- ;;
- start-non-tservers)
- echo "'$ARG_CMD' is deprecated. Please specify the servers you wish to start instead"
- parse_config
- ARG_MANAGER=1
- ARG_GC=1
- ARG_MONITOR=1
- ARG_COORDINATOR=1
- ARG_SSERVER=1
- ARG_COMPACTOR=1
- control_services start
- ;;
- start-servers)
- echo "'$ARG_CMD' is deprecated. Please use 'start' instead"
- parse_config
- control_services start
- ;;
- stop-servers)
- echo "'$ARG_CMD' is deprecated. Please use 'stop' instead"
- parse_config
- control_services stop
- ;;
- start-tservers)
- echo "'$ARG_CMD' is deprecated. Please use 'start --tservers' instead"
- ARG_TSERVER=1
- control_services start
- ;;
- stop-tservers)
- echo "'$ARG_CMD' is deprecated. Please use 'stop --tservers' instead"
- ARG_TSERVER=1
- control_services stop
-- ;;
*)
- invalid_args "$cmd is an invalid <command>"
+ invalid_args "'$ARG_CMD' is an invalid <command>"
;;
esac
}