You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2024/12/22 18:44:46 UTC
(accumulo) branch 3.1 updated (90ebfc67fd -> 8fc9ff9daf)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a change to branch 3.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 90ebfc67fd Use new jquery api in monitor (#5190)
add 9e146f95a4 Backport and improve the cluster scripts to 2.1 (#5174)
new 8fc9ff9daf Merge branch '2.1' into 3.1
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
assemble/bin/accumulo-cluster | 1127 ++++++++++++++++++-------------------
assemble/bin/accumulo-service | 104 +++-
assemble/conf/accumulo-env.sh | 32 +-
assemble/conf/accumulo.properties | 5 +
4 files changed, 658 insertions(+), 610 deletions(-)
(accumulo) 01/01: Merge branch '2.1' into 3.1
Posted by ct...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 3.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 8fc9ff9daf32bc83be03531a15818838f90ff6b5
Merge: 90ebfc67fd 9e146f95a4
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Sun Dec 22 02:17:53 2024 -0500
Merge branch '2.1' into 3.1
assemble/bin/accumulo-cluster | 1127 ++++++++++++++++++-------------------
assemble/bin/accumulo-service | 104 +++-
assemble/conf/accumulo-env.sh | 32 +-
assemble/conf/accumulo.properties | 5 +
4 files changed, 658 insertions(+), 610 deletions(-)
diff --cc assemble/bin/accumulo-cluster
index 056026b12e,ab39fcec62..8367fb2a1b
--- a/assemble/bin/accumulo-cluster
+++ b/assemble/bin/accumulo-cluster
@@@ -96,118 -290,104 +290,95 @@@ function canRunOnHost()
exit 1
fi
- if [[ -n $COMPACTION_QUEUES && -z $COORDINATOR_HOSTS ]]; then
- echo "WARN: External compaction queues configured, but no coordinator configured"
- fi
-
- for queue in $COMPACTION_QUEUES; do
- Q="COMPACTOR_HOSTS_${queue}"
- if [[ -z ${!Q} ]]; then
- echo "WARN: External compaction queue $queue configured, but no compactors configured for it"
+ local found=0
+ local addr
+ for addr in "${LOCAL_HOST_ADDRESSES[@]}"; do
+ if [[ $1 == "$addr" ]]; then
+ found=1
+ break
fi
done
+ [[ $found == 1 ]]
+ }
- for group in $SSERVER_GROUPS; do
- G="SSERVER_HOSTS_${group}"
- if [[ -z ${!G} ]]; then
- echo "WARN: ScanServer group $group configured, but no hosts configured for it"
- fi
- done
+ function parse_config() {
- unset manager1
- manager1=$(echo "${MANAGER_HOSTS}" | cut -d" " -f1)
- for file in slaves tservers monitor tracers gc managers masters; do
- if [[ -f $conf/$file ]]; then
- echo "$(red ERROR): A $(yellow "$file") file was found in $(yellow "$conf/")"
- echo "$(red ERROR): Accumulo now uses cluster host configuration information from $(yellow cluster.yaml)"
- echo "$(red ERROR): and requires that the $(yellow "$file") file not be present to reduce confusion."
- exit 1
- fi
- done
-
+ local manager1
+ local tservers_found
+ local group
+ local G
- if [[ -z $MONITOR_HOSTS ]]; then
- echo "WARN: monitors not found in ${conf}/cluster.yaml, using first manager host $manager1"
- MONITOR_HOSTS=$manager1
+ if [[ ! -f $conf/cluster.yaml ]]; then
+ echo "$(red ERROR): File not found $(yellow "$conf/cluster.yaml")"
+ echo "$(red ERROR): Please make sure it exists and is configured with the host information."
+ echo "$(red ERROR): Run $(yellow accumulo-cluster create-config) to create an example configuration."
+ exit 1
fi
- if [[ -z $GC_HOSTS ]]; then
- echo "WARN: gc not found in ${conf}/cluster.yaml, using first manager host $manager1"
- GC_HOSTS=$manager1
+ trap 'rm -f "$CONFIG_FILE"' EXIT
+ CONFIG_FILE=$(mktemp --tmpdir "ClusterConfigParser-XXXXXXXX.out") || exit 1
+ "$accumulo_cmd" org.apache.accumulo.core.conf.cluster.ClusterConfigParser "$conf/cluster.yaml" "$CONFIG_FILE" || parse_fail
+ #shellcheck source=/dev/null
+ . "$CONFIG_FILE"
+ debug "Parsed config from $(white "$conf/cluster.yaml")"
+ local line
+ if isDebug; then
+ while read -r line; do
+ debug "$(white "$line")"
+ done <"$CONFIG_FILE"
fi
+ rm -f "$CONFIG_FILE"
- if [[ -z $NUM_TSERVERS ]]; then
- echo "INFO: ${NUM_TSERVERS} tservers will be started per host"
+ # this might not be possible, since the ClusterConfigParser would probably fail instead
+ if [[ -z $MANAGER_HOSTS ]]; then
+ echo "$(red ERROR): $(yellow managers) not found in $(yellow "$conf/cluster.yaml")"
+ exit 1
fi
+ # This version doesn't support configurable tserver groups, so
+ # use a default group to allow code reuse with newer branches
+ # Shellcheck thinks these aren't used, but they are referenced
+ # indirectly by group name.
+ TSERVER_GROUPS="default"
+ #shellcheck disable=SC2034
+ TSERVERS_PER_HOST_default=${NUM_TSERVERS:-1}
+ #shellcheck disable=SC2034
+ TSERVER_HOSTS_default=$TSERVER_HOSTS
+
+ # Rename variables from this version's config parser to ones that
+ # this script expects, to support code reuse with newer branches
+ COMPACTOR_GROUPS=$COMPACTION_QUEUES
+ local var_name
for group in $SSERVER_GROUPS; do
- var_name="NUM_SSERVERS_${group}"
+ var_name="NUM_SSERVERS_$group"
if [[ -n ${!var_name} ]]; then
- echo "INFO: ${!var_name} scan servers will be started per host for group ${group}"
+ declare "SSERVERS_PER_HOST_$group"="${!var_name}"
+ else
+ declare "SSERVERS_PER_HOST_$group"="${NUM_SSERVERS:-1}"
fi
done
-
- for queue in $COMPACTION_QUEUES; do
- var_name="NUM_COMPACTORS_${queue}"
+ for group in $COMPACTOR_GROUPS; do
+ var_name="NUM_COMPACTORS_$group"
if [[ -n ${!var_name} ]]; then
- echo "INFO: ${!var_name} compactors will be started per host for queue ${queue}"
- fi
- done
- }
-
- function control_service() {
- control_cmd="$1"
- host="$2"
- service="$3"
-
- local last_instance_id
- last_instance_id=1
- [[ $service == "tserver" ]] && last_instance_id=${NUM_TSERVERS:-1}
- [[ $service == "sserver" ]] && last_instance_id=${NUM_SSERVERS:-1}
- [[ $service == "compactor" ]] && last_instance_id=${NUM_COMPACTORS:-1}
-
- for ((inst_id = 1; inst_id <= last_instance_id; inst_id++)); do
- # Only increment the service name when more than one service is desired.
- ACCUMULO_SERVICE_INSTANCE=""
- [[ last_instance_id -gt 1 ]] && ACCUMULO_SERVICE_INSTANCE="${inst_id}"
- [[ $service =~ ^compactor|sserver$ ]] && ACCUMULO_SERVICE_INSTANCE="${ACCUMULO_SERVICE_INSTANCE}_${5}"
-
- if [[ $host == localhost || $host == "$(hostname -s)" || $host == "$(hostname -f)" || "$(hostname -I)" =~ $host ]]; then
- #
- # The server processes take arguments (e.g. -p, -o, -q [in the case of the Compactor]). Always add the -a argument
- # using the value of $host
- #
- if [[ $# -gt 3 ]]; then
- debugAndRun ACCUMULO_SERVICE_INSTANCE="${ACCUMULO_SERVICE_INSTANCE}" "${bin}/accumulo-service" "$service" "$control_cmd" "-o" "general.process.bind.addr=$host" "${@:4}"
- else
- debugAndRun ACCUMULO_SERVICE_INSTANCE="${ACCUMULO_SERVICE_INSTANCE}" "${bin}/accumulo-service" "$service" "$control_cmd" "-o" "general.process.bind.addr=$host"
- fi
+ declare "COMPACTORS_PER_HOST_$group"="${!var_name}"
else
- if [[ $# -gt 3 ]]; then
- EXTRA_ARGS="${*:4}"
- debugAndRun "$SSH" "$host" "bash -c 'ACCUMULO_SERVICE_INSTANCE=${ACCUMULO_SERVICE_INSTANCE} ${bin}/accumulo-service \"$service\" \"$control_cmd\" \"-o\" \"general.process.bind.addr=$host\" $EXTRA_ARGS '"
- else
- debugAndRun "$SSH" "$host" "bash -c 'ACCUMULO_SERVICE_INSTANCE=${ACCUMULO_SERVICE_INSTANCE} ${bin}/accumulo-service \"$service\" \"$control_cmd\" \"-o\" \"general.process.bind.addr=$host\"'"
- fi
+ declare "COMPACTORS_PER_HOST_$group"="${NUM_COMPACTORS:-1}"
fi
done
- }
- function start_service() {
- control_service start "$@"
- }
+ # This version requires a compaction coordinator for compactors
+ if [[ -n $COMPACTOR_GROUPS && -z $COORDINATOR_HOSTS ]]; then
+ echo "$(yellow WARN): External compactor group(s) configured, but no coordinator configured"
+ fi
- function start_compactors() {
- if [[ -z $COMPACTION_QUEUES ]]; then
- echo "No compactor queues configured..."
+ if [[ -z $COMPACTOR_GROUPS ]]; then
+ # no compactor groups are required for this version
+ debug "No compactor groups configured"
else
- echo -n "Starting compactor servers ..."
- queues=$COMPACTION_QUEUES
- if [[ -n $1 ]]; then
- queues="$1"
- echo "Only starting servers for group: ${queues}"
- fi
- for queue in $queues; do
- var_name="NUM_COMPACTORS_${queue}"
- [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
- if [[ -n ${!Q} ]]; then
- for compactor in ${!Q}; do
- start_service "$compactor" compactor "-o" "compactor.queue=$queue"
- done
- else
- echo "${queue} is not a valid queue ...exiting"
+ for group in $COMPACTOR_GROUPS; do
+ G="COMPACTOR_HOSTS_$group"
+ if [[ -z ${!G} ]]; then
+ echo "$(yellow WARN): External compactor group $(yellow "$group") configured, but no compactors configured for it"
fi
done
fi
@@@ -286,331 -419,219 +410,219 @@@
fi
done
fi
- }
-
- function start_tservers() {
- echo -n "Starting tablet servers ..."
- count=1
- for server in $TSERVER_HOSTS; do
- echo -n "."
- start_service "$server" tserver &
- if ((++count % 72 == 0)); then
- echo
- wait
- fi
- done
- echo " done"
- }
- function start_all() {
- unset DISPLAY
+ manager1=$(echo "$MANAGER_HOSTS" | cut -d" " -f1)
- if [[ $1 != "--no-tservers" ]]; then
- start_tservers
+ if [[ -z $MONITOR_HOSTS ]]; then
+ echo "$(yellow WARN): monitors not found in $(yellow "$conf/cluster.yaml"), using first manager host $(green "$manager1")"
+ MONITOR_HOSTS=$manager1
fi
- for manager in $MANAGER_HOSTS; do
- start_service "$manager" manager
- done
+ if [[ -z $GC_HOSTS ]]; then
+ echo "$(yellow WARN): gc not found in $(yellow "$conf/cluster.yaml"), using first manager host $(green "$manager1")"
+ GC_HOSTS=$manager1
+ fi
- for gc in $GC_HOSTS; do
- start_service "$gc" gc
- done
+ }
- for monitor in $MONITOR_HOSTS; do
- start_service "$monitor" monitor
- done
+ function execute_command() {
+ control_cmd=$1
+ host=$2
+ service=$3
+ group=$4
+ shift 4
- for group in $SSERVER_GROUPS; do
- var_name="NUM_SSERVERS_${group}"
- [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
- G="SSERVER_HOSTS_${group}"
- for sserver in ${!G}; do
- start_service "$sserver" sserver "-o" "sserver.group=$group"
- done
- done
+ local S
+ local servers_per_host
- for coordinator in $COORDINATOR_HOSTS; do
- start_service "$coordinator" compaction-coordinator
- done
+ S="${service^^}S_PER_HOST_$group"
+ S="${S//-/_}" # replace dashes in service/group name with underscores
+ servers_per_host="${!S:-1}"
- for queue in $COMPACTION_QUEUES; do
- var_name="NUM_COMPACTORS_${queue}"
- [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
- for compactor in ${!Q}; do
- start_service "$compactor" compactor "-o" "compactor.queue=$queue"
- done
- done
+ if [[ $ARG_LOCAL == 1 ]]; then
- debugOrRun bash -c "ACCUMULO_CLUSTER_ARG=$servers_per_host \"$bin/accumulo-service\" $service $control_cmd -a $host $*"
++ debugOrRun bash -c "ACCUMULO_CLUSTER_ARG=$servers_per_host \"$bin/accumulo-service\" $service $control_cmd -o general.process.bind.addr=$host $*"
+ else
- debugOrRun "${SSH[@]}" "$host" "bash -c 'ACCUMULO_CLUSTER_ARG=$servers_per_host \"$bin/accumulo-service\" $service $control_cmd -a $host $*'"
++ debugOrRun "${SSH[@]}" "$host" "bash -c 'ACCUMULO_CLUSTER_ARG=$servers_per_host \"$bin/accumulo-service\" $service $control_cmd -o general.process.bind.addr=$host $*'"
+ fi
+ }
+ function get_localhost_addresses() {
+ local localaddresses
+ local localinterfaces
+ local x
+ if [[ -n $ACCUMULO_LOCALHOST_ADDRESSES ]]; then
+ read -r -a localaddresses <<<"$ACCUMULO_LOCALHOST_ADDRESSES"
+ else
+ read -r -a localinterfaces <<<"$(hostname -I)"
+ read -r -a localaddresses <<<"$(getent hosts 127.0.0.1 ::1 "${localinterfaces[@]}" | paste -sd' ')"
+ fi
+ for x in "${localaddresses[@]}"; do echo "$x"; done | sort -u
}
- function start_here() {
+ function control_services() {
+ unset DISPLAY
+ local operation=$1
- local_hosts="$(hostname -a 2>/dev/null) $(hostname) localhost 127.0.0.1 $(hostname -I)"
+ if [[ $operation != "start" && $operation != "stop" && $operation != "kill" ]]; then
+ echo "$(red ERROR): Invalid operation: $(yellow "$operation")"
+ exit 1
+ fi
- for host in $local_hosts; do
- for tserver in $TSERVER_HOSTS; do
- if echo "$tserver" | grep -q "^${host}\$"; then
- start_service "$host" tserver
- break
+ local tserver_groups
+ local addr
+ local group
+ local tserver
+ local G
+ if [[ $ARG_ALL == 1 && $ARG_LOCAL == 0 && $operation == "stop" ]]; then
+ echo "Stopping Accumulo cluster..."
+ if ! isDebug; then
+ if ! "$accumulo_cmd" admin stopAll; then
+ echo "Invalid password or unable to connect to the manager"
+ echo "Initiating forced shutdown in 15 seconds (Ctrl-C to abort)"
+ sleep 10
+ echo "Initiating forced shutdown in 5 seconds (Ctrl-C to abort)"
+ else
+ echo "Accumulo shut down cleanly"
+ echo "Utilities and unresponsive servers will shut down in 5 seconds (Ctrl-C to abort)"
fi
+ sleep 5
+ fi
+ elif [[ $ARG_LOCAL == 1 && $ARG_TSERVER == 1 && $operation == "stop" ]]; then
+ tserver_groups=$TSERVER_GROUPS
+ if [[ -n $ARG_TSERVER_GROUP ]]; then
+ tserver_groups=$ARG_TSERVER_GROUP
+ fi
+ for addr in "${LOCAL_HOST_ADDRESSES[@]}"; do
+ for group in $tserver_groups; do
+ G="TSERVER_HOSTS_$group"
+ for tserver in ${!G}; do
+ if echo "$tserver" | grep -q "$addr"; then
+ if ! isDebug; then
+ "$accumulo_cmd" admin stop "$addr"
+ else
+ debug "Stopping tservers on $addr via admin command"
+ fi
+ fi
+ done
+ done
done
- done
+ elif [[ $ARG_ALL == 1 && $operation == "kill" ]]; then
+ echo "Killing Accumulo cluster..."
+ fi
+
+ local count
+ local hosts
+ if [[ $ARG_ALL == 1 || $ARG_TSERVER == 1 ]]; then
+ tserver_groups=$TSERVER_GROUPS
+ if [[ -n $ARG_TSERVER_GROUP ]]; then
+ tserver_groups=$ARG_TSERVER_GROUP
+ fi
+ for group in $tserver_groups; do
+ local msg
+ local msgNoColor
+ msg="Executing $(green "$ARG_CMD") on $(purple tablet servers) for group $(yellow "$group") ..."
+ msgNoColor=$(decolorize "$msg")
+ count=${#msgNoColor}
+ ((count > 71)) && count=69 # only print up to 3 more dots if the line is too long
+ echo -n "$msg"
+ hosts="TSERVER_HOSTS_$group"
+ for tserver in ${!hosts}; do
+ if canRunOnHost "$tserver"; then
+ echo -n "."
+ execute_command "$operation" "$tserver" tserver "$group"
+ if ((++count % 72 == 0)); then
+ echo
+ wait
+ fi
+ fi
+ done
+ done
+ echo "done"
+ fi
- for host in $local_hosts; do
+ local manager
+ if [[ $ARG_ALL == 1 || $ARG_MANAGER == 1 ]]; then
for manager in $MANAGER_HOSTS; do
- if echo "$manager" | grep -q "^${host}\$"; then
- start_service "$host" manager
- break
+ if canRunOnHost "$manager"; then
+ echo "Executing $(green "$ARG_CMD") on $(purple managers)"
+ execute_command "$operation" "$manager" manager "default"
fi
done
- done
+ fi
- for host in $local_hosts; do
+ local gc
+ if [[ $ARG_ALL == 1 || $ARG_GC == 1 ]]; then
for gc in $GC_HOSTS; do
- if echo "$gc" | grep -q "^${host}\$"; then
- start_service "$host" gc
- break
+ if canRunOnHost "$gc"; then
+ echo "Executing $(green "$ARG_CMD") on $(purple garbage collectors)"
+ execute_command "$operation" "$gc" gc "default"
fi
done
- done
+ fi
- for host in $local_hosts; do
+ local monitor
+ if [[ $ARG_ALL == 1 || $ARG_MONITOR == 1 ]]; then
for monitor in $MONITOR_HOSTS; do
- if echo "$monitor" | grep -q "^${host}\$"; then
- start_service "$host" monitor
- break
+ if canRunOnHost "$monitor"; then
+ echo "Executing $(green "$ARG_CMD") on $(purple monitors)"
+ execute_command "$operation" "$monitor" monitor "default"
fi
done
- done
-
- for group in $SSERVER_GROUPS; do
- var_name="NUM_SSERVERS_${group}"
- [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
- for host in $local_hosts; do
- G="SSERVER_HOSTS_${group}"
- for sserver in ${!G}; do
- if echo "$sserver" | grep -q "^${host}\$"; then
- start_service "$sserver" sserver "-g" "$group"
- fi
- done
- done
- done
+ fi
- for host in $local_hosts; do
+ local coordinator
+ if [[ $ARG_ALL == 1 || $ARG_COORDINATOR == 1 ]]; then
for coordinator in $COORDINATOR_HOSTS; do
- if echo "$coordinator" | grep -q "^${host}\$"; then
- start_service "$coordinator" compaction-coordinator
+ if canRunOnHost "$coordinator"; then
+ echo "Executing $(green "$ARG_CMD") on $(purple compaction coordinators)"
+ execute_command "$operation" "$coordinator" compaction-coordinator "default"
fi
done
- done
+ fi
- for queue in $COMPACTION_QUEUES; do
- for host in $local_hosts; do
- var_name="NUM_COMPACTORS_${queue}"
- [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
- for compactor in ${!Q}; do
- if echo "$compactor" | grep -q "^${host}\$"; then
- start_service "$compactor" compactor "-q" "$queue"
+ local sserver_groups
+ local sserver
+ if [[ $ARG_ALL == 1 || $ARG_SSERVER == 1 ]]; then
+ sserver_groups=$SSERVER_GROUPS
+ if [[ -n $ARG_SSERVER_GROUP ]]; then
+ sserver_groups=$ARG_SSERVER_GROUP
+ fi
+ for group in $sserver_groups; do
+ echo "Executing $(green "$ARG_CMD") on $(purple scan servers) for group $(yellow "$group")"
+ hosts="SSERVER_HOSTS_$group"
+ for sserver in ${!hosts}; do
+ if canRunOnHost "$sserver"; then
- execute_command "$operation" "$sserver" sserver "$group" "-g" "$group"
++ execute_command "$operation" "$sserver" sserver "$group" "-o" "sserver.group=$group"
fi
done
done
- done
-
- }
-
- function end_service() {
- control_service "$@"
- }
-
- function stop_service() {
- end_service "stop" "$@"
- }
-
- function kill_service() {
- end_service "kill" "$@"
- }
-
- function stop_tservers() {
-
- echo "Stopping unresponsive tablet servers (if any)..."
- for host in $TSERVER_HOSTS; do
- stop_service "$host" tserver &
- done
-
- sleep 10
-
- echo "Stopping unresponsive tablet servers hard (if any)..."
- for host in $TSERVER_HOSTS; do
- kill_service "$host" tserver &
- done
-
- echo "Cleaning tablet server entries from zookeeper"
- if ! isDebug; then
- ${accumulo_cmd} org.apache.accumulo.server.util.ZooZap -tservers
- fi
- }
-
- function kill_all() {
- echo "Killing Accumulo cluster..."
-
- for manager in $MANAGER_HOSTS; do
- kill_service "$manager" manager
- done
-
- for gc in $GC_HOSTS; do
- kill_service "$gc" gc
- done
-
- for monitor in $MONITOR_HOSTS; do
- kill_service "$monitor" monitor
- done
-
- for group in $SSERVER_GROUPS; do
- var_name="NUM_SSERVERS_${group}"
- [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
- G="SSERVER_HOSTS_${group}"
- for sserver in ${!G}; do
- kill_service "$sserver" sserver "-o" "sserver.group=$group"
- done
- done
-
- for host in $TSERVER_HOSTS; do
- kill_service "$host" tserver
- done
-
- for coordinator in $COORDINATOR_HOSTS; do
- kill_service "$coordinator" compaction-coordinator
- done
-
- for queue in $COMPACTION_QUEUES; do
- var_name="NUM_COMPACTORS_${queue}"
- [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
- for compactor in ${!Q}; do
- kill_service "$compactor" compactor "-o" "compactor.queue=$queue"
- done
- done
-
- echo "Cleaning all server entries in ZooKeeper"
- if ! isDebug; then
- ${accumulo_cmd} org.apache.accumulo.server.util.ZooZap -manager -tservers -compaction-coordinators -compactors -sservers
fi
- }
- function stop_all() {
- echo "Stopping Accumulo cluster..."
- if ! isDebug; then
- if ! ${accumulo_cmd} admin stopAll; then
- echo "Invalid password or unable to connect to the manager"
- echo "Initiating forced shutdown in 15 seconds (Ctrl-C to abort)"
- sleep 10
- echo "Initiating forced shutdown in 5 seconds (Ctrl-C to abort)"
- else
- echo "Accumulo shut down cleanly"
- echo "Utilities and unresponsive servers will shut down in 5 seconds (Ctrl-C to abort)"
+ local compactor_groups
+ local compactor
+ if [[ $ARG_ALL == 1 || $ARG_COMPACTOR == 1 ]]; then
+ compactor_groups=$COMPACTOR_GROUPS
+ if [[ -n $ARG_COMPACTOR_GROUP ]]; then
+ compactor_groups=$ARG_COMPACTOR_GROUP
fi
- sleep 5
- fi
-
- # Look for processes not killed by 'admin stopAll'
-
- for end_cmd in "stop" "kill"; do
-
- for manager in $MANAGER_HOSTS; do
- end_service $end_cmd "$manager" manager
- done
-
- for gc in $GC_HOSTS; do
- end_service $end_cmd "$gc" gc
- done
-
- for monitor in $MONITOR_HOSTS; do
- end_service $end_cmd "$monitor" monitor
- done
-
- for group in $SSERVER_GROUPS; do
- var_name="NUM_SSERVERS_${group}"
- [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
- G="SSERVER_HOSTS_${group}"
- for sserver in ${!G}; do
- end_service $end_cmd "$sserver" sserver "-o" "sserver.group=$group"
- done
- done
-
- for coordinator in $COORDINATOR_HOSTS; do
- end_service $end_cmd "$coordinator" compaction-coordinator
- done
-
- for queue in $COMPACTION_QUEUES; do
- var_name="NUM_COMPACTORS_${queue}"
- [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
- for compactor in ${!Q}; do
- end_service $end_cmd "$compactor" compactor "-o" "compactor.queue=$queue"
+ for group in $compactor_groups; do
+ echo "Executing $(green "$ARG_CMD") on $(purple compactors) for group $(yellow "$group")"
+ hosts="COMPACTOR_HOSTS_$group"
+ for compactor in ${!hosts}; do
+ if canRunOnHost "$compactor"; then
- execute_command "$operation" "$compactor" compactor "$group" "-q" "$group"
++ execute_command "$operation" "$compactor" compactor "$group" "-o" "compactor.queue=$group"
+ fi
done
done
-
- done
-
- # stop tserver still running
- stop_tservers
-
- echo "Cleaning all server entries in ZooKeeper"
- if ! isDebug; then
- ${accumulo_cmd} org.apache.accumulo.server.util.ZooZap -manager -tservers -compaction-coordinators -compactors -sservers
fi
- }
-
- function stop_here() {
- # Determine hostname without errors to user
- hosts_to_check=("$(hostname -a 2>/dev/null | head -1)" "$(hostname -f)")
- if echo "${TSERVER_HOSTS}" | grep -Eq 'localhost|127[.]0[.]0[.]1'; then
+ if [[ $ARG_LOCAL == 0 && $ARG_ALL == 1 && ($operation == "stop" || $operation == "kill") ]]; then
if ! isDebug; then
- ${accumulo_cmd} admin stop localhost
- else
- debug "Stopping tservers on localhost via admin command"
+ echo "Cleaning all server entries in ZooKeeper"
+ "$accumulo_cmd" org.apache.accumulo.server.util.ZooZap -manager -tservers -compaction-coordinators -compactors -sservers
fi
- else
- for host in "${hosts_to_check[@]}"; do
- for tserver in $TSERVER_HOSTS; do
- if echo "$tserver" | grep -q "$host"; then
- if ! isDebug; then
- ${accumulo_cmd} admin stop "$host"
- else
- debug "Stopping tservers on $host via admin command"
- fi
- fi
- done
- done
fi
- for host in "${hosts_to_check[@]}"; do
- for end_cmd in "stop" "kill"; do
- for svc in tserver gc manager monitor compaction-coordinator; do
- end_service $end_cmd "$host" $svc
- done
- for group in $SSERVER_GROUPS; do
- var_name="NUM_SSERVERS_${group}"
- [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
- G="SSERVER_HOSTS_${group}"
- for sserver in ${!G}; do
- end_service $end_cmd "$sserver" sserver "-o" "sserver.group=$group"
- done
- done
- for queue in $COMPACTION_QUEUES; do
- var_name="NUM_COMPACTORS_${queue}"
- [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
- for compactor in ${!Q}; do
- end_service $end_cmd "$host" compactor "-o" "compactor.queue=$queue"
- done
- done
- done
- done
}
function main() {
diff --cc assemble/bin/accumulo-service
index c8258257ca,2bc754c212..ad9f96c65d
--- a/assemble/bin/accumulo-service
+++ b/assemble/bin/accumulo-service
@@@ -59,6 -60,24 +59,18 @@@ function rotate_log()
fi
}
+ function get_group() {
+ # Find the group parameter if any
+ local group="default"
- local group_param=""
+ local param
+ for param in "$@"; do
- if [[ -n $group_param ]]; then
- # grab the group if the previous arg was the group param
- group="$param"
- break
- elif [[ $param == '-q' || $param == '-g' ]]; then
- # found the group parameter, the next arg is the group
- group_param=$param
++ if [[ $param =~ ^[a-z]*[.]group=(.*)$ ]]; then
++ group="${BASH_REMATCH[1]}"
+ fi
+ done
+ echo "$group"
+ }
+
function start_service() {
local service_type=$1
local service_name=$2
@@@ -199,11 -265,16 +258,11 @@@ function main()
local service_type="$1"
local command_name="$2"
shift 2
- local service_name=$service_type
+ local service_name=""
local all_flag=false
- if [[ $service_type == "master" ]]; then
- echo "WARN : Use of 'master' service name is deprecated; use 'manager' instead."
- service_type="manager"
- fi
-
# Check and see if accumulo-cluster is calling this script
- if [[ -z $ACCUMULO_SERVICE_INSTANCE ]]; then
+ if [[ -z $ACCUMULO_CLUSTER_ARG ]]; then
# The rest of the arguments are from a user
if [[ $1 == "--all" ]]; then
all_flag=true
diff --cc assemble/conf/accumulo-env.sh
index 9e4acc1786,dc1799c7fc..db481a056c
--- a/assemble/conf/accumulo-env.sh
+++ b/assemble/conf/accumulo-env.sh
@@@ -89,15 -89,25 +89,25 @@@ JAVA_OPTS=
## JVM options set for individual applications
# cmd is set by calling script that sources this env file
#shellcheck disable=SC2154
- case "$cmd" in
- manager) JAVA_OPTS=('-Xmx512m' '-Xms512m' "${JAVA_OPTS[@]}") ;;
- monitor) JAVA_OPTS=('-Xmx256m' '-Xms256m' "${JAVA_OPTS[@]}") ;;
- gc) JAVA_OPTS=('-Xmx256m' '-Xms256m' "${JAVA_OPTS[@]}") ;;
- tserver) JAVA_OPTS=('-Xmx768m' '-Xms768m' "${JAVA_OPTS[@]}") ;;
- compaction-coordinator) JAVA_OPTS=('-Xmx512m' '-Xms512m' "${JAVA_OPTS[@]}") ;;
- compactor) JAVA_OPTS=('-Xmx256m' '-Xms256m' "${JAVA_OPTS[@]}") ;;
- sserver) JAVA_OPTS=('-Xmx512m' '-Xms512m' "${JAVA_OPTS[@]}") ;;
- *) JAVA_OPTS=('-Xmx256m' '-Xms64m' "${JAVA_OPTS[@]}") ;;
+ case "${ACCUMULO_RESOURCE_GROUP:-default}" in
+ default)
+ # shellcheck disable=SC2154
+ # $cmd is exported in the accumulo script, but not the accumulo-service script
+ case "$cmd" in
- manager | master) JAVA_OPTS=('-Xmx512m' '-Xms512m' "${JAVA_OPTS[@]}") ;;
++ manager) JAVA_OPTS=('-Xmx512m' '-Xms512m' "${JAVA_OPTS[@]}") ;;
+ monitor) JAVA_OPTS=('-Xmx256m' '-Xms256m' "${JAVA_OPTS[@]}") ;;
+ gc) JAVA_OPTS=('-Xmx256m' '-Xms256m' "${JAVA_OPTS[@]}") ;;
+ tserver) JAVA_OPTS=('-Xmx768m' '-Xms768m' "${JAVA_OPTS[@]}") ;;
+ compaction-coordinator) JAVA_OPTS=('-Xmx512m' '-Xms512m' "${JAVA_OPTS[@]}") ;;
+ compactor) JAVA_OPTS=('-Xmx256m' '-Xms256m' "${JAVA_OPTS[@]}") ;;
+ sserver) JAVA_OPTS=('-Xmx512m' '-Xms512m' "${JAVA_OPTS[@]}") ;;
+ *) JAVA_OPTS=('-Xmx256m' '-Xms64m' "${JAVA_OPTS[@]}") ;;
+ esac
+ ;;
+ *)
+ echo "ACCUMULO_RESOURCE_GROUP named $ACCUMULO_RESOURCE_GROUP is not configured in accumulo-env.sh"
+ exit 1
+ ;;
esac
## JVM options set for logging. Review log4j2.properties file to see how they are used.