[kudu] 01/02: [docker] KUDU-3307: allow using multiple directories

[aw...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

awong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit e7cd1f6cfab4d8448ad4ebd834727c83f23f4899
Author: Andrew Wong <aw...@cloudera.com>
AuthorDate: Wed Jul 28 15:54:29 2021 -0700

    [docker] KUDU-3307: allow using multiple directories
    
    Previously, the Docker entrypoint would by default use a single root
    directory, suffixed with either 'master' or 'tserver'. This makes it a
    poor fit for production systems that may want to use Docker, since most
    reasonable deployments use multple data directories, and don't colocate
    WALs and data.
    
    This patch adds a couple of environment variables to the entrypoint,
    FS_WAL_DIR and FS_DATA_DIRS, that map directly to the --fs_wal_dir and
    --fs_data_dirs gflags that Kudu masters and tservers know and love.
    
    To maintain compatibility with existing guides, the existing single-root
    DATA_DIR remains, but the new environment variables, if set, take
    precedence.
    
    Change-Id: I20e5f75c1f1f8280dca60de75255a4a948e44be1
    Reviewed-on: http://gerrit.cloudera.org:8080/17738
    Tested-by: Kudu Jenkins
    Reviewed-by: Bankim Bhavsar <ba...@cloudera.com>
---
 docker/kudu-entrypoint.sh | 49 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 43 insertions(+), 6 deletions(-)

diff --git a/docker/kudu-entrypoint.sh b/docker/kudu-entrypoint.sh
index 814bf0d..21485db 100755
--- a/docker/kudu-entrypoint.sh
+++ b/docker/kudu-entrypoint.sh
@@ -39,8 +39,18 @@ function print_help {
   echo "  Defines the kudu-master and kudu-tserver configured master addresses."
   echo "  Defaults to localhost."
   echo "DATA_DIR:"
-  echo "  Defines the root data directory to use."
+  echo "  Defines the root directory to use. Subdirectories are added depending on whether a "
+  echo "  Kudu master or a Kudu tablet server is being deployed. Ignored if the FS_WAL_DIR "
+  echo "  environment variable is set."
+  echo "  NOTE: this variable is deprecated. FS_WAL_DIR should be used instead."
   echo "  Defaults to /var/lib/kudu."
+  echo "FS_WAL_DIR:"
+  echo "  Defines the WAL directory to use. Takes precedence over the DATA_DIR environment "
+  echo "  variable."
+  echo "FS_DATA_DIRS:"
+  echo "  Defines the data directories to use. If set, the FS_WAL_DIR environment variable "
+  echo "  must also be set."
+  echo "  Defaults to the value of the FS_WAL_DIR environment variable."
   echo "MASTER_ARGS:"
   echo "  Defines the arguments passed to kudu-master."
   echo "  Defaults to the value of the DEFAULT_ARGS environment variable."
@@ -51,12 +61,29 @@ function print_help {
   echo "  Defines a recommended base set of arguments."
 }
 
+if [[ -z "$FS_WAL_DIR" && -n "$FS_DATA_DIRS" ]]; then
+  echo "If FS_DATA_DIRS is set, FS_WAL_DIR must also be set"
+  echo "FS_WAL_DIR: $FS_WAL_DIR"
+  echo "FS_DATA_DIRS: $FS_DATA_DIRS"
+  exit 1
+fi
+
+DATA_DIR=${DATA_DIR:="/var/lib/kudu"}
+if [[ -n "$FS_WAL_DIR" ]]; then
+  # Use the WAL directory for data if a data directory is not specified.
+  WAL_DIR="$FS_WAL_DIR"
+  DATA_DIRS=${FS_DATA_DIRS:="$FS_WAL_DIR"}
+else
+  # If no WAL directory is specified, use a subdirectory in the root directory.
+  WAL_DIR="$DATA_DIR/$1"
+  DATA_DIRS="$DATA_DIR/$1"
+fi
+
 # Define the defaults environment variables.
 KUDU_MASTERS=${KUDU_MASTERS:=""}
-DATA_DIR=${DATA_DIR:="/var/lib/kudu"}
-SERVICE_DIR="$DATA_DIR/$1"
  # TODO: Remove use_hybrid_clock=false when ntpd is setup.
-DEFAULT_ARGS="--fs_wal_dir=$SERVICE_DIR \
+DEFAULT_ARGS="--fs_wal_dir=$WAL_DIR \
+ --fs_data_dirs=$DATA_DIRS \
  --webserver_doc_root=/opt/kudu/www \
  --stderrthreshold=0 \
  --use_hybrid_clock=false"
@@ -85,6 +112,16 @@ function wait_for_master_hosts() {
   unset IFS
 }
 
+function make_directories() {
+  IFS=","
+  mkdir -p $WAL_DIR
+  for DIR in $DATA_DIRS
+  do
+    mkdir -p $DIR
+  done
+  unset IFS
+}
+
 # If no arguments are passed, print the help.
 if [[ -z "$1" ]]; then
   print_help
@@ -94,7 +131,7 @@ fi
 # Note: we use "master" and "tserver" here so the kudu-master and kudu-tserver
 # binaries can be manually invoked if needed.
 if [[ "$1" == "master" ]]; then
-  mkdir -p "$SERVICE_DIR"
+  make_directories
   wait_for_master_hosts
   # Supply --master_addresses even if a single master address is specified.
   if [[ -n "$KUDU_MASTERS" ]]; then
@@ -102,7 +139,7 @@ if [[ "$1" == "master" ]]; then
   fi
   exec kudu master run ${DEFAULT_ARGS} ${MASTER_ARGS}
 elif [[ "$1" == "tserver" ]]; then
-  mkdir -p "$SERVICE_DIR"
+  make_directories
   wait_for_master_hosts
   if [[ -n "$KUDU_MASTERS" ]]; then
     TSERVER_ARGS="--tserver_master_addrs=$KUDU_MASTERS $TSERVER_ARGS"