[5/5] impala git commit: IMPALA-6791: distcc server setup script

[ta...@apache.org]

IMPALA-6791: distcc server setup script

Adds a script that can be run to configure a working distcc server with
ccache. This is based roughly on the manual steps in the README.

Testing:
Tested the scripts on Ubuntu and CentOS 6 systems. They resulted in a
working distcc server.

Change-Id: I9a145911f095eb8e173694475cc2dac65eb7c7bb
Reviewed-on: http://gerrit.cloudera.org:8080/9901
Reviewed-by: Taras Bobrovytsky <tb...@cloudera.com>
Tested-by: Tim Armstrong <ta...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/cf76953c
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/cf76953c
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/cf76953c

Branch: refs/heads/master
Commit: cf76953cc6cd1e464ff403089498a3ab55c12847
Parents: c55c52b
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Mon Apr 2 17:35:05 2018 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Tue Apr 3 23:45:03 2018 +0000

----------------------------------------------------------------------
 bin/distcc/README.md              |  19 +++++-
 bin/distcc/distcc_server_setup.sh | 113 +++++++++++++++++++++++++++++++++
 2 files changed, 131 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/impala/blob/cf76953c/bin/distcc/README.md
----------------------------------------------------------------------
diff --git a/bin/distcc/README.md b/bin/distcc/README.md
index 4ebca45..a8487b2 100644
--- a/bin/distcc/README.md
+++ b/bin/distcc/README.md
@@ -26,6 +26,14 @@ The rest of the setup is done for you; here is a short description of what they
 # Usage
 
 ### First time
+1. Symlink /opt/Impala-Toolchain to a directory writable by your user and point
+  IMPALA_TOOLCHAIN at that directory. This ensures toolchain binaries are at the
+  same path locally as on the distcc servers
+
+        mkdir -p "$IMPALA_TOOLCHAIN"/toolchain
+        sudo ln -s "$IMPALA_TOOLCHAIN"/toolchain /opt/Impala-Toolchain
+        echo 'export IMPALA_TOOLCHAIN=/opt/Impala-Toolchain' >> bin/impala-config-local.sh
+
 1. Source bin/impala-config.sh in the Impala repo. Step #2 depends on this.
 
         source "$IMPALA_HOME"/bin/impala-config.sh
@@ -70,7 +78,16 @@ source "$IMPALA_HOME"/bin/impala-config.sh   # Skip if already done
 source "$IMPALA_HOME"/bin/distcc/distcc_env.sh
 ```
 
-# Setting up a new distcc server
+# Setting up a new distcc server (automatic)
+1. Run distcc_server_setup.sh to do initial setup for a server that accepts connections
+   from the 172.\* private IP address block:
+  sudo ./bin/distcc/distcc_server_setup.sh 172.16.0.0/12
+1. Download toolchain packages. This is run initially and can be rerun to download
+  new packages that a new version of Impala depends on.
+  (. ./bin/impala-config.sh && ./infra/python/deps/download_requirements &&
+   DOWNLOAD_CDH_COMPONENTS=false ./bin/bootstrap_toolchain.py)
+
+# Setting up a new distcc server (manual)
 
 1. Install "distccd" and "ccache".
 1. Configure distccd (edit /etc/sysconfig/distccd on a RHEL server) with the options

http://git-wip-us.apache.org/repos/asf/impala/blob/cf76953c/bin/distcc/distcc_server_setup.sh
----------------------------------------------------------------------
diff --git a/bin/distcc/distcc_server_setup.sh b/bin/distcc/distcc_server_setup.sh
new file mode 100755
index 0000000..893a82d
--- /dev/null
+++ b/bin/distcc/distcc_server_setup.sh
@@ -0,0 +1,113 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script automates setup of distcc servers on Ubuntu and CentOS. It has been tested
+# on Ubuntu 14.04 and 16.04 and CentOS 6 and 7. See bin/distcc/README.md for manual setup
+# instructions and background.
+#
+# Usage:
+# ------
+# The script expects to be run as root and requires that the user specify a range of
+# IPs that it should accept connections from. E.g. To configure a distcc server that
+# accepts connections from the 172.* private IP address block. The IP address argument
+# is forwarded to the --allow flag of distccd.
+#
+#   sudo ./bin/distcc/distcc_server_setup.sh 172.16.0.0/12
+#
+# Environment overrides:
+# ---------------------
+# CCACHE_DIR: directory to use for distccd's ccache.
+# CCACHE_SIZE: size of ccache, passed to ccache's -M option
+set -eu -o pipefail
+trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+
+if [[ $# != 1 ]]; then
+  echo "Usage: $0 <allowed IP address range>"
+  exit 1
+fi
+set -x
+ALLOWED_NETS=$1
+
+# Find the absolute path to repo root.
+IMPALA_HOME=$(cd "$(dirname "$0")/../.."; pwd)
+
+LSB_ID=$(lsb_release -is)
+LSB_VERSION=$(lsb_release -rs)
+if [[ "$LSB_ID" == Ubuntu ]]; then
+  if ! [[ $LSB_VERSION == 14.04 || $LSB_VERSION == 16.04 ]]; then
+    echo "This script only supports Ubuntu 14.04 and 16.04" >&2
+    exit 1
+  fi
+  LINUX_FLAVOUR=ubuntu
+  DISTCCD_USER=distccd
+  DISTCCD_SERVICE=distcc
+elif [[ "$LSB_ID" == CentOS ]]; then
+  if ! [[ $LSB_VERSION == 6.* || $LSB_VERSION = 7.* ]]; then
+    echo "This script only supports CentOS 6 and 7" >&2
+    exit 1
+  fi
+  LINUX_FLAVOUR=redhat
+  DISTCCD_USER=nobody
+  DISTCCD_SERVICE=distccd
+else
+  echo "This script only supports Ubuntu and CentOS" >&2
+  exit 1
+fi
+
+echo "Installing required packages"
+if [[ $LINUX_FLAVOUR = ubuntu ]]; then
+  apt-get install --yes distcc ccache
+else
+  yum install -y distcc-server ccache
+fi
+
+echo "Configuring ccache for distccd user"
+export CCACHE_DIR=${CCACHE_DIR-/opt/ccache}
+CCACHE_SIZE=${CCACHE_SIZE:-25G}
+mkdir -p "${CCACHE_DIR}"
+chown ${DISTCCD_USER} "${CCACHE_DIR}"
+HOME=${CCACHE_DIR} sudo -E -u ${DISTCCD_USER} ccache -M "${CCACHE_SIZE}"
+
+echo "Configuring distcc"
+if [[ $LINUX_FLAVOUR = ubuntu ]]; then
+  cat << EOF >> /etc/default/distcc
+# BEGIN: Settings automatically generated by distcc_server_setup.sh
+STARTDISTCC="true"
+ALLOWEDNETS="${ALLOWED_NETS}"
+LISTENER="0.0.0.0"
+JOBS=$(nproc)
+# CCACHE_DIR is picked up by ccache from environment.
+export CCACHE_DIR=${CCACHE_DIR}
+# END: Settings automatically generated by distcc_server_setup.sh
+EOF
+else
+  cat << EOF >> /etc/sysconfig/distccd
+# BEGIN: Settings automatically generated by distcc_server_setup.sh
+OPTIONS="--jobs $(($(nproc) * 2)) --allow ${ALLOWED_NETS} --log-level=warn --nice=-15"
+# CCACHE_DIR is picked up by ccache from environment. CentOS 6 requires CCACHE_DIR to
+# be exported while CentOS 7 seems to ignore the "export VAR=val" syntax.
+CCACHE_DIR=${CCACHE_DIR}
+export CCACHE_DIR
+# END: Settings automatically generated by distcc_server_setup.sh
+EOF
+fi
+service ${DISTCCD_SERVICE} restart
+
+echo "Symlinking /opt/Impala-Toolchain to default toolchain location"
+ln -f -s -T "${IMPALA_HOME}/toolchain" /opt/Impala-Toolchain