You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2018/02/24 16:33:53 UTC
[2/3] drill git commit: DRILL-6068: Support user/distrib-specific
config checks during startup
DRILL-6068: Support user/distrib-specific config checks during startup
1. Allows for distrib/user specific checks to be done
2. Place-holder files for distribution and user specific checks
3. Moved JVM Version Check to head of script
Separation of setups into 3 levels
1. Apache Drill (Default)
2. Distribition Specific
3. User-Defined
1 & 2 are mutually exclusive.
3 is additional checks that the user can specify.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/266250cf
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/266250cf
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/266250cf
Branch: refs/heads/master
Commit: 266250cf6ca2269f8772796d6c4eb903aa795111
Parents: 27aa236
Author: Kunal Khatua <kk...@maprtech.com>
Authored: Thu Jan 11 15:45:04 2018 -0800
Committer: Aman Sinha <as...@maprtech.com>
Committed: Fri Feb 23 17:55:06 2018 -0800
----------------------------------------------------------------------
distribution/src/assemble/bin.xml | 15 ++
distribution/src/resources/auto-setup.sh | 29 ++++
distribution/src/resources/distrib-setup.sh | 29 ++++
distribution/src/resources/drill-config.sh | 178 +++++++++++++++--------
distribution/src/resources/drill-setup.sh | 26 ++++
5 files changed, 217 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index b4290bb..bed34ed 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -347,6 +347,21 @@
<outputDirectory>conf</outputDirectory>
</file>
<file>
+ <source>src/resources/auto-setup.sh</source>
+ <fileMode>0755</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/drill-setup.sh</source>
+ <fileMode>0755</fileMode>
+ <outputDirectory>conf</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/distrib-setup.sh</source>
+ <fileMode>0755</fileMode>
+ <outputDirectory>conf</outputDirectory>
+ </file>
+ <file>
<source>src/resources/drill-override-example.conf</source>
<outputDirectory>conf</outputDirectory>
</file>
http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/auto-setup.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/auto-setup.sh b/distribution/src/resources/auto-setup.sh
new file mode 100644
index 0000000..75bdda0
--- /dev/null
+++ b/distribution/src/resources/auto-setup.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is invoked by drill-config.sh during a Drillbit startup and provides
+# default checks and autoconfiguration.
+# Distributions should not put anything in this file. Checks can be
+# specified in ${DRILL_HOME}/conf/distrib-setup.sh
+# Users should not put anything in this file. Additional checks can be defined
+# and put in ${DRILL_CONF_DIR}/drill-setup.sh instead.
+# To FAIL any check, return with a non-zero return code
+# e.g.
+# if [ $status == "FAILED" ]; return 1; fi
+
+###==========================================================================
+# FEATURES (Added here to describe supported additions)
+###==========================================================================
http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/distrib-setup.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/distrib-setup.sh b/distribution/src/resources/distrib-setup.sh
new file mode 100644
index 0000000..dbd2b90
--- /dev/null
+++ b/distribution/src/resources/distrib-setup.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is empty by default. Drill startup checks will be
+# performed in drill-config.sh. Distributions can replace this file with a
+# distribution-specific version that checks (and sets) environment variables
+# and options specific to that distribution.
+# Users should not put anything in this file. Additional checks can be defined
+# and put in drill-setup.sh instead.
+# To FAIL any check, return with a non-zero return code
+# e.g.
+# if [ $status == "FAILED" ]; return 1; fi
+
+###==========================================================================
+# FEATURES (Added here to describe supported additions)
+###==========================================================================
http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/drill-config.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh
index 2341540..d92031f 100644
--- a/distribution/src/resources/drill-config.sh
+++ b/distribution/src/resources/drill-config.sh
@@ -64,6 +64,77 @@ while [ -h "$this" ]; do
fi
done
+# JVM and Operating System Platform information
+# Test for cygwin
+is_cygwin=false
+case "`uname`" in
+CYGWIN*) is_cygwin=true;;
+esac
+
+if $is_cygwin; then
+ JAVA_BIN="java.exe"
+else
+ JAVA_BIN="java"
+fi
+
+# Test for or find JAVA_HOME
+
+if [ -z "$JAVA_HOME" ]; then
+ SOURCE=`which java`
+ if [ -e $SOURCE ]; then
+ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ # if $SOURCE was a relative symlink, we need to resolve it relative
+ # to the path where the symlink file was located
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
+ done
+ JAVA_HOME="$( cd -P "$( dirname "$SOURCE" )" && cd .. && pwd )"
+ JAVA=$SOURCE
+ fi
+ # if we didn't set it
+ if [ -z "$JAVA_HOME" ]; then
+ fatal_error "JAVA_HOME is not set and Java could not be found"
+ fi
+fi
+
+# Now, verify that 'java' binary exists and is suitable for Drill.
+# If we started with `which java` above, use that path (after replacing
+# symlinks.) If we started with JAVA_HOME, try in bin. Doing so handles
+# the case in which JAVA_HOME is a JDK that has a nested JRE; we prefer
+# the JDK bin. Finally, if nothing else works, just search for the
+# executable.
+
+if [ -z "$JAVA" ]; then
+ JAVA="$JAVA_HOME/bin/$JAVA_BIN"
+ if [[ ! -e $JAVA ]]; then
+ JAVA=`find -L "$JAVA_HOME" -name $JAVA_BIN -type f | head -n 1`
+ fi
+fi
+if [ ! -e "$JAVA" ]; then
+ fatal_error "Java not found at JAVA_HOME=$JAVA_HOME."
+fi
+
+# Ensure that Java version is at least 1.7
+"$JAVA" -version 2>&1 | grep "version" | egrep -e "1\.4|1\.5|1\.6" > /dev/null
+if [ $? -eq 0 ]; then
+ fatal_error "Java 1.7 or later is required to run Apache Drill."
+fi
+
+# Check if a file exists and has relevant lines for execution
+# Help in deciding which of the auto scripts to execute: distrib/drill
+function checkExecutableLineCount() {
+ local lineCount=0
+ #Empty input
+ if [ -z "$1" ]; then echo $lineCount; return; fi
+ #Not valid file
+ if [ ! -f "$1" ]; then echo $lineCount; return; fi
+ # Filtering out commented and empty lines
+ let lineCount=`cat $1 | grep -v '#' | grep -v ^$ | wc -l`
+ echo $lineCount
+ return
+}
+
# convert relative path to absolute path
bin=`dirname "$this"`
script=`basename "$this"`
@@ -180,18 +251,61 @@ else
fi
fi
-# Default memory settings if none provided by the environment or
+# Execute distrib-setup.sh for any distribution-specific setup (e.g. checks).
+# distrib-setup.sh is optional; it is created by some distribution installers
+# that need additional distribution-specific setup to be done.
+# Because installers will have site-specific steps, the file
+# should be moved into the site directory, if the user employs one.
+
+# Checking if being executed in context of Drillbit and not SQLLine
+if [ "$DRILLBIT_CONTEXT" == "1" ]; then
+ # Check whether to run exclusively distrib-setup.sh OR auto-setup.sh
+ distribSetup="$DRILL_CONF_DIR/distrib-setup.sh" ; #Site-based distrib-setup.sh
+ if [ $(checkExecutableLineCount $distribSetup) -eq 0 ]; then
+ distribSetup="$DRILL_HOME/conf/distrib-setup.sh" ; #Install-based distrib-setup.sh
+ if [ $(checkExecutableLineCount $distribSetup) -eq 0 ]; then
+ # Run Default Auto Setup
+ distribSetup="$DRILL_HOME/bin/auto-setup.sh"
+ fi
+ fi
+ # Check and run additional setup defined by user
+ drillSetup="$DRILL_CONF_DIR/drill-setup.sh" ; #Site-based drill-setup.sh
+ if [ $(checkExecutableLineCount $drillSetup) -eq 0 ]; then
+ drillSetup="$DRILL_HOME/conf/drill-setup.sh" ; #Install-based drill-setup.sh
+ if [ $(checkExecutableLineCount $drillSetup) -eq 0 ]; then drillSetup=""; fi
+ fi
+
+ # Enforcing checks in order (distrib-setup.sh , drill-setup.sh)
+ # (NOTE: A script is executed only if it has relevant executable lines)
+ # Both distribSetup & drillSetup are executed because the user might have introduced additional checks
+ if [ -n "$distribSetup" ]; then
+ . "$distribSetup"
+ if [ $? -gt 0 ]; then fatal_error "Aborting Drill Startup due failed setup by $distribSetup"; fi
+ fi
+ if [ -n "$drillSetup" ] && [ $(checkExecutableLineCount $drillSetup) -gt 0 ]; then
+ . "$drillSetup"
+ if [ $? -gt 0 ]; then fatal_error "Aborting Drill Startup due failed setup from $drillSetup"; fi
+ fi
+fi
+
+# DEFAULT memory settings if none provided by the environment or
# above config files.
# The Drillbit needs a large code cache.
-
export DRILL_MAX_DIRECT_MEMORY=${DRILL_MAX_DIRECT_MEMORY:-"8G"}
export DRILL_HEAP=${DRILL_HEAP:-"4G"}
-export DRILLBIT_MAX_PERM=${DRILLBIT_MAX_PERM:-"512M"}
export DRILLBIT_CODE_CACHE_SIZE=${DRILLBIT_CODE_CACHE_SIZE:-"1G"}
+export DRILLBIT_MAX_PERM=${DRILLBIT_MAX_PERM:-"512M"}
export DRILLBIT_OPTS="-Xms$DRILL_HEAP -Xmx$DRILL_HEAP -XX:MaxDirectMemorySize=$DRILL_MAX_DIRECT_MEMORY"
export DRILLBIT_OPTS="$DRILLBIT_OPTS -XX:ReservedCodeCacheSize=$DRILLBIT_CODE_CACHE_SIZE -Ddrill.exec.enable-epoll=false"
-export DRILLBIT_OPTS="$DRILLBIT_OPTS -XX:MaxPermSize=$DRILLBIT_MAX_PERM"
+
+# Remove MaxPermSize for JVM version >= 1.8.0
+jvmVersion=`"$JAVA" -version 2>&1 | grep "version" | tr '"_' '\n' | grep '[0-9].[0.9]'`
+if [[ $jvmVersion < "1.8.0" ]]; then
+ export DRILLBIT_OPTS="$DRILLBIT_OPTS -XX:MaxPermSize=$DRILLBIT_MAX_PERM"
+else
+ unset DRILLBIT_MAX_PERM
+fi
# Under YARN, the log directory is usually YARN-provided. Replace any
# value that may have been set in drill-env.sh.
@@ -364,62 +478,6 @@ if [[ ! -d "$DRILL_TMP_DIR" || ! -w "$DRILL_TMP_DIR" ]]; then
fatal_error "Temporary directory does not exist or is not writable: $DRILL_TMP_DIR"
fi
-# Test for cygwin
-is_cygwin=false
-case "`uname`" in
-CYGWIN*) is_cygwin=true;;
-esac
-
-if $is_cygwin; then
- JAVA_BIN="java.exe"
-else
- JAVA_BIN="java"
-fi
-
-# Test for or find JAVA_HOME
-
-if [ -z "$JAVA_HOME" ]; then
- SOURCE=`which java`
- if [ -e $SOURCE ]; then
- while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- # if $SOURCE was a relative symlink, we need to resolve it relative
- # to the path where the symlink file was located
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
- done
- JAVA_HOME="$( cd -P "$( dirname "$SOURCE" )" && cd .. && pwd )"
- JAVA=$SOURCE
- fi
- # if we didn't set it
- if [ -z "$JAVA_HOME" ]; then
- fatal_error "JAVA_HOME is not set and Java could not be found"
- fi
-fi
-
-# Now, verify that 'java' binary exists and is suitable for Drill.
-# If we started with `which java` above, use that path (after replacing
-# symlinks.) If we started with JAVA_HOME, try in bin. Doing so handles
-# the case in which JAVA_HOME is a JDK that has a nested JRE; we prefer
-# the JDK bin. Finally, if nothing else works, just search for the
-# executable.
-
-if [ -z "$JAVA" ]; then
- JAVA="$JAVA_HOME/bin/$JAVA_BIN"
- if [[ ! -e $JAVA ]]; then
- JAVA=`find -L "$JAVA_HOME" -name $JAVA_BIN -type f | head -n 1`
- fi
-fi
-if [ ! -e "$JAVA" ]; then
- fatal_error "Java not found at JAVA_HOME=$JAVA_HOME."
-fi
-
-# Ensure that Java version is at least 1.7
-"$JAVA" -version 2>&1 | grep "version" | egrep -e "1\.4|1\.5|1\.6" > /dev/null
-if [ $? -eq 0 ]; then
- fatal_error "Java 1.7 or later is required to run Apache Drill."
-fi
-
# Adjust paths for CYGWIN
if $is_cygwin; then
DRILL_HOME=`cygpath -w "$DRILL_HOME"`
http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/drill-setup.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-setup.sh b/distribution/src/resources/drill-setup.sh
new file mode 100644
index 0000000..f899b65
--- /dev/null
+++ b/distribution/src/resources/drill-setup.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is empty by default. It provides user-specific startup checks and
+# is invoked by drill-config.sh after any distribution-specific checks (and
+# setting of environment variables) have been performed.
+# To FAIL any check, return with a non-zero return code
+# e.g.
+# if [ $status == "FAILED" ]; return 1; fi
+
+###==========================================================================
+# FEATURES (Added here to describe supported additions)
+###==========================================================================