You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yi...@apache.org on 2023/06/27 00:59:14 UTC
[spark-docker] branch master updated: [SPARK-44177] Add 'set -eo pipefail' to entrypoint and quote variables
This is an automated email from the ASF dual-hosted git repository.
yikun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-docker.git
The following commit(s) were added to refs/heads/master by this push:
new 6022289 [SPARK-44177] Add 'set -eo pipefail' to entrypoint and quote variables
6022289 is described below
commit 60222892836549f05c56edd49ac81c688c8e7356
Author: Yikun Jiang <yi...@gmail.com>
AuthorDate: Tue Jun 27 08:59:03 2023 +0800
[SPARK-44177] Add 'set -eo pipefail' to entrypoint and quote variables
### What changes were proposed in this pull request?
Add 'set -eo pipefail' to entrypoint and quote variables
### Why are the changes needed?
Address DOI comments:
1. Have you considered a set -eo pipefail on the entrypoint script to help prevent any errors from being silently ignored?
2. You probably want to quote this (and many of the other variables in this execution); ala --driver-url "$SPARK_DRIVER_URL"
[1] https://github.com/docker-library/official-images/pull/13089#issuecomment-1601334895
[2] https://github.com/docker-library/official-images/pull/13089#issuecomment-1601813499
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
CI passed
Closes #49 from Yikun/quote.
Authored-by: Yikun Jiang <yi...@gmail.com>
Signed-off-by: Yikun Jiang <yi...@gmail.com>
---
3.4.0/scala2.12-java11-ubuntu/entrypoint.sh | 31 ++++++++++++++++-------------
3.4.1/scala2.12-java11-ubuntu/entrypoint.sh | 31 ++++++++++++++++-------------
entrypoint.sh.template | 31 ++++++++++++++++-------------
3 files changed, 51 insertions(+), 42 deletions(-)
diff --git a/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh b/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh
index 08fc925..2e3d2a8 100755
--- a/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh
+++ b/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh
@@ -15,6 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+# Prevent any errors from being silently ignored
+set -eo pipefail
+
attempt_setup_fake_passwd_entry() {
# Check whether there is a passwd entry for the container UID
local myuid; myuid="$(id -u)"
@@ -51,10 +54,10 @@ if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
fi
-if ! [ -z ${PYSPARK_PYTHON+x} ]; then
+if ! [ -z "${PYSPARK_PYTHON+x}" ]; then
export PYSPARK_PYTHON
fi
-if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then
+if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then
export PYSPARK_DRIVER_PYTHON
fi
@@ -64,13 +67,13 @@ if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
fi
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
+if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then
SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
fi
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
+if ! [ -z "${SPARK_CONF_DIR+x}" ]; then
SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
+elif ! [ -z "${SPARK_HOME+x}" ]; then
SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
fi
@@ -99,17 +102,17 @@ case "$1" in
CMD=(
${JAVA_HOME}/bin/java
"${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
+ -Xms"$SPARK_EXECUTOR_MEMORY"
+ -Xmx"$SPARK_EXECUTOR_MEMORY"
-cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
+ --driver-url "$SPARK_DRIVER_URL"
+ --executor-id "$SPARK_EXECUTOR_ID"
+ --cores "$SPARK_EXECUTOR_CORES"
+ --app-id "$SPARK_APPLICATION_ID"
+ --hostname "$SPARK_EXECUTOR_POD_IP"
+ --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID"
+ --podName "$SPARK_EXECUTOR_POD_NAME"
)
attempt_setup_fake_passwd_entry
# Execute the container CMD under tini for better hygiene
diff --git a/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh b/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh
index 08fc925..2e3d2a8 100755
--- a/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh
+++ b/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh
@@ -15,6 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+# Prevent any errors from being silently ignored
+set -eo pipefail
+
attempt_setup_fake_passwd_entry() {
# Check whether there is a passwd entry for the container UID
local myuid; myuid="$(id -u)"
@@ -51,10 +54,10 @@ if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
fi
-if ! [ -z ${PYSPARK_PYTHON+x} ]; then
+if ! [ -z "${PYSPARK_PYTHON+x}" ]; then
export PYSPARK_PYTHON
fi
-if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then
+if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then
export PYSPARK_DRIVER_PYTHON
fi
@@ -64,13 +67,13 @@ if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
fi
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
+if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then
SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
fi
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
+if ! [ -z "${SPARK_CONF_DIR+x}" ]; then
SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
+elif ! [ -z "${SPARK_HOME+x}" ]; then
SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
fi
@@ -99,17 +102,17 @@ case "$1" in
CMD=(
${JAVA_HOME}/bin/java
"${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
+ -Xms"$SPARK_EXECUTOR_MEMORY"
+ -Xmx"$SPARK_EXECUTOR_MEMORY"
-cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
+ --driver-url "$SPARK_DRIVER_URL"
+ --executor-id "$SPARK_EXECUTOR_ID"
+ --cores "$SPARK_EXECUTOR_CORES"
+ --app-id "$SPARK_APPLICATION_ID"
+ --hostname "$SPARK_EXECUTOR_POD_IP"
+ --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID"
+ --podName "$SPARK_EXECUTOR_POD_NAME"
)
attempt_setup_fake_passwd_entry
# Execute the container CMD under tini for better hygiene
diff --git a/entrypoint.sh.template b/entrypoint.sh.template
index 08fc925..2e3d2a8 100644
--- a/entrypoint.sh.template
+++ b/entrypoint.sh.template
@@ -15,6 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+# Prevent any errors from being silently ignored
+set -eo pipefail
+
attempt_setup_fake_passwd_entry() {
# Check whether there is a passwd entry for the container UID
local myuid; myuid="$(id -u)"
@@ -51,10 +54,10 @@ if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
fi
-if ! [ -z ${PYSPARK_PYTHON+x} ]; then
+if ! [ -z "${PYSPARK_PYTHON+x}" ]; then
export PYSPARK_PYTHON
fi
-if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then
+if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then
export PYSPARK_DRIVER_PYTHON
fi
@@ -64,13 +67,13 @@ if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
fi
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
+if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then
SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
fi
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
+if ! [ -z "${SPARK_CONF_DIR+x}" ]; then
SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
+elif ! [ -z "${SPARK_HOME+x}" ]; then
SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
fi
@@ -99,17 +102,17 @@ case "$1" in
CMD=(
${JAVA_HOME}/bin/java
"${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
+ -Xms"$SPARK_EXECUTOR_MEMORY"
+ -Xmx"$SPARK_EXECUTOR_MEMORY"
-cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
+ --driver-url "$SPARK_DRIVER_URL"
+ --executor-id "$SPARK_EXECUTOR_ID"
+ --cores "$SPARK_EXECUTOR_CORES"
+ --app-id "$SPARK_APPLICATION_ID"
+ --hostname "$SPARK_EXECUTOR_POD_IP"
+ --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID"
+ --podName "$SPARK_EXECUTOR_POD_NAME"
)
attempt_setup_fake_passwd_entry
# Execute the container CMD under tini for better hygiene
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org