You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yi...@apache.org on 2023/06/01 06:36:28 UTC
[spark-docker] branch master updated: [SPARK-43370] Switch spark user only when run driver and executor
This is an automated email from the ASF dual-hosted git repository.
yikun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-docker.git
The following commit(s) were added to refs/heads/master by this push:
new 2dc12d9 [SPARK-43370] Switch spark user only when run driver and executor
2dc12d9 is described below
commit 2dc12d96910710aa6ee2d717c4c723ddd75127a1
Author: Yikun Jiang <yi...@gmail.com>
AuthorDate: Thu Jun 1 14:36:17 2023 +0800
[SPARK-43370] Switch spark user only when run driver and executor
### What changes were proposed in this pull request?
Switch spark user only when run driver and executor
### Why are the changes needed?
Address doi comments: question 7 [1]
[1] https://github.com/docker-library/official-images/pull/13089#issuecomment-1533540388
[2] https://github.com/docker-library/official-images/pull/13089#issuecomment-1561793792
### Does this PR introduce _any_ user-facing change?
Yes
### How was this patch tested?
1. test mannuly
```
cd ~/spark-docker/3.4.0/scala2.12-java11-ubuntu
$ docker build . -t spark-test
$ docker run -ti spark-test bash
sparkafa78af05cf8:/opt/spark/work-dir$
$ docker run --user root -ti spark-test bash
root095e0d7651fd:/opt/spark/work-dir#
```
2. ci passed
Closes: https://github.com/apache/spark-docker/pull/44
Closes #43 from Yikun/SPARK-43370.
Authored-by: Yikun Jiang <yi...@gmail.com>
Signed-off-by: Yikun Jiang <yi...@gmail.com>
---
3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile | 4 ++++
3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile | 4 ++++
3.4.0/scala2.12-java11-r-ubuntu/Dockerfile | 4 ++++
3.4.0/scala2.12-java11-ubuntu/Dockerfile | 2 ++
3.4.0/scala2.12-java11-ubuntu/entrypoint.sh | 23 +++++++++++-----------
Dockerfile.template | 2 ++
entrypoint.sh.template | 23 +++++++++++-----------
r-python.template | 4 ++++
8 files changed, 44 insertions(+), 22 deletions(-)
diff --git a/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
index 7734100..0f1962f 100644
--- a/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ b/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
@@ -16,6 +16,8 @@
#
FROM spark:3.4.0-scala2.12-java11-ubuntu
+USER root
+
RUN set -ex; \
apt-get update; \
apt install -y python3 python3-pip; \
@@ -24,3 +26,5 @@ RUN set -ex; \
rm -rf /var/lib/apt/lists/*
ENV R_HOME /usr/lib/R
+
+USER spark
diff --git a/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile b/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile
index 6c12c30..258d806 100644
--- a/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile
+++ b/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile
@@ -16,8 +16,12 @@
#
FROM spark:3.4.0-scala2.12-java11-ubuntu
+USER root
+
RUN set -ex; \
apt-get update; \
apt install -y python3 python3-pip; \
rm -rf /var/cache/apt/*; \
rm -rf /var/lib/apt/lists/*
+
+USER spark
diff --git a/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile b/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile
index 24cd41a..4c928c6 100644
--- a/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile
+++ b/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile
@@ -16,6 +16,8 @@
#
FROM spark:3.4.0-scala2.12-java11-ubuntu
+USER root
+
RUN set -ex; \
apt-get update; \
apt install -y r-base r-base-dev; \
@@ -23,3 +25,5 @@ RUN set -ex; \
rm -rf /var/lib/apt/lists/*
ENV R_HOME /usr/lib/R
+
+USER spark
diff --git a/3.4.0/scala2.12-java11-ubuntu/Dockerfile b/3.4.0/scala2.12-java11-ubuntu/Dockerfile
index 205b399..a680106 100644
--- a/3.4.0/scala2.12-java11-ubuntu/Dockerfile
+++ b/3.4.0/scala2.12-java11-ubuntu/Dockerfile
@@ -77,4 +77,6 @@ ENV SPARK_HOME /opt/spark
WORKDIR /opt/spark/work-dir
+USER spark
+
ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh b/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh
index 716f1af..6def3f9 100755
--- a/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh
+++ b/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh
@@ -69,6 +69,13 @@ elif ! [ -z ${SPARK_HOME+x} ]; then
SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
fi
+# Switch to spark if no USER specified (root by default) otherwise use USER directly
+switch_spark_if_root() {
+ if [ $(id -u) -eq 0 ]; then
+ echo gosu spark
+ fi
+}
+
case "$1" in
driver)
shift 1
@@ -78,6 +85,8 @@ case "$1" in
--deploy-mode client
"$@"
)
+ # Execute the container CMD under tini for better hygiene
+ exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
;;
executor)
shift 1
@@ -96,20 +105,12 @@ case "$1" in
--resourceProfileId $SPARK_RESOURCE_PROFILE_ID
--podName $SPARK_EXECUTOR_POD_NAME
)
+ # Execute the container CMD under tini for better hygiene
+ exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
;;
*)
# Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
+ exec "$@"
;;
esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/Dockerfile.template b/Dockerfile.template
index 8b13e4a..d1188bc 100644
--- a/Dockerfile.template
+++ b/Dockerfile.template
@@ -77,4 +77,6 @@ ENV SPARK_HOME /opt/spark
WORKDIR /opt/spark/work-dir
+USER spark
+
ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/entrypoint.sh.template b/entrypoint.sh.template
index 716f1af..6def3f9 100644
--- a/entrypoint.sh.template
+++ b/entrypoint.sh.template
@@ -69,6 +69,13 @@ elif ! [ -z ${SPARK_HOME+x} ]; then
SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
fi
+# Switch to spark if no USER specified (root by default) otherwise use USER directly
+switch_spark_if_root() {
+ if [ $(id -u) -eq 0 ]; then
+ echo gosu spark
+ fi
+}
+
case "$1" in
driver)
shift 1
@@ -78,6 +85,8 @@ case "$1" in
--deploy-mode client
"$@"
)
+ # Execute the container CMD under tini for better hygiene
+ exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
;;
executor)
shift 1
@@ -96,20 +105,12 @@ case "$1" in
--resourceProfileId $SPARK_RESOURCE_PROFILE_ID
--podName $SPARK_EXECUTOR_POD_NAME
)
+ # Execute the container CMD under tini for better hygiene
+ exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
;;
*)
# Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
+ exec "$@"
;;
esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/r-python.template b/r-python.template
index d3f4ef7..2cc3be0 100644
--- a/r-python.template
+++ b/r-python.template
@@ -16,6 +16,8 @@
#
FROM spark:{{ SPARK_VERSION }}-scala{{ SCALA_VERSION }}-java{{ JAVA_VERSION }}-ubuntu
+USER root
+
RUN set -ex; \
apt-get update; \
{%- if HAVE_PY %}
@@ -30,3 +32,5 @@ RUN set -ex; \
ENV R_HOME /usr/lib/R
{%- endif %}
+
+USER spark
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org