You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2025/02/28 02:25:23 UTC
(spark-docker) branch master updated: [SPARK-51345] Remove all EOL versions from `spark-docker` repository (#83)
This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-docker.git
The following commit(s) were added to refs/heads/master by this push:
new 6a20987 [SPARK-51345] Remove all EOL versions from `spark-docker` repository (#83)
6a20987 is described below
commit 6a209875c42a9e878baa41baa436bde402e8cf57
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Feb 27 18:25:18 2025 -0800
[SPARK-51345] Remove all EOL versions from `spark-docker` repository (#83)
---
.github/workflows/build_3.3.0.yaml | 43 ----
.github/workflows/build_3.3.1.yaml | 43 ----
.github/workflows/build_3.3.2.yaml | 43 ----
.github/workflows/build_3.3.3.yaml | 43 ----
.github/workflows/build_3.4.0.yaml | 43 ----
.github/workflows/build_3.4.1.yaml | 41 ----
.github/workflows/build_3.4.2.yaml | 41 ----
.github/workflows/build_3.4.3.yaml | 41 ----
.github/workflows/build_3.4.4.yaml | 41 ----
.github/workflows/publish-java11.yml | 98 ---------
.github/workflows/test.yml | 9 -
3.3.0/scala2.12-java11-python3-r-ubuntu/Dockerfile | 86 --------
.../entrypoint.sh | 114 -----------
3.3.0/scala2.12-java11-python3-ubuntu/Dockerfile | 83 --------
.../scala2.12-java11-python3-ubuntu/entrypoint.sh | 114 -----------
3.3.0/scala2.12-java11-r-ubuntu/Dockerfile | 82 --------
3.3.0/scala2.12-java11-r-ubuntu/entrypoint.sh | 107 ----------
3.3.0/scala2.12-java11-ubuntu/Dockerfile | 79 --------
3.3.0/scala2.12-java11-ubuntu/entrypoint.sh | 107 ----------
3.3.1/scala2.12-java11-python3-r-ubuntu/Dockerfile | 86 --------
.../entrypoint.sh | 114 -----------
3.3.1/scala2.12-java11-python3-ubuntu/Dockerfile | 83 --------
.../scala2.12-java11-python3-ubuntu/entrypoint.sh | 114 -----------
3.3.1/scala2.12-java11-r-ubuntu/Dockerfile | 82 --------
3.3.1/scala2.12-java11-r-ubuntu/entrypoint.sh | 107 ----------
3.3.1/scala2.12-java11-ubuntu/Dockerfile | 79 --------
3.3.1/scala2.12-java11-ubuntu/entrypoint.sh | 107 ----------
3.3.2/scala2.12-java11-python3-r-ubuntu/Dockerfile | 86 --------
.../entrypoint.sh | 114 -----------
3.3.2/scala2.12-java11-python3-ubuntu/Dockerfile | 83 --------
.../scala2.12-java11-python3-ubuntu/entrypoint.sh | 114 -----------
3.3.2/scala2.12-java11-r-ubuntu/Dockerfile | 82 --------
3.3.2/scala2.12-java11-r-ubuntu/entrypoint.sh | 107 ----------
3.3.2/scala2.12-java11-ubuntu/Dockerfile | 79 --------
3.3.2/scala2.12-java11-ubuntu/entrypoint.sh | 107 ----------
3.3.3/scala2.12-java11-python3-r-ubuntu/Dockerfile | 29 ---
3.3.3/scala2.12-java11-python3-ubuntu/Dockerfile | 26 ---
3.3.3/scala2.12-java11-r-ubuntu/Dockerfile | 28 ---
3.3.3/scala2.12-java11-ubuntu/Dockerfile | 79 --------
3.3.3/scala2.12-java11-ubuntu/entrypoint.sh | 126 ------------
3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile | 29 ---
3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile | 26 ---
3.4.0/scala2.12-java11-r-ubuntu/Dockerfile | 28 ---
3.4.0/scala2.12-java11-ubuntu/Dockerfile | 80 --------
3.4.0/scala2.12-java11-ubuntu/entrypoint.sh | 126 ------------
3.4.1/scala2.12-java11-python3-r-ubuntu/Dockerfile | 29 ---
3.4.1/scala2.12-java11-python3-ubuntu/Dockerfile | 26 ---
3.4.1/scala2.12-java11-r-ubuntu/Dockerfile | 28 ---
3.4.1/scala2.12-java11-ubuntu/Dockerfile | 80 --------
3.4.1/scala2.12-java11-ubuntu/entrypoint.sh | 126 ------------
3.4.2/scala2.12-java11-python3-r-ubuntu/Dockerfile | 29 ---
3.4.2/scala2.12-java11-python3-ubuntu/Dockerfile | 26 ---
3.4.2/scala2.12-java11-r-ubuntu/Dockerfile | 28 ---
3.4.2/scala2.12-java11-ubuntu/Dockerfile | 80 --------
3.4.2/scala2.12-java11-ubuntu/entrypoint.sh | 126 ------------
3.4.3/scala2.12-java11-python3-r-ubuntu/Dockerfile | 29 ---
3.4.3/scala2.12-java11-python3-ubuntu/Dockerfile | 26 ---
3.4.3/scala2.12-java11-r-ubuntu/Dockerfile | 28 ---
3.4.3/scala2.12-java11-ubuntu/Dockerfile | 80 --------
3.4.3/scala2.12-java11-ubuntu/entrypoint.sh | 126 ------------
3.4.4/scala2.12-java11-python3-r-ubuntu/Dockerfile | 29 ---
3.4.4/scala2.12-java11-python3-ubuntu/Dockerfile | 26 ---
3.4.4/scala2.12-java11-r-ubuntu/Dockerfile | 28 ---
3.4.4/scala2.12-java11-ubuntu/Dockerfile | 81 --------
3.4.4/scala2.12-java11-ubuntu/entrypoint.sh | 130 ------------
versions.json | 224 ---------------------
66 files changed, 4764 deletions(-)
diff --git a/.github/workflows/build_3.3.0.yaml b/.github/workflows/build_3.3.0.yaml
deleted file mode 100644
index a4f8224..0000000
--- a/.github/workflows/build_3.3.0.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Build and Test (3.3.0)"
-
-on:
- pull_request:
- branches:
- - 'master'
- paths:
- - '3.3.0/**'
- - '.github/workflows/build_3.3.0.yaml'
- - '.github/workflows/main.yml'
-
-jobs:
- run-build:
- strategy:
- matrix:
- image-type: ["all", "python", "scala", "r"]
- name: Run
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: 3.3.0
- scala: 2.12
- java: 11
- image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/build_3.3.1.yaml b/.github/workflows/build_3.3.1.yaml
deleted file mode 100644
index 9e5c082..0000000
--- a/.github/workflows/build_3.3.1.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Build and Test (3.3.1)"
-
-on:
- pull_request:
- branches:
- - 'master'
- paths:
- - '3.3.1/**'
- - '.github/workflows/build_3.3.1.yaml'
- - '.github/workflows/main.yml'
-
-jobs:
- run-build:
- strategy:
- matrix:
- image-type: ["all", "python", "scala", "r"]
- name: Run
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: 3.3.1
- scala: 2.12
- java: 11
- image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/build_3.3.2.yaml b/.github/workflows/build_3.3.2.yaml
deleted file mode 100644
index 9ae1a13..0000000
--- a/.github/workflows/build_3.3.2.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Build and Test (3.3.2)"
-
-on:
- pull_request:
- branches:
- - 'master'
- paths:
- - '3.3.2/**'
- - '.github/workflows/build_3.3.2.yaml'
- - '.github/workflows/main.yml'
-
-jobs:
- run-build:
- strategy:
- matrix:
- image-type: ["all", "python", "scala", "r"]
- name: Run
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: 3.3.2
- scala: 2.12
- java: 11
- image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/build_3.3.3.yaml b/.github/workflows/build_3.3.3.yaml
deleted file mode 100644
index 75d3948..0000000
--- a/.github/workflows/build_3.3.3.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Build and Test (3.3.3)"
-
-on:
- pull_request:
- branches:
- - 'master'
- paths:
- - '3.3.3/**'
- - '.github/workflows/build_3.3.3.yaml'
- - '.github/workflows/main.yml'
-
-jobs:
- run-build:
- strategy:
- matrix:
- image-type: ["all", "python", "scala", "r"]
- name: Run
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: 3.3.3
- scala: 2.12
- java: 11
- image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/build_3.4.0.yaml b/.github/workflows/build_3.4.0.yaml
deleted file mode 100644
index 8dd4e1e..0000000
--- a/.github/workflows/build_3.4.0.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Build and Test (3.4.0)"
-
-on:
- pull_request:
- branches:
- - 'master'
- paths:
- - '3.4.0/**'
- - '.github/workflows/build_3.4.0.yaml'
- - '.github/workflows/main.yml'
-
-jobs:
- run-build:
- strategy:
- matrix:
- image-type: ["all", "python", "scala", "r"]
- name: Run
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: 3.4.0
- scala: 2.12
- java: 11
- image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/build_3.4.1.yaml b/.github/workflows/build_3.4.1.yaml
deleted file mode 100644
index 2eba18e..0000000
--- a/.github/workflows/build_3.4.1.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Build and Test (3.4.1)"
-
-on:
- pull_request:
- branches:
- - 'master'
- paths:
- - '3.4.1/**'
-
-jobs:
- run-build:
- strategy:
- matrix:
- image-type: ["all", "python", "scala", "r"]
- name: Run
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: 3.4.1
- scala: 2.12
- java: 11
- image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/build_3.4.2.yaml b/.github/workflows/build_3.4.2.yaml
deleted file mode 100644
index 8ae17d1..0000000
--- a/.github/workflows/build_3.4.2.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Build and Test (3.4.2)"
-
-on:
- pull_request:
- branches:
- - 'master'
- paths:
- - '3.4.2/**'
-
-jobs:
- run-build:
- strategy:
- matrix:
- image-type: ["all", "python", "scala", "r"]
- name: Run
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: 3.4.2
- scala: 2.12
- java: 11
- image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/build_3.4.3.yaml b/.github/workflows/build_3.4.3.yaml
deleted file mode 100644
index dd288ef..0000000
--- a/.github/workflows/build_3.4.3.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Build and Test (3.4.3)"
-
-on:
- pull_request:
- branches:
- - 'master'
- paths:
- - '3.4.3/**'
-
-jobs:
- run-build:
- strategy:
- matrix:
- image-type: ["all", "python", "scala", "r"]
- name: Run
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: 3.4.3
- scala: 2.12
- java: 11
- image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/build_3.4.4.yaml b/.github/workflows/build_3.4.4.yaml
deleted file mode 100644
index 5fd23e9..0000000
--- a/.github/workflows/build_3.4.4.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Build and Test (3.4.4)"
-
-on:
- pull_request:
- branches:
- - 'master'
- paths:
- - '3.4.4/**'
-
-jobs:
- run-build:
- strategy:
- matrix:
- image-type: ["all", "python", "scala", "r"]
- name: Run
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: 3.4.4
- scala: 2.12
- java: 11
- image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/publish-java11.yml b/.github/workflows/publish-java11.yml
deleted file mode 100644
index 661ba02..0000000
--- a/.github/workflows/publish-java11.yml
+++ /dev/null
@@ -1,98 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Publish (Java 11 only)"
-
-on:
- workflow_dispatch:
- inputs:
- spark:
- description: 'The Spark version of Spark image.'
- required: true
- default: '3.4.4'
- type: choice
- options:
- - 3.4.4
- - 3.4.3
- - 3.4.2
- - 3.4.1
- - 3.4.0
- - 3.3.3
- - 3.3.2
- - 3.3.1
- - 3.3.0
- publish:
- description: 'Publish the image or not.'
- default: false
- type: boolean
- required: true
- repository:
- description: The registry to be published (Available only when publish is true).
- required: false
- default: ghcr.io/apache/spark-docker
- type: choice
- options:
- # GHCR: This required the write permission of apache/spark-docker (Spark Committer)
- - ghcr.io/apache/spark-docker
- # Dockerhub: This required the DOCKERHUB_TOKEN and DOCKERHUB_USER (Spark Committer)
- - apache
-
-jobs:
- # We first build and publish the base image
- run-base-build:
- # if: startsWith(inputs.spark, '3.3')
- strategy:
- matrix:
- scala: [2.12]
- java: [11]
- image-type: ["scala"]
- permissions:
- packages: write
- name: Run Base
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: ${{ inputs.spark }}
- scala: ${{ matrix.scala }}
- java: ${{ matrix.java }}
- publish: ${{ inputs.publish }}
- repository: ${{ inputs.repository }}
- image-type: ${{ matrix.image-type }}
-
- # Then publish the all / python / r images
- run-build:
- needs: run-base-build
- # if: startsWith(inputs.spark, '3.3')
- strategy:
- matrix:
- scala: [2.12]
- java: [11]
- image-type: ["all", "python", "r"]
- permissions:
- packages: write
- name: Run
- secrets: inherit
- uses: ./.github/workflows/main.yml
- with:
- spark: ${{ inputs.spark }}
- scala: ${{ matrix.scala }}
- java: ${{ matrix.java }}
- publish: ${{ inputs.publish }}
- repository: ${{ inputs.repository }}
- image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 02ef44d..20de694 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -35,15 +35,6 @@ on:
- 3.5.2
- 3.5.1
- 3.5.0
- - 3.4.4
- - 3.4.3
- - 3.4.2
- - 3.4.1
- - 3.4.0
- - 3.3.3
- - 3.3.2
- - 3.3.1
- - 3.3.0
java:
description: 'The Java version of Spark image.'
default: 11
diff --git a/3.3.0/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.3.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
deleted file mode 100644
index fb48b80..0000000
--- a/3.3.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,86 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- apt install -y python3 python3-pip && \
- apt install -y r-base r-base-dev && \
- mkdir -p /opt/spark && \
- mkdir /opt/spark/python && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz.asc \
- GPG_KEY=80FB8EBE8EBA68504989703491B5DC815DBF10D3
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- mv R /opt/spark/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-ENV R_HOME /usr/lib/R
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.0/scala2.12-java11-python3-r-ubuntu/entrypoint.sh b/3.3.0/scala2.12-java11-python3-r-ubuntu/entrypoint.sh
deleted file mode 100644
index 4bb1557..0000000
--- a/3.3.0/scala2.12-java11-python3-r-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z ${PYSPARK_PYTHON+x} ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.0/scala2.12-java11-python3-ubuntu/Dockerfile b/3.3.0/scala2.12-java11-python3-ubuntu/Dockerfile
deleted file mode 100644
index 1b6a02c..0000000
--- a/3.3.0/scala2.12-java11-python3-ubuntu/Dockerfile
+++ /dev/null
@@ -1,83 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- apt install -y python3 python3-pip && \
- mkdir -p /opt/spark && \
- mkdir /opt/spark/python && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz.asc \
- GPG_KEY=80FB8EBE8EBA68504989703491B5DC815DBF10D3
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.0/scala2.12-java11-python3-ubuntu/entrypoint.sh b/3.3.0/scala2.12-java11-python3-ubuntu/entrypoint.sh
deleted file mode 100644
index 4bb1557..0000000
--- a/3.3.0/scala2.12-java11-python3-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z ${PYSPARK_PYTHON+x} ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.0/scala2.12-java11-r-ubuntu/Dockerfile b/3.3.0/scala2.12-java11-r-ubuntu/Dockerfile
deleted file mode 100644
index 0b49aef..0000000
--- a/3.3.0/scala2.12-java11-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,82 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- apt install -y r-base r-base-dev && \
- mkdir -p /opt/spark && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz.asc \
- GPG_KEY=80FB8EBE8EBA68504989703491B5DC815DBF10D3
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv R /opt/spark/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-ENV R_HOME /usr/lib/R
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.0/scala2.12-java11-r-ubuntu/entrypoint.sh b/3.3.0/scala2.12-java11-r-ubuntu/entrypoint.sh
deleted file mode 100644
index 159d539..0000000
--- a/3.3.0/scala2.12-java11-r-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.0/scala2.12-java11-ubuntu/Dockerfile b/3.3.0/scala2.12-java11-ubuntu/Dockerfile
deleted file mode 100644
index cbdf6c5..0000000
--- a/3.3.0/scala2.12-java11-ubuntu/Dockerfile
+++ /dev/null
@@ -1,79 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- mkdir -p /opt/spark && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz.asc \
- GPG_KEY=80FB8EBE8EBA68504989703491B5DC815DBF10D3
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.0/scala2.12-java11-ubuntu/entrypoint.sh b/3.3.0/scala2.12-java11-ubuntu/entrypoint.sh
deleted file mode 100644
index 159d539..0000000
--- a/3.3.0/scala2.12-java11-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.1/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.3.1/scala2.12-java11-python3-r-ubuntu/Dockerfile
deleted file mode 100644
index def11ff..0000000
--- a/3.3.1/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,86 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- apt install -y python3 python3-pip && \
- apt install -y r-base r-base-dev && \
- mkdir -p /opt/spark && \
- mkdir /opt/spark/python && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz.asc \
- GPG_KEY=86727D43E73A415F67A0B1A14E68B3E6CD473653
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- mv R /opt/spark/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-ENV R_HOME /usr/lib/R
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.1/scala2.12-java11-python3-r-ubuntu/entrypoint.sh b/3.3.1/scala2.12-java11-python3-r-ubuntu/entrypoint.sh
deleted file mode 100644
index 4bb1557..0000000
--- a/3.3.1/scala2.12-java11-python3-r-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z ${PYSPARK_PYTHON+x} ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.1/scala2.12-java11-python3-ubuntu/Dockerfile b/3.3.1/scala2.12-java11-python3-ubuntu/Dockerfile
deleted file mode 100644
index 3da8521..0000000
--- a/3.3.1/scala2.12-java11-python3-ubuntu/Dockerfile
+++ /dev/null
@@ -1,83 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- apt install -y python3 python3-pip && \
- mkdir -p /opt/spark && \
- mkdir /opt/spark/python && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz.asc \
- GPG_KEY=86727D43E73A415F67A0B1A14E68B3E6CD473653
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.1/scala2.12-java11-python3-ubuntu/entrypoint.sh b/3.3.1/scala2.12-java11-python3-ubuntu/entrypoint.sh
deleted file mode 100644
index 4bb1557..0000000
--- a/3.3.1/scala2.12-java11-python3-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z ${PYSPARK_PYTHON+x} ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.1/scala2.12-java11-r-ubuntu/Dockerfile b/3.3.1/scala2.12-java11-r-ubuntu/Dockerfile
deleted file mode 100644
index 04ab3ea..0000000
--- a/3.3.1/scala2.12-java11-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,82 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- apt install -y r-base r-base-dev && \
- mkdir -p /opt/spark && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz.asc \
- GPG_KEY=86727D43E73A415F67A0B1A14E68B3E6CD473653
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv R /opt/spark/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-ENV R_HOME /usr/lib/R
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.1/scala2.12-java11-r-ubuntu/entrypoint.sh b/3.3.1/scala2.12-java11-r-ubuntu/entrypoint.sh
deleted file mode 100644
index 159d539..0000000
--- a/3.3.1/scala2.12-java11-r-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.1/scala2.12-java11-ubuntu/Dockerfile b/3.3.1/scala2.12-java11-ubuntu/Dockerfile
deleted file mode 100644
index 8b342c4..0000000
--- a/3.3.1/scala2.12-java11-ubuntu/Dockerfile
+++ /dev/null
@@ -1,79 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- mkdir -p /opt/spark && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz.asc \
- GPG_KEY=86727D43E73A415F67A0B1A14E68B3E6CD473653
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.1/scala2.12-java11-ubuntu/entrypoint.sh b/3.3.1/scala2.12-java11-ubuntu/entrypoint.sh
deleted file mode 100644
index 159d539..0000000
--- a/3.3.1/scala2.12-java11-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.2/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.3.2/scala2.12-java11-python3-r-ubuntu/Dockerfile
deleted file mode 100644
index b518021..0000000
--- a/3.3.2/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,86 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- apt install -y python3 python3-pip && \
- apt install -y r-base r-base-dev && \
- mkdir -p /opt/spark && \
- mkdir /opt/spark/python && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz.asc \
- GPG_KEY=C56349D886F2B01F8CAE794C653C2301FEA493EE
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- mv R /opt/spark/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-ENV R_HOME /usr/lib/R
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.2/scala2.12-java11-python3-r-ubuntu/entrypoint.sh b/3.3.2/scala2.12-java11-python3-r-ubuntu/entrypoint.sh
deleted file mode 100644
index 4bb1557..0000000
--- a/3.3.2/scala2.12-java11-python3-r-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z ${PYSPARK_PYTHON+x} ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.2/scala2.12-java11-python3-ubuntu/Dockerfile b/3.3.2/scala2.12-java11-python3-ubuntu/Dockerfile
deleted file mode 100644
index 37e1205..0000000
--- a/3.3.2/scala2.12-java11-python3-ubuntu/Dockerfile
+++ /dev/null
@@ -1,83 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- apt install -y python3 python3-pip && \
- mkdir -p /opt/spark && \
- mkdir /opt/spark/python && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz.asc \
- GPG_KEY=C56349D886F2B01F8CAE794C653C2301FEA493EE
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.2/scala2.12-java11-python3-ubuntu/entrypoint.sh b/3.3.2/scala2.12-java11-python3-ubuntu/entrypoint.sh
deleted file mode 100644
index 4bb1557..0000000
--- a/3.3.2/scala2.12-java11-python3-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z ${PYSPARK_PYTHON+x} ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.2/scala2.12-java11-r-ubuntu/Dockerfile b/3.3.2/scala2.12-java11-r-ubuntu/Dockerfile
deleted file mode 100644
index 330c746..0000000
--- a/3.3.2/scala2.12-java11-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,82 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- apt install -y r-base r-base-dev && \
- mkdir -p /opt/spark && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz.asc \
- GPG_KEY=C56349D886F2B01F8CAE794C653C2301FEA493EE
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv R /opt/spark/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-ENV R_HOME /usr/lib/R
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.2/scala2.12-java11-r-ubuntu/entrypoint.sh b/3.3.2/scala2.12-java11-r-ubuntu/entrypoint.sh
deleted file mode 100644
index 159d539..0000000
--- a/3.3.2/scala2.12-java11-r-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.2/scala2.12-java11-ubuntu/Dockerfile b/3.3.2/scala2.12-java11-ubuntu/Dockerfile
deleted file mode 100644
index 5bead91..0000000
--- a/3.3.2/scala2.12-java11-ubuntu/Dockerfile
+++ /dev/null
@@ -1,79 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex && \
- apt-get update && \
- ln -s /lib /lib64 && \
- apt install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu && \
- mkdir -p /opt/spark && \
- mkdir -p /opt/spark/examples && \
- mkdir -p /opt/spark/work-dir && \
- touch /opt/spark/RELEASE && \
- chown -R spark:spark /opt/spark && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
- rm -rf /var/cache/apt/* && \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz.asc \
- GPG_KEY=C56349D886F2B01F8CAE794C653C2301FEA493EE
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-RUN chmod g+w /opt/spark/work-dir
-RUN chmod a+x /opt/decom.sh
-RUN chmod a+x /opt/entrypoint.sh
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.2/scala2.12-java11-ubuntu/entrypoint.sh b/3.3.2/scala2.12-java11-ubuntu/entrypoint.sh
deleted file mode 100644
index 159d539..0000000
--- a/3.3.2/scala2.12-java11-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
- fi
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z ${SPARK_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z ${SPARK_HOME+x} ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
- --podName $SPARK_EXECUTOR_POD_NAME
- )
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- CMD=("$@")
- ;;
-esac
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-# Execute the container CMD under tini for better hygiene
-exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/3.3.3/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.3.3/scala2.12-java11-python3-r-ubuntu/Dockerfile
deleted file mode 100644
index 3d8599a..0000000
--- a/3.3.3/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.3.3-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.3.3/scala2.12-java11-python3-ubuntu/Dockerfile b/3.3.3/scala2.12-java11-python3-ubuntu/Dockerfile
deleted file mode 100644
index 68afdd5..0000000
--- a/3.3.3/scala2.12-java11-python3-ubuntu/Dockerfile
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.3.3-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- rm -rf /var/lib/apt/lists/*
-
-USER spark
diff --git a/3.3.3/scala2.12-java11-r-ubuntu/Dockerfile b/3.3.3/scala2.12-java11-r-ubuntu/Dockerfile
deleted file mode 100644
index b89d87f..0000000
--- a/3.3.3/scala2.12-java11-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.3.3-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.3.3/scala2.12-java11-ubuntu/Dockerfile b/3.3.3/scala2.12-java11-ubuntu/Dockerfile
deleted file mode 100644
index 1bc6900..0000000
--- a/3.3.3/scala2.12-java11-ubuntu/Dockerfile
+++ /dev/null
@@ -1,79 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper; \
- mkdir -p /opt/spark; \
- mkdir /opt/spark/python; \
- mkdir -p /opt/spark/examples; \
- mkdir -p /opt/spark/work-dir; \
- chmod g+w /opt/spark/work-dir; \
- touch /opt/spark/RELEASE; \
- chown -R spark:spark /opt/spark; \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.3.3/spark-3.3.3-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.3.3/spark-3.3.3-bin-hadoop3.tgz.asc \
- GPG_KEY=F6468A4FF8377B4F1C07BC2AA077F928A0BF68D8
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- mv R /opt/spark/; \
- chmod a+x /opt/decom.sh; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-
-USER spark
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.3.3/scala2.12-java11-ubuntu/entrypoint.sh b/3.3.3/scala2.12-java11-ubuntu/entrypoint.sh
deleted file mode 100755
index 2e3d2a8..0000000
--- a/3.3.3/scala2.12-java11-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Prevent any errors from being silently ignored
-set -eo pipefail
-
-attempt_setup_fake_passwd_entry() {
- # Check whether there is a passwd entry for the container UID
- local myuid; myuid="$(id -u)"
- # If there is no passwd entry for the container UID, attempt to fake one
- # You can also refer to the https://github.com/docker-library/official-images/pull/13089#issuecomment-1534706523
- # It's to resolve OpenShift random UID case.
- # See also: https://github.com/docker-library/postgres/pull/448
- if ! getent passwd "$myuid" &> /dev/null; then
- local wrapper
- for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do
- if [ -s "$wrapper" ]; then
- NSS_WRAPPER_PASSWD="$(mktemp)"
- NSS_WRAPPER_GROUP="$(mktemp)"
- export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP
- local mygid; mygid="$(id -g)"
- printf 'spark:x:%s:%s:${SPARK_USER_NAME:-anonymous uid}:%s:/bin/false\n' "$myuid" "$mygid" "$SPARK_HOME" > "$NSS_WRAPPER_PASSWD"
- printf 'spark:x:%s:\n' "$mygid" > "$NSS_WRAPPER_GROUP"
- break
- fi
- done
- fi
-}
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-for v in "${!SPARK_JAVA_OPT_@}"; do
- SPARK_EXECUTOR_JAVA_OPTS+=( "${!v}" )
-done
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z "${PYSPARK_PYTHON+x}" ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z "${SPARK_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z "${SPARK_HOME+x}" ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms"$SPARK_EXECUTOR_MEMORY"
- -Xmx"$SPARK_EXECUTOR_MEMORY"
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url "$SPARK_DRIVER_URL"
- --executor-id "$SPARK_EXECUTOR_ID"
- --cores "$SPARK_EXECUTOR_CORES"
- --app-id "$SPARK_APPLICATION_ID"
- --hostname "$SPARK_EXECUTOR_POD_IP"
- --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID"
- --podName "$SPARK_EXECUTOR_POD_NAME"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- exec "$@"
- ;;
-esac
diff --git a/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
deleted file mode 100644
index 10aa23e..0000000
--- a/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.0-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile b/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile
deleted file mode 100644
index 3240e57..0000000
--- a/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.0-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- rm -rf /var/lib/apt/lists/*
-
-USER spark
diff --git a/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile b/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile
deleted file mode 100644
index 266392f..0000000
--- a/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.0-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.4.0/scala2.12-java11-ubuntu/Dockerfile b/3.4.0/scala2.12-java11-ubuntu/Dockerfile
deleted file mode 100644
index c756d8a..0000000
--- a/3.4.0/scala2.12-java11-ubuntu/Dockerfile
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper; \
- mkdir -p /opt/spark; \
- mkdir /opt/spark/python; \
- mkdir -p /opt/spark/examples; \
- mkdir -p /opt/spark/work-dir; \
- chmod g+w /opt/spark/work-dir; \
- touch /opt/spark/RELEASE; \
- chown -R spark:spark /opt/spark; \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.4.0/spark-3.4.0-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.4.0/spark-3.4.0-bin-hadoop3.tgz.asc \
- GPG_KEY=CC68B3D16FE33A766705160BA7E57908C7A4E1B1
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv RELEASE /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- mv R /opt/spark/; \
- chmod a+x /opt/decom.sh; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-
-USER spark
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh b/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh
deleted file mode 100755
index 2e3d2a8..0000000
--- a/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Prevent any errors from being silently ignored
-set -eo pipefail
-
-attempt_setup_fake_passwd_entry() {
- # Check whether there is a passwd entry for the container UID
- local myuid; myuid="$(id -u)"
- # If there is no passwd entry for the container UID, attempt to fake one
- # You can also refer to the https://github.com/docker-library/official-images/pull/13089#issuecomment-1534706523
- # It's to resolve OpenShift random UID case.
- # See also: https://github.com/docker-library/postgres/pull/448
- if ! getent passwd "$myuid" &> /dev/null; then
- local wrapper
- for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do
- if [ -s "$wrapper" ]; then
- NSS_WRAPPER_PASSWD="$(mktemp)"
- NSS_WRAPPER_GROUP="$(mktemp)"
- export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP
- local mygid; mygid="$(id -g)"
- printf 'spark:x:%s:%s:${SPARK_USER_NAME:-anonymous uid}:%s:/bin/false\n' "$myuid" "$mygid" "$SPARK_HOME" > "$NSS_WRAPPER_PASSWD"
- printf 'spark:x:%s:\n' "$mygid" > "$NSS_WRAPPER_GROUP"
- break
- fi
- done
- fi
-}
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-for v in "${!SPARK_JAVA_OPT_@}"; do
- SPARK_EXECUTOR_JAVA_OPTS+=( "${!v}" )
-done
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z "${PYSPARK_PYTHON+x}" ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z "${SPARK_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z "${SPARK_HOME+x}" ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms"$SPARK_EXECUTOR_MEMORY"
- -Xmx"$SPARK_EXECUTOR_MEMORY"
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url "$SPARK_DRIVER_URL"
- --executor-id "$SPARK_EXECUTOR_ID"
- --cores "$SPARK_EXECUTOR_CORES"
- --app-id "$SPARK_APPLICATION_ID"
- --hostname "$SPARK_EXECUTOR_POD_IP"
- --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID"
- --podName "$SPARK_EXECUTOR_POD_NAME"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- exec "$@"
- ;;
-esac
diff --git a/3.4.1/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.4.1/scala2.12-java11-python3-r-ubuntu/Dockerfile
deleted file mode 100644
index 30e6b86..0000000
--- a/3.4.1/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.1-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.4.1/scala2.12-java11-python3-ubuntu/Dockerfile b/3.4.1/scala2.12-java11-python3-ubuntu/Dockerfile
deleted file mode 100644
index 124ef71..0000000
--- a/3.4.1/scala2.12-java11-python3-ubuntu/Dockerfile
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.1-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- rm -rf /var/lib/apt/lists/*
-
-USER spark
diff --git a/3.4.1/scala2.12-java11-r-ubuntu/Dockerfile b/3.4.1/scala2.12-java11-r-ubuntu/Dockerfile
deleted file mode 100644
index 1c9fc38..0000000
--- a/3.4.1/scala2.12-java11-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.1-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.4.1/scala2.12-java11-ubuntu/Dockerfile b/3.4.1/scala2.12-java11-ubuntu/Dockerfile
deleted file mode 100644
index c18afb0..0000000
--- a/3.4.1/scala2.12-java11-ubuntu/Dockerfile
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper; \
- mkdir -p /opt/spark; \
- mkdir /opt/spark/python; \
- mkdir -p /opt/spark/examples; \
- mkdir -p /opt/spark/work-dir; \
- chmod g+w /opt/spark/work-dir; \
- touch /opt/spark/RELEASE; \
- chown -R spark:spark /opt/spark; \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz.asc \
- GPG_KEY=F28C9C925C188C35E345614DEDA00CE834F0FC5C
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv RELEASE /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- mv R /opt/spark/; \
- chmod a+x /opt/decom.sh; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-
-USER spark
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh b/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh
deleted file mode 100755
index 2e3d2a8..0000000
--- a/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Prevent any errors from being silently ignored
-set -eo pipefail
-
-attempt_setup_fake_passwd_entry() {
- # Check whether there is a passwd entry for the container UID
- local myuid; myuid="$(id -u)"
- # If there is no passwd entry for the container UID, attempt to fake one
- # You can also refer to the https://github.com/docker-library/official-images/pull/13089#issuecomment-1534706523
- # It's to resolve OpenShift random UID case.
- # See also: https://github.com/docker-library/postgres/pull/448
- if ! getent passwd "$myuid" &> /dev/null; then
- local wrapper
- for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do
- if [ -s "$wrapper" ]; then
- NSS_WRAPPER_PASSWD="$(mktemp)"
- NSS_WRAPPER_GROUP="$(mktemp)"
- export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP
- local mygid; mygid="$(id -g)"
- printf 'spark:x:%s:%s:${SPARK_USER_NAME:-anonymous uid}:%s:/bin/false\n' "$myuid" "$mygid" "$SPARK_HOME" > "$NSS_WRAPPER_PASSWD"
- printf 'spark:x:%s:\n' "$mygid" > "$NSS_WRAPPER_GROUP"
- break
- fi
- done
- fi
-}
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-for v in "${!SPARK_JAVA_OPT_@}"; do
- SPARK_EXECUTOR_JAVA_OPTS+=( "${!v}" )
-done
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z "${PYSPARK_PYTHON+x}" ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z "${SPARK_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z "${SPARK_HOME+x}" ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms"$SPARK_EXECUTOR_MEMORY"
- -Xmx"$SPARK_EXECUTOR_MEMORY"
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url "$SPARK_DRIVER_URL"
- --executor-id "$SPARK_EXECUTOR_ID"
- --cores "$SPARK_EXECUTOR_CORES"
- --app-id "$SPARK_APPLICATION_ID"
- --hostname "$SPARK_EXECUTOR_POD_IP"
- --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID"
- --podName "$SPARK_EXECUTOR_POD_NAME"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- exec "$@"
- ;;
-esac
diff --git a/3.4.2/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.4.2/scala2.12-java11-python3-r-ubuntu/Dockerfile
deleted file mode 100644
index 7c7e96a..0000000
--- a/3.4.2/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.2-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.4.2/scala2.12-java11-python3-ubuntu/Dockerfile b/3.4.2/scala2.12-java11-python3-ubuntu/Dockerfile
deleted file mode 100644
index a9fc9e4..0000000
--- a/3.4.2/scala2.12-java11-python3-ubuntu/Dockerfile
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.2-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- rm -rf /var/lib/apt/lists/*
-
-USER spark
diff --git a/3.4.2/scala2.12-java11-r-ubuntu/Dockerfile b/3.4.2/scala2.12-java11-r-ubuntu/Dockerfile
deleted file mode 100644
index ec5015d..0000000
--- a/3.4.2/scala2.12-java11-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.2-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.4.2/scala2.12-java11-ubuntu/Dockerfile b/3.4.2/scala2.12-java11-ubuntu/Dockerfile
deleted file mode 100644
index 4c28cc0..0000000
--- a/3.4.2/scala2.12-java11-ubuntu/Dockerfile
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper; \
- mkdir -p /opt/spark; \
- mkdir /opt/spark/python; \
- mkdir -p /opt/spark/examples; \
- mkdir -p /opt/spark/work-dir; \
- chmod g+w /opt/spark/work-dir; \
- touch /opt/spark/RELEASE; \
- chown -R spark:spark /opt/spark; \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3.tgz.asc \
- GPG_KEY=F28C9C925C188C35E345614DEDA00CE834F0FC5C
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv RELEASE /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- mv R /opt/spark/; \
- chmod a+x /opt/decom.sh; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-
-USER spark
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.4.2/scala2.12-java11-ubuntu/entrypoint.sh b/3.4.2/scala2.12-java11-ubuntu/entrypoint.sh
deleted file mode 100755
index 2e3d2a8..0000000
--- a/3.4.2/scala2.12-java11-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Prevent any errors from being silently ignored
-set -eo pipefail
-
-attempt_setup_fake_passwd_entry() {
- # Check whether there is a passwd entry for the container UID
- local myuid; myuid="$(id -u)"
- # If there is no passwd entry for the container UID, attempt to fake one
- # You can also refer to the https://github.com/docker-library/official-images/pull/13089#issuecomment-1534706523
- # It's to resolve OpenShift random UID case.
- # See also: https://github.com/docker-library/postgres/pull/448
- if ! getent passwd "$myuid" &> /dev/null; then
- local wrapper
- for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do
- if [ -s "$wrapper" ]; then
- NSS_WRAPPER_PASSWD="$(mktemp)"
- NSS_WRAPPER_GROUP="$(mktemp)"
- export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP
- local mygid; mygid="$(id -g)"
- printf 'spark:x:%s:%s:${SPARK_USER_NAME:-anonymous uid}:%s:/bin/false\n' "$myuid" "$mygid" "$SPARK_HOME" > "$NSS_WRAPPER_PASSWD"
- printf 'spark:x:%s:\n' "$mygid" > "$NSS_WRAPPER_GROUP"
- break
- fi
- done
- fi
-}
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-for v in "${!SPARK_JAVA_OPT_@}"; do
- SPARK_EXECUTOR_JAVA_OPTS+=( "${!v}" )
-done
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z "${PYSPARK_PYTHON+x}" ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z "${SPARK_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z "${SPARK_HOME+x}" ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms"$SPARK_EXECUTOR_MEMORY"
- -Xmx"$SPARK_EXECUTOR_MEMORY"
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url "$SPARK_DRIVER_URL"
- --executor-id "$SPARK_EXECUTOR_ID"
- --cores "$SPARK_EXECUTOR_CORES"
- --app-id "$SPARK_APPLICATION_ID"
- --hostname "$SPARK_EXECUTOR_POD_IP"
- --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID"
- --podName "$SPARK_EXECUTOR_POD_NAME"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- exec "$@"
- ;;
-esac
diff --git a/3.4.3/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.4.3/scala2.12-java11-python3-r-ubuntu/Dockerfile
deleted file mode 100644
index a552299..0000000
--- a/3.4.3/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.3-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.4.3/scala2.12-java11-python3-ubuntu/Dockerfile b/3.4.3/scala2.12-java11-python3-ubuntu/Dockerfile
deleted file mode 100644
index 03bdaa1..0000000
--- a/3.4.3/scala2.12-java11-python3-ubuntu/Dockerfile
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.3-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- rm -rf /var/lib/apt/lists/*
-
-USER spark
diff --git a/3.4.3/scala2.12-java11-r-ubuntu/Dockerfile b/3.4.3/scala2.12-java11-r-ubuntu/Dockerfile
deleted file mode 100644
index 58e228e..0000000
--- a/3.4.3/scala2.12-java11-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.3-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.4.3/scala2.12-java11-ubuntu/Dockerfile b/3.4.3/scala2.12-java11-ubuntu/Dockerfile
deleted file mode 100644
index 4432dea..0000000
--- a/3.4.3/scala2.12-java11-ubuntu/Dockerfile
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper; \
- mkdir -p /opt/spark; \
- mkdir /opt/spark/python; \
- mkdir -p /opt/spark/examples; \
- mkdir -p /opt/spark/work-dir; \
- chmod g+w /opt/spark/work-dir; \
- touch /opt/spark/RELEASE; \
- chown -R spark:spark /opt/spark; \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.4.3/spark-3.4.3-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.4.3/spark-3.4.3-bin-hadoop3.tgz.asc \
- GPG_KEY=F28C9C925C188C35E345614DEDA00CE834F0FC5C
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv RELEASE /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- mv R /opt/spark/; \
- chmod a+x /opt/decom.sh; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-
-USER spark
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.4.3/scala2.12-java11-ubuntu/entrypoint.sh b/3.4.3/scala2.12-java11-ubuntu/entrypoint.sh
deleted file mode 100755
index 2e3d2a8..0000000
--- a/3.4.3/scala2.12-java11-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Prevent any errors from being silently ignored
-set -eo pipefail
-
-attempt_setup_fake_passwd_entry() {
- # Check whether there is a passwd entry for the container UID
- local myuid; myuid="$(id -u)"
- # If there is no passwd entry for the container UID, attempt to fake one
- # You can also refer to the https://github.com/docker-library/official-images/pull/13089#issuecomment-1534706523
- # It's to resolve OpenShift random UID case.
- # See also: https://github.com/docker-library/postgres/pull/448
- if ! getent passwd "$myuid" &> /dev/null; then
- local wrapper
- for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do
- if [ -s "$wrapper" ]; then
- NSS_WRAPPER_PASSWD="$(mktemp)"
- NSS_WRAPPER_GROUP="$(mktemp)"
- export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP
- local mygid; mygid="$(id -g)"
- printf 'spark:x:%s:%s:${SPARK_USER_NAME:-anonymous uid}:%s:/bin/false\n' "$myuid" "$mygid" "$SPARK_HOME" > "$NSS_WRAPPER_PASSWD"
- printf 'spark:x:%s:\n' "$mygid" > "$NSS_WRAPPER_GROUP"
- break
- fi
- done
- fi
-}
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-for v in "${!SPARK_JAVA_OPT_@}"; do
- SPARK_EXECUTOR_JAVA_OPTS+=( "${!v}" )
-done
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z "${PYSPARK_PYTHON+x}" ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z "${SPARK_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z "${SPARK_HOME+x}" ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms"$SPARK_EXECUTOR_MEMORY"
- -Xmx"$SPARK_EXECUTOR_MEMORY"
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url "$SPARK_DRIVER_URL"
- --executor-id "$SPARK_EXECUTOR_ID"
- --cores "$SPARK_EXECUTOR_CORES"
- --app-id "$SPARK_APPLICATION_ID"
- --hostname "$SPARK_EXECUTOR_POD_IP"
- --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID"
- --podName "$SPARK_EXECUTOR_POD_NAME"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- exec "$@"
- ;;
-esac
diff --git a/3.4.4/scala2.12-java11-python3-r-ubuntu/Dockerfile b/3.4.4/scala2.12-java11-python3-r-ubuntu/Dockerfile
deleted file mode 100644
index 9b0f38a..0000000
--- a/3.4.4/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.4-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.4.4/scala2.12-java11-python3-ubuntu/Dockerfile b/3.4.4/scala2.12-java11-python3-ubuntu/Dockerfile
deleted file mode 100644
index d450ca5..0000000
--- a/3.4.4/scala2.12-java11-python3-ubuntu/Dockerfile
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.4-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y python3 python3-pip; \
- rm -rf /var/lib/apt/lists/*
-
-USER spark
diff --git a/3.4.4/scala2.12-java11-r-ubuntu/Dockerfile b/3.4.4/scala2.12-java11-r-ubuntu/Dockerfile
deleted file mode 100644
index 1625536..0000000
--- a/3.4.4/scala2.12-java11-r-ubuntu/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM spark:3.4.4-scala2.12-java11-ubuntu
-
-USER root
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y r-base r-base-dev; \
- rm -rf /var/lib/apt/lists/*
-
-ENV R_HOME /usr/lib/R
-
-USER spark
diff --git a/3.4.4/scala2.12-java11-ubuntu/Dockerfile b/3.4.4/scala2.12-java11-ubuntu/Dockerfile
deleted file mode 100644
index 39cb49a..0000000
--- a/3.4.4/scala2.12-java11-ubuntu/Dockerfile
+++ /dev/null
@@ -1,81 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM eclipse-temurin:11-jre-focal
-
-ARG spark_uid=185
-
-RUN groupadd --system --gid=${spark_uid} spark && \
- useradd --system --uid=${spark_uid} --gid=spark spark
-
-RUN set -ex; \
- apt-get update; \
- apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper; \
- mkdir -p /opt/spark; \
- mkdir /opt/spark/python; \
- mkdir -p /opt/spark/examples; \
- mkdir -p /opt/spark/work-dir; \
- chmod g+w /opt/spark/work-dir; \
- touch /opt/spark/RELEASE; \
- chown -R spark:spark /opt/spark; \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \
- rm -rf /var/lib/apt/lists/*
-
-# Install Apache Spark
-# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.4.4/spark-3.4.4-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.4.4/spark-3.4.4-bin-hadoop3.tgz.asc \
- GPG_KEY=F28C9C925C188C35E345614DEDA00CE834F0FC5C
-
-RUN set -ex; \
- export SPARK_TMP="$(mktemp -d)"; \
- cd $SPARK_TMP; \
- wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
- wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
- gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
- gpg --batch --verify spark.tgz.asc spark.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" spark.tgz.asc; \
- \
- tar -xf spark.tgz --strip-components=1; \
- chown -R spark:spark .; \
- mv jars /opt/spark/; \
- mv RELEASE /opt/spark/; \
- mv bin /opt/spark/; \
- mv sbin /opt/spark/; \
- mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
- mv examples /opt/spark/; \
- ln -s "$(basename /opt/spark/examples/jars/spark-examples_*.jar)" /opt/spark/examples/jars/spark-examples.jar; \
- mv kubernetes/tests /opt/spark/; \
- mv data /opt/spark/; \
- mv python/pyspark /opt/spark/python/pyspark/; \
- mv python/lib /opt/spark/python/lib/; \
- mv R /opt/spark/; \
- chmod a+x /opt/decom.sh; \
- cd ..; \
- rm -rf "$SPARK_TMP";
-
-COPY entrypoint.sh /opt/
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark/work-dir
-
-USER spark
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/3.4.4/scala2.12-java11-ubuntu/entrypoint.sh b/3.4.4/scala2.12-java11-ubuntu/entrypoint.sh
deleted file mode 100755
index c576d8f..0000000
--- a/3.4.4/scala2.12-java11-ubuntu/entrypoint.sh
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Prevent any errors from being silently ignored
-set -eo pipefail
-
-attempt_setup_fake_passwd_entry() {
- # Check whether there is a passwd entry for the container UID
- local myuid; myuid="$(id -u)"
- # If there is no passwd entry for the container UID, attempt to fake one
- # You can also refer to the https://github.com/docker-library/official-images/pull/13089#issuecomment-1534706523
- # It's to resolve OpenShift random UID case.
- # See also: https://github.com/docker-library/postgres/pull/448
- if ! getent passwd "$myuid" &> /dev/null; then
- local wrapper
- for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do
- if [ -s "$wrapper" ]; then
- NSS_WRAPPER_PASSWD="$(mktemp)"
- NSS_WRAPPER_GROUP="$(mktemp)"
- export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP
- local mygid; mygid="$(id -g)"
- printf 'spark:x:%s:%s:${SPARK_USER_NAME:-anonymous uid}:%s:/bin/false\n' "$myuid" "$mygid" "$SPARK_HOME" > "$NSS_WRAPPER_PASSWD"
- printf 'spark:x:%s:\n' "$mygid" > "$NSS_WRAPPER_GROUP"
- break
- fi
- done
- fi
-}
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}')
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-for v in "${!SPARK_JAVA_OPT_@}"; do
- SPARK_EXECUTOR_JAVA_OPTS+=( "${!v}" )
-done
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-if ! [ -z "${PYSPARK_PYTHON+x}" ]; then
- export PYSPARK_PYTHON
-fi
-if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then
- export PYSPARK_DRIVER_PYTHON
-fi
-
-# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
-# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then
- export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
-fi
-
-if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH";
-fi
-
-if ! [ -z "${SPARK_CONF_DIR+x}" ]; then
- SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH";
-elif ! [ -z "${SPARK_HOME+x}" ]; then
- SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH";
-fi
-
-# SPARK-43540: add current working directory into executor classpath
-SPARK_CLASSPATH="$SPARK_CLASSPATH:$PWD"
-
-# Switch to spark if no USER specified (root by default) otherwise use USER directly
-switch_spark_if_root() {
- if [ $(id -u) -eq 0 ]; then
- echo gosu spark
- fi
-}
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --conf "spark.executorEnv.SPARK_DRIVER_POD_IP=$SPARK_DRIVER_BIND_ADDRESS"
- --deploy-mode client
- "$@"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms"$SPARK_EXECUTOR_MEMORY"
- -Xmx"$SPARK_EXECUTOR_MEMORY"
- -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH"
- org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend
- --driver-url "$SPARK_DRIVER_URL"
- --executor-id "$SPARK_EXECUTOR_ID"
- --cores "$SPARK_EXECUTOR_CORES"
- --app-id "$SPARK_APPLICATION_ID"
- --hostname "$SPARK_EXECUTOR_POD_IP"
- --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID"
- --podName "$SPARK_EXECUTOR_POD_NAME"
- )
- attempt_setup_fake_passwd_entry
- # Execute the container CMD under tini for better hygiene
- exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}"
- ;;
-
- *)
- # Non-spark-on-k8s command provided, proceeding in pass-through mode...
- exec "$@"
- ;;
-esac
diff --git a/versions.json b/versions.json
index 355ba98..00e0e20 100644
--- a/versions.json
+++ b/versions.json
@@ -396,230 +396,6 @@
"tags": [
"3.5.0-scala2.12-java11-python3-r-ubuntu"
]
- },
- {
- "path": "3.4.4/scala2.12-java11-python3-ubuntu",
- "tags": [
- "3.4.4-scala2.12-java11-python3-ubuntu",
- "3.4.4-python3",
- "3.4.4"
- ]
- },
- {
- "path": "3.4.4/scala2.12-java11-r-ubuntu",
- "tags": [
- "3.4.4-scala2.12-java11-r-ubuntu",
- "3.4.4-r"
- ]
- },
- {
- "path": "3.4.4/scala2.12-java11-ubuntu",
- "tags": [
- "3.4.4-scala2.12-java11-ubuntu",
- "3.4.4-scala"
- ]
- },
- {
- "path": "3.4.4/scala2.12-java11-python3-r-ubuntu",
- "tags": [
- "3.4.4-scala2.12-java11-python3-r-ubuntu"
- ]
- },
- {
- "path": "3.4.3/scala2.12-java11-python3-ubuntu",
- "tags": [
- "3.4.3-scala2.12-java11-python3-ubuntu",
- "3.4.3-python3",
- "3.4.3"
- ]
- },
- {
- "path": "3.4.3/scala2.12-java11-r-ubuntu",
- "tags": [
- "3.4.3-scala2.12-java11-r-ubuntu",
- "3.4.3-r"
- ]
- },
- {
- "path": "3.4.3/scala2.12-java11-ubuntu",
- "tags": [
- "3.4.3-scala2.12-java11-ubuntu",
- "3.4.3-scala"
- ]
- },
- {
- "path": "3.4.3/scala2.12-java11-python3-r-ubuntu",
- "tags": [
- "3.4.3-scala2.12-java11-python3-r-ubuntu"
- ]
- },
- {
- "path": "3.4.2/scala2.12-java11-python3-ubuntu",
- "tags": [
- "3.4.2-scala2.12-java11-python3-ubuntu",
- "3.4.2-python3",
- "3.4.2"
- ]
- },
- {
- "path": "3.4.2/scala2.12-java11-r-ubuntu",
- "tags": [
- "3.4.2-scala2.12-java11-r-ubuntu",
- "3.4.2-r"
- ]
- },
- {
- "path": "3.4.2/scala2.12-java11-ubuntu",
- "tags": [
- "3.4.2-scala2.12-java11-ubuntu",
- "3.4.2-scala"
- ]
- },
- {
- "path": "3.4.2/scala2.12-java11-python3-r-ubuntu",
- "tags": [
- "3.4.2-scala2.12-java11-python3-r-ubuntu"
- ]
- },
- {
- "path": "3.4.1/scala2.12-java11-python3-ubuntu",
- "tags": [
- "3.4.1-scala2.12-java11-python3-ubuntu",
- "3.4.1-python3",
- "3.4.1"
- ]
- },
- {
- "path": "3.4.1/scala2.12-java11-r-ubuntu",
- "tags": [
- "3.4.1-scala2.12-java11-r-ubuntu",
- "3.4.1-r"
- ]
- },
- {
- "path": "3.4.1/scala2.12-java11-ubuntu",
- "tags": [
- "3.4.1-scala2.12-java11-ubuntu",
- "3.4.1-scala"
- ]
- },
- {
- "path": "3.4.1/scala2.12-java11-python3-r-ubuntu",
- "tags": [
- "3.4.1-scala2.12-java11-python3-r-ubuntu"
- ]
- },
- {
- "path": "3.4.0/scala2.12-java11-python3-ubuntu",
- "tags": [
- "3.4.0-scala2.12-java11-python3-ubuntu",
- "3.4.0-python3",
- "3.4.0"
- ]
- },
- {
- "path": "3.4.0/scala2.12-java11-r-ubuntu",
- "tags": [
- "3.4.0-scala2.12-java11-r-ubuntu",
- "3.4.0-r"
- ]
- },
- {
- "path": "3.4.0/scala2.12-java11-ubuntu",
- "tags": [
- "3.4.0-scala2.12-java11-ubuntu",
- "3.4.0-scala"
- ]
- },
- {
- "path": "3.4.0/scala2.12-java11-python3-r-ubuntu",
- "tags": [
- "3.4.0-scala2.12-java11-python3-r-ubuntu"
- ]
- },
- {
- "path": "3.3.3/scala2.12-java11-python3-ubuntu",
- "tags": [
- "3.3.3-scala2.12-java11-python3-ubuntu",
- "3.3.3-python3",
- "3.3.3"
- ]
- },
- {
- "path": "3.3.3/scala2.12-java11-r-ubuntu",
- "tags": [
- "3.3.3-scala2.12-java11-r-ubuntu",
- "3.3.3-r"
- ]
- },
- {
- "path": "3.3.3/scala2.12-java11-ubuntu",
- "tags": [
- "3.3.3-scala2.12-java11-ubuntu",
- "3.3.3-scala"
- ]
- },
- {
- "path": "3.3.3/scala2.12-java11-python3-r-ubuntu",
- "tags": [
- "3.3.3-scala2.12-java11-python3-r-ubuntu"
- ]
- },
- {
- "path": "3.3.1/scala2.12-java11-python3-ubuntu",
- "tags": [
- "3.3.1-scala2.12-java11-python3-ubuntu",
- "3.3.1-python3",
- "3.3.1"
- ]
- },
- {
- "path": "3.3.1/scala2.12-java11-r-ubuntu",
- "tags": [
- "3.3.1-scala2.12-java11-r-ubuntu",
- "3.3.1-r"
- ]
- },
- {
- "path": "3.3.1/scala2.12-java11-ubuntu",
- "tags": [
- "3.3.1-scala2.12-java11-ubuntu",
- "3.3.1-scala"
- ]
- },
- {
- "path": "3.3.1/scala2.12-java11-python3-r-ubuntu",
- "tags": [
- "3.3.1-scala2.12-java11-python3-r-ubuntu"
- ]
- },
- {
- "path": "3.3.0/scala2.12-java11-python3-ubuntu",
- "tags": [
- "3.3.0-scala2.12-java11-python3-ubuntu",
- "3.3.0-python3",
- "3.3.0"
- ]
- },
- {
- "path": "3.3.0/scala2.12-java11-r-ubuntu",
- "tags": [
- "3.3.0-scala2.12-java11-r-ubuntu",
- "3.3.0-r"
- ]
- },
- {
- "path": "3.3.0/scala2.12-java11-ubuntu",
- "tags": [
- "3.3.0-scala2.12-java11-ubuntu",
- "3.3.0-scala"
- ]
- },
- {
- "path": "3.3.0/scala2.12-java11-python3-r-ubuntu",
- "tags": [
- "3.3.0-scala2.12-java11-python3-r-ubuntu"
- ]
}
]
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org