You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2021/07/21 21:43:41 UTC
[incubator-sdap-nexus] 05/07: make deployment run without
root-containers, upgrade chart for spark 3.1.1
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch moc_deployment
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit ed654f83188bb3430caedf0d71c01c0cea1a6a96
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Wed Jul 7 11:04:47 2021 -0700
make deployment run without root-containers, upgrade chart for spark 3.1.1
---
docker/nexus-webapp/Dockerfile | 66 +++++++++++------------------------------
helm/templates/_helpers.tpl | 3 ++
helm/templates/rolebinding.yaml | 5 ++--
helm/templates/webapp.yml | 6 ++--
4 files changed, 27 insertions(+), 53 deletions(-)
diff --git a/docker/nexus-webapp/Dockerfile b/docker/nexus-webapp/Dockerfile
index 68f7d87..6a0cdcc 100644
--- a/docker/nexus-webapp/Dockerfile
+++ b/docker/nexus-webapp/Dockerfile
@@ -13,9 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM nexusjpl/alpine-pyspark:3.1.1
-
-
+#FROM nexusjpl/alpine-pyspark:3.1.1
+FROM tloubrieu/openshift-pyshark
MAINTAINER Apache SDAP "dev@sdap.apache.org"
@@ -23,53 +22,23 @@ ARG CONDA_VERSION="4.7.12.1"
ARG CONDA_MD5="81c773ff87af5cfac79ab862942ab6b3"
ARG CONDA_DIR="/opt/conda"
-ENV \
- PYTHONPATH=/opt/conda/share/py4j/py4j0.10.9.2.jar \
- NEXUS_SRC=/tmp/incubator-sdap-nexus \
- PROJ_LIB=/opt/conda/lib/python3.8/site-packages/pyproj/data \
- PATH="$CONDA_DIR/bin:$PATH" \
- PYTHONDONTWRITEBYTECODE=1 \
- SPARK_HOME=/opt/spark \
- PYSPARK_DRIVER_PYTHON=/opt/conda/lib/python3.8 \
- PYSPARK_PYTHON=/opt/conda/lib/python3.8 \
- LD_LIBRARY_PATH=/usr/lib
-
-RUN apk add --update --no-cache \
- bzip2 \
- gcc \
- git \
- mesa-gl \
- wget \
- curl \
- which \
- python3 \
- bash==4.4.19-r1 \
- libc-dev \
- libressl2.7-libcrypto
-
-RUN apk upgrade musl
+ENV PATH="/opt/conda/bin:${PATH}"
+ARG PATH="/opt/conda/bin:${PATH}"
+
+RUN apt-get update && apt-get install -y wget && rm -rf /var/lib/apt/lists/*
WORKDIR /tmp
-RUN apk del libc6-compat
-RUN apk --no-cache add wget zlib && \
- wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \
- wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.30-r0/glibc-2.30-r0.apk && \
- apk add glibc-2.30-r0.apk && \
- ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \
- ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \
- ln -s /usr/lib/libgcc_s.so.1 /usr/glibc-compat/lib
+RUN wget \
+ https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+ && mkdir /root/.conda \
+ && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \
+ && rm -f Miniconda3-latest-Linux-x86_64.sh
+
+
-COPY docker/nexus-webapp/install_conda.sh ./install_conda.sh
-RUN /tmp/install_conda.sh
RUN conda install python=3.8
-RUN cd /usr/lib && ln -s libcom_err.so.2 libcom_err.so.3 && \
- cd /opt/conda/lib && \
- ln -s libnetcdf.so.11 libnetcdf.so.7 && \
- ln -s libkea.so.1.4.6 libkea.so.1.4.5 && \
- ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \
- ln -s libjpeg.so.9 libjpeg.so.8
# Change REBUILD_CODE if you want tell Docker not to use cached layers from this line on
ARG REBUILD_CODE=0
@@ -77,6 +46,7 @@ ARG REBUILD_CODE=0
ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git
ARG APACHE_NEXUSPROTO_BRANCH=master
+RUN apt-get update && apt-get install -y git
COPY docker/nexus-webapp/install_nexusproto.sh ./install_nexusproto.sh
RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH
@@ -97,8 +67,8 @@ RUN pip3 install -r requirements.txt
RUN pip3 install cython
RUN rm requirements.txt
-WORKDIR /incubator-sdap-nexus
+RUN chmod -R a+rwX /opt/conda/
-# Upgrade kubernetes client jar from the default version
-RUN rm /opt/spark/jars/kubernetes-client-4.12.0.jar
-ADD https://repo1.maven.org/maven2/io/fabric8/kubernetes-client/4.12.0/kubernetes-client-4.12.0.jar /opt/spark/jars
+ENV PROJ_LIB=/opt/conda/lib/python3.8/site-packages/pyproj/data
+
+WORKDIR /incubator-sdap-nexus
diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
index 3084fa9..f1cf360 100644
--- a/helm/templates/_helpers.tpl
+++ b/helm/templates/_helpers.tpl
@@ -35,6 +35,9 @@ The data volume which is used in both the Collection Manager and the Granule Ing
{{- if .Values.ingestion.granules.path -}}
hostPath:
path: {{ .Values.ingestion.granules.path }}
+ {{- else }}
+ persistentVolumeClaim:
+ claimName: {{ .Values.ingestion.granules.pvc }}
{{- end -}}
{{- end -}}
{{- end -}}
diff --git a/helm/templates/rolebinding.yaml b/helm/templates/rolebinding.yaml
index 34ee010..9366fa4 100644
--- a/helm/templates/rolebinding.yaml
+++ b/helm/templates/rolebinding.yaml
@@ -5,7 +5,8 @@ metadata:
app.kubernetes.io/instance: spark-operator
app.kubernetes.io/name: spark-operator
app.kubernetes.io/version: v1beta2-1.2.3-3.1.1
- name: spark
+ name: sdap-spark-rolebinding
+ namespace: {{ .Release.Namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
@@ -13,5 +14,5 @@ roleRef:
subjects:
- kind: ServiceAccount
name: sdap-spark
- namespace: $(namespace)
+ namespace: {{ .Release.Namespace }}
diff --git a/helm/templates/webapp.yml b/helm/templates/webapp.yml
index 88a024a..f921407 100644
--- a/helm/templates/webapp.yml
+++ b/helm/templates/webapp.yml
@@ -16,7 +16,7 @@ spec:
- --cassandra-username={{ include "nexus.credentials.cassandra.username" . }}
- --cassandra-password={{ include "nexus.credentials.cassandra.password" . }}
- --solr-host={{ include "nexus.urls.solr" . }}
- sparkVersion: "2.4.4"
+ sparkVersion: "3.1.1"
restartPolicy:
type: OnFailure
onFailureRetries: 10
@@ -26,12 +26,12 @@ spec:
driver:
{{ .Values.webapp.distributed.driver | toYaml | indent 4 }}
labels:
- version: 2.4.4
+ version: 3.1.1
serviceAccount: sdap-spark
executor:
{{ .Values.webapp.distributed.executor| toYaml | indent 4 }}
labels:
- version: 2.4.4
+ version: 3.1.1
---