You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by fg...@apache.org on 2019/08/22 01:58:22 UTC
[incubator-sdap-nexus] branch 1.1.0-SNAPSHOT updated: SDAP-207
fixed gdal issues (#82)
This is an automated email from the ASF dual-hosted git repository.
fgreg pushed a commit to branch 1.1.0-SNAPSHOT
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
The following commit(s) were added to refs/heads/1.1.0-SNAPSHOT by this push:
new 2aa50fe SDAP-207 fixed gdal issues (#82)
2aa50fe is described below
commit 2aa50fef8c3131c4b91245a92f8a1345dbf15dd0
Author: fgreg <fg...@gmail.com>
AuthorDate: Wed Aug 21 18:58:17 2019 -0700
SDAP-207 fixed gdal issues (#82)
---
docker/nexus-webapp/install_conda.sh | 15 ++++++++++++---
.../nexus-webapp/mesos/agent/docker-entrypoint.sh | 4 +++-
docker/nexus-webapp/mesos/base/Dockerfile | 18 +++++++++---------
.../nexus-webapp/mesos/webapp/docker-entrypoint.sh | 2 ++
docker/nexus-webapp/standalone/Dockerfile | 21 +++++++++++----------
docker/nexus-webapp/standalone/docker-entrypoint.sh | 2 ++
6 files changed, 39 insertions(+), 23 deletions(-)
diff --git a/docker/nexus-webapp/install_conda.sh b/docker/nexus-webapp/install_conda.sh
index 7d99ef2..b5d0a45 100755
--- a/docker/nexus-webapp/install_conda.sh
+++ b/docker/nexus-webapp/install_conda.sh
@@ -16,12 +16,21 @@
set -ebx
-URL=${1:-"https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh"}
-CONDA=${2:-"/usr/local/anaconda2"}
+URL=${1:-"https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh"}
+CONDA=${2:-"/usr/local/anaconda3"}
+
+CONDA_ENV_NAME=${CONDA_ENV_NAME:-"nexus-webapp"}
pushd /tmp
wget -q ${URL} -O install_anaconda.sh
/bin/bash install_anaconda.sh -b -p ${CONDA}
rm install_anaconda.sh
-${CONDA}/bin/conda update -n base conda
+ln -s ${CONDA}/etc/profile.d/conda.sh /etc/profile.d/conda.sh
+chmod +x /usr/local/anaconda3/etc/profile.d/conda.sh
+
+conda update -n base conda
+conda config --set channel_priority strict
+conda config --prepend channels conda-forge
+conda create -y --name ${CONDA_ENV_NAME} python=2
+
popd
diff --git a/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh b/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh
index 3ce4313..c2013ab 100755
--- a/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh
+++ b/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh
@@ -17,6 +17,8 @@
set -ebx
+source activate ${CONDA_ENV_NAME}
+
if [ -n "$TORNADO_HOST" ]; then
sed -i "s/server.socket_host.*/server.socket_host=${TORNADO_HOST}/g" ${NEXUS_SRC}/analysis/webservice/config/web.ini
fi
@@ -39,4 +41,4 @@ cd ${NEXUS_SRC}/analysis
python setup.py install --force
# Set PROJ_LIB env var as workaround for missing environment variable for basemap https://github.com/conda-forge/basemap-feedstock/issues/30
-${MESOS_HOME}/build/bin/mesos-agent.sh --no-systemd_enable_support --launcher=posix --no-switch_user --executor_environment_variables='{"PYTHON_EGG_CACHE": "/tmp", "PROJ_LIB":"/usr/local/anaconda2/share/proj"}' "$@"
+${MESOS_HOME}/build/bin/mesos-agent.sh --no-systemd_enable_support --launcher=posix --no-switch_user --executor_environment_variables="{\"PYTHON_EGG_CACHE\": \"/tmp\", \"PROJ_LIB\": \"${CONDA_HOME}/envs/${CONDA_ENV_NAME}/share/proj\"}" "$@"
diff --git a/docker/nexus-webapp/mesos/base/Dockerfile b/docker/nexus-webapp/mesos/base/Dockerfile
index c62a986..0442eb4 100644
--- a/docker/nexus-webapp/mesos/base/Dockerfile
+++ b/docker/nexus-webapp/mesos/base/Dockerfile
@@ -46,7 +46,8 @@ ENV INSTALL_LOC=/usr/local \
JAVA_HOME=/usr/java/default \
NEXUS_SRC=/tmp/incubator-sdap-nexus
-ENV CONDA_HOME=${INSTALL_LOC}/anaconda2 \
+ENV CONDA_HOME=${INSTALL_LOC}/anaconda3 \
+ CONDA_ENV_NAME=nexus-webapp \
MESOS_HOME=${INSTALL_LOC}/mesos-${MESOS_VER} \
SPARK_DIR=spark-${SPARK_VERSION} \
SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop2.7 \
@@ -54,11 +55,9 @@ ENV CONDA_HOME=${INSTALL_LOC}/anaconda2 \
MESOS_PACKAGE=mesos-${MESOS_VER}.tar.gz
ENV SPARK_HOME=${INSTALL_LOC}/${SPARK_DIR} \
- PYSPARK_DRIVER_PYTHON=${CONDA_HOME}/bin/python \
- PYSPARK_PYTHON=${CONDA_HOME}/bin/python \
- PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell" \
- # Workaround for missing environment variable for basemap https://github.com/conda-forge/basemap-feedstock/issues/30
- PROJ_LIB=${CONDA_HOME}/share/proj
+ PYSPARK_DRIVER_PYTHON=${CONDA_HOME}/envs/${CONDA_ENV_NAME}/bin/python \
+ PYSPARK_PYTHON=${CONDA_HOME}/envs/${CONDA_ENV_NAME}/bin/python \
+ PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell"
ENV PYTHONPATH=${PYTHONPATH}:${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${SPARK_HOME}/python/lib/pyspark.zip \
MESOS_NATIVE_JAVA_LIBRARY=${INSTALL_LOC}/lib/libmesos.so \
@@ -79,10 +78,10 @@ RUN ./install_spark.sh "http://d3kbcqa49mib13.cloudfront.net" ${SPARK_VERSION} $
# Install Anaconda
COPY install_conda.sh ./install_conda.sh
-RUN ./install_conda.sh "https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh" ${CONDA_HOME}
+RUN ./install_conda.sh "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" ${CONDA}
# Conda dependencies for nexus
-RUN conda config --add channels conda-forge && \
+RUN source activate ${CONDA_ENV_NAME} && \
conda install -y netCDF4 \
backports.functools_lru_cache \
numpy \
@@ -102,7 +101,8 @@ ARG APACHE_NEXUS_BRANCH=master
ARG REBUILD_CODE=1
COPY install_nexusproto.sh ./install_nexusproto.sh
COPY install_nexus.sh ./install_nexus.sh
-RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH && \
+RUN source activate ${CONDA_ENV_NAME} && \
+ /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH && \
/tmp/install_nexus.sh $APACHE_NEXUS $APACHE_NEXUS_BRANCH $NEXUS_SRC
CMD ["/bin/bash"]
diff --git a/docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh b/docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh
index e4d1abb..564f1bf 100755
--- a/docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh
+++ b/docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh
@@ -17,6 +17,8 @@
set -ebx
+source activate ${CONDA_ENV_NAME}
+
if [ -n "$TORNADO_HOST" ]; then
sed -i "s/server.socket_host.*/server.socket_host=${TORNADO_HOST}/g" ${NEXUS_SRC}/analysis/webservice/config/web.ini
fi
diff --git a/docker/nexus-webapp/standalone/Dockerfile b/docker/nexus-webapp/standalone/Dockerfile
index 5570938..b474255 100644
--- a/docker/nexus-webapp/standalone/Dockerfile
+++ b/docker/nexus-webapp/standalone/Dockerfile
@@ -29,22 +29,23 @@ RUN yum -y update && \
yum clean all
ARG SPARK_VERSION=2.2.0
+ENV CONDA=/usr/local/anaconda3 \
+ CONDA_ENV_NAME=nexus-webapp
ENV SPARK_LOCAL_IP=127.0.0.1 \
CASSANDRA_CONTACT_POINTS=127.0.0.1 \
+ CASSANDRA_PORT=9042 \
CASSANDRA_LOCAL_DATACENTER=datacenter1 \
SOLR_URL_PORT=127.0.0.1:8983 \
SPARK_DIR=spark-${SPARK_VERSION} \
SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop2.7 \
SPARK_HOME=/usr/local/spark-${SPARK_VERSION} \
- PYSPARK_DRIVER_PYTHON=/usr/local/anaconda2/bin/python \
- PYSPARK_PYTHON=/usr/local/anaconda2/bin/python \
+ PYSPARK_DRIVER_PYTHON=${CONDA}/envs/${CONDA_ENV_NAME}/bin/python \
+ PYSPARK_PYTHON=${CONDA}/envs/${CONDA_ENV_NAME}/bin/python \
PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell" \
PYTHONPATH=${PYTHONPATH}:/usr/local/spark-${SPARK_VERSION}/python:/usr/local/spark-${SPARK_VERSION}/python/lib/py4j-0.10.4-src.zip:/usr/local/spark-${SPARK_VERSION}/python/lib/pyspark.zip \
SPARK_EXECUTOR_URI=/usr/local/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz \
- NEXUS_SRC=/tmp/incubator-sdap-nexus \
- PATH=/usr/local/anaconda2/bin:$PATH \
- # Workaround for missing environment variable for basemap https://github.com/conda-forge/basemap-feedstock/issues/30
- PROJ_LIB=/usr/local/anaconda2/share/proj
+ NEXUS_SRC=/usr/local/incubator-sdap-nexus \
+ PATH=${CONDA}/bin:${PATH}
WORKDIR /tmp
@@ -58,11 +59,10 @@ RUN ./install_spark.sh "http://d3kbcqa49mib13.cloudfront.net" ${SPARK_VERSION} $
# Install Anaconda
COPY install_conda.sh ./install_conda.sh
-RUN ./install_conda.sh "https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh" /usr/local/anaconda2
+RUN ./install_conda.sh "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" ${CONDA}
# Conda dependencies for nexus
-# RUN conda install -c conda-forge -y netCDF4 backports.functools_lru_cache=1.3 && \
-RUN conda config --add channels conda-forge && \
+RUN source activate ${CONDA_ENV_NAME} && \
conda install -y netCDF4 \
backports.functools_lru_cache \
numpy \
@@ -82,7 +82,8 @@ ARG APACHE_NEXUS_BRANCH=master
ARG REBUILD_CODE=1
COPY install_nexusproto.sh ./install_nexusproto.sh
COPY install_nexus.sh ./install_nexus.sh
-RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH && \
+RUN source activate ${CONDA_ENV_NAME} && \
+ /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH && \
/tmp/install_nexus.sh $APACHE_NEXUS $APACHE_NEXUS_BRANCH $NEXUS_SRC
COPY standalone/docker-entrypoint.sh ./docker-entrypoint.sh
diff --git a/docker/nexus-webapp/standalone/docker-entrypoint.sh b/docker/nexus-webapp/standalone/docker-entrypoint.sh
index 20270ac..adfcebc 100755
--- a/docker/nexus-webapp/standalone/docker-entrypoint.sh
+++ b/docker/nexus-webapp/standalone/docker-entrypoint.sh
@@ -17,6 +17,8 @@
set -ebx
+source activate ${CONDA_ENV_NAME}
+
if [ -n "$TORNADO_HOST" ]; then
sed -i "s/server.socket_host.*/server.socket_host=${TORNADO_HOST}/g" ${NEXUS_SRC}/analysis/webservice/config/web.ini
fi