[GitHub] [airflow] svivier-orange commented on a change in pull request #6501: [AIRFLOW-5831] Prod image support. Depends on [AIRFLOW-5704] [AIRFLOW-5842] [AIRFLOW-5828]

[GitBox <gi...@apache.org>]

svivier-orange commented on a change in pull request #6501: [AIRFLOW-5831] Prod image support. Depends on [AIRFLOW-5704] [AIRFLOW-5842] [AIRFLOW-5828]
URL: https://github.com/apache/airflow/pull/6501#discussion_r345599242
 
 

 ##########
 File path: Dockerfile
 ##########
 @@ -355,21 +348,135 @@ COPY .coveragerc .rat-excludes .flake8 pylintrc LICENSE MANIFEST.in NOTICE CHANG
      setup.cfg setup.py \
      ${AIRFLOW_SOURCES}/
 
-WORKDIR ${AIRFLOW_SOURCES}
+# Reinstall airflow again - this time with sources and remove the sources after installation
+# It is not perfect because the sources are added as layer but it is still better
+RUN pip install --user -e ".[${AIRFLOW_CI_EXTRAS}]"
 
 # Additional python deps to install
 ARG ADDITIONAL_PYTHON_DEPS=""
 
 RUN if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
-        pip install ${ADDITIONAL_PYTHON_DEPS}; \
+        pip install --user ${ADDITIONAL_PYTHON_DEPS}; \
     fi
 
+
 WORKDIR ${AIRFLOW_SOURCES}
 
 ENV PATH="${HOME}:${PATH}"
 
 EXPOSE 8080
 
-ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/entrypoint.sh"]
+ENTRYPOINT ["/root/.local/bin/dumb-init", "--", "/entrypoint.sh"]
+
+CMD ["--help"]
+
+############################################################################################################
+# This is separate stage for packaging. WWW files with npm so that no node is needed for production image
+############################################################################################################
+FROM ${NODE_BASE_IMAGE} as airflow-www
+
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+ARG AIRFLOW_SOURCES=/opt/airflow
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+COPY airflow/www/ ${AIRFLOW_SOURCES}/airflow/www/
+
+WORKDIR ${AIRFLOW_SOURCES}/airflow/www
+
+RUN npm ci
+
+RUN mkdir -p "${AIRFLOW_SOURCES}/airflow/www/static" \
+    && mkdir -p "${AIRFLOW_SOURCES}/docs/build/_html" \
+    && pushd "${AIRFLOW_SOURCES}/airflow/www/static" || exit \
+    && ln -sf ../../../docs/_build/html docs \
+    && popd || exit
+
+# Package NPM for production
+RUN npm run prod
+
+# Remove node modules
+RUN rm -rf ${AIRFLOW_SOURCES}/airflow/www/node_modules
+
+############################################################################################################
+# Airflow code for copying
+############################################################################################################
+FROM airflow-base-devel as airflow-code
+
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+# Airflow Extras installed
+ARG AIRFLOW_PROD_EXTRAS="all"
+ENV AIRFLOW_PROD_EXTRAS=${AIRFLOW_PROD_EXTRAS}
+
+# Cache for this line will be automatically invalidated if any
+# of airflow sources change
+COPY . ${AIRFLOW_SOURCES}/
+
+# Setting to 1 speeds up building the image. Cassandra driver without CYTHON saves around 10 minutes
+# But might not be suitable for production image
+ENV CASS_DRIVER_NO_CYTHON=""
+ENV CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+# Reinstall airflow again - this time with sources and remove the sources after installation
+# It is not perfect because the sources are added as layer but it is still better
+RUN pip install --user ".[${AIRFLOW_PROD_EXTRAS}]"
+
+############################################################################################################
+# Production-ready Airflow image
+############################################################################################################
+FROM airflow-base as airflow-prod
+
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+# Airflow Extras installed
+ARG AIRFLOW_PROD_EXTRAS="all"
+ENV AIRFLOW_PROD_EXTRAS=${AIRFLOW_PROD_EXTRAS}
+
+RUN echo "Installing with extras: ${AIRFLOW_PROD_EXTRAS}."
+
+ARG AIRFLOW_HOME=/home/airflow/airflow_home
+ENV AIRFLOW_HOME=${AIRFLOW_HOME}
+
+RUN chown airflow.airflow ${AIRFLOW_HOME}
+
+USER airflow
+
+ARG AIRFLOW_SOURCES=/opt/airflow
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+WORKDIR ${AIRFLOW_SOURCES}
+
+ENV PATH="/home/airflow/.local/bin:${AIRFLOW_HOME}:${PATH}"
+
+COPY --chown=airflow:airflow --from=airflow-code /root/.local /home/airflow/.local
+
+# Additional python deps to install
+ARG ADDITIONAL_PYTHON_DEPS=""
+
+RUN if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
+        pip install --user ${ADDITIONAL_PYTHON_DEPS}; \
+    fi
+
+COPY --chown=airflow:airflow ./scripts/docker/entrypoint.sh /entrypoint.sh
+
+# Copy Airflow www packages
+COPY --chown=airflow:airflow --from=airflow-www /opt/airflow/airflow/www ${HOME}/.local/airflow/www
+
+RUN mkdir -pv "${AIRFLOW_HOME}" \
+    && mkdir -pv "${AIRFLOW_HOME}/dags" \
+    && mkdir -pv "${AIRFLOW_HOME}/logs"
+
+ENV AIRFLOW_USER=airflow
+ENV HOME=/home/airflow
+
+# Set writing bytecode back in prod environment
+ENV PYTHONDONTWRITEBYTECODE=""
+
+WORKDIR ${AIRFLOW_HOME}
+
+EXPOSE 8080
+
+ENTRYPOINT ["/home/airflow/.local/bin/dumb-init", "--", "/entrypoint.sh"]
 
 Review comment:
   This entrypoint is notcompatible with the current helm chart because a lot of bash cmd is used, and therefore it will fail .

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services