You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2025/11/28 21:53:24 UTC
(airflow) branch main updated: feat: add PYTHON_LTO build arg for FIPS compliance (#58583)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 1914cff318c feat: add PYTHON_LTO build arg for FIPS compliance (#58583)
1914cff318c is described below
commit 1914cff318ca074d9737049e10d1e4e77f2057b9
Author: Subham <su...@adypu.edu.in>
AuthorDate: Sat Nov 29 03:23:17 2025 +0530
feat: add PYTHON_LTO build arg for FIPS compliance (#58583)
* feat: add PYTHON_LTO build arg for FIPS compliance
* Fix static checks and add newsfragment for FIPS support
* docs: add comprehensive LTO-FIPS documentation with MD5 explanation
* docs: add FIPS compliance documentation and note about other potential incompatibilities
* Fix spelling and update Dockerfiles
* Fix formatting errors via pre-commit
---
Dockerfile | 33 ++++++++++++++++++++++++++--
Dockerfile.ci | 29 ++++++++++++++++++++++--
airflow-core/newsfragments/58337.feature.rst | 1 +
docker-stack-docs/build.rst | 24 ++++++++++++++++++++
docs/spelling_wordlist.txt | 1 +
scripts/docker/install_os_dependencies.sh | 11 +++++++++-
6 files changed, 94 insertions(+), 5 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 21be4de77e4..cf1dee4200d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -51,6 +51,21 @@ ARG AIRFLOW_VERSION="3.1.3"
ARG BASE_IMAGE="debian:bookworm-slim"
ARG AIRFLOW_PYTHON_VERSION="3.12.12"
+# PYTHON_LTO: Controls whether Python is built with Link-Time Optimization (LTO).
+#
+# Link-Time Optimization uses MD5 checksums during the compilation process to verify
+# object files and intermediate representations. In FIPS-compliant environments, MD5
+# is blocked as it's not an approved cryptographic algorithm (see FIPS 140-2/140-3).
+# This can cause Python builds with LTO to fail when FIPS mode is enabled.
+#
+# When building FIPS-compliant images, set this to "false" to disable LTO:
+# docker build --build-arg PYTHON_LTO="false" ...
+#
+# Default: "true" (LTO enabled for better performance)
+#
+# Related: https://github.com/apache/airflow/issues/58337
+ARG PYTHON_LTO="true"
+
# You can swap comments between those two args to test pip from the main version
# When you attempt to test if the version of `pip` from specified branch works for our builds
# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
@@ -104,6 +119,7 @@ if [[ "$#" != 1 ]]; then
fi
AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}
+PYTHON_LTO=${PYTHON_LTO:-true}
GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
if [[ "${1}" == "runtime" ]]; then
@@ -389,9 +405,17 @@ function install_python() {
EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer";
LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
LDFLAGS="${LDFLAGS:--Wl},--strip-all"
+ # Link-Time Optimization (LTO) uses MD5 checksums for object file verification during
+ # compilation. In FIPS mode, MD5 is blocked as a non-approved algorithm, causing builds
+ # to fail. The PYTHON_LTO variable allows disabling LTO for FIPS-compliant builds.
+ # See: https://github.com/apache/airflow/issues/58337
+ local lto_option=""
+ if [[ "${PYTHON_LTO:-true}" == "true" ]]; then
+ lto_option="--with-lto"
+ fi
./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip --build="$gnuArch" \
--enable-loadable-sqlite-extensions --enable-option-checking=fatal \
- --enable-shared --with-lto
+ --enable-shared ${lto_option}
make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python
make -s -j "$(nproc)" install
@@ -1684,8 +1708,11 @@ ENV DEV_APT_DEPS=${DEV_APT_DEPS} \
ADDITIONAL_DEV_APT_ENV=${ADDITIONAL_DEV_APT_ENV} \
AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION}
+ARG PYTHON_LTO
+
+
COPY --from=scripts install_os_dependencies.sh /scripts/docker/
-RUN bash /scripts/docker/install_os_dependencies.sh dev
+RUN PYTHON_LTO=${PYTHON_LTO} bash /scripts/docker/install_os_dependencies.sh dev
# In case system python is installed, setting LD_LIBRARY_PATH prevents any case the system python
# libraries will be accidentally used before the library installed from sources (which is newer and
@@ -1931,6 +1958,8 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm" \
AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD}
+ARG PYTHON_LTO
+
COPY --from=airflow-build-image "/usr/python/" "/usr/python/"
COPY --from=scripts install_os_dependencies.sh /scripts/docker/
RUN bash /scripts/docker/install_os_dependencies.sh runtime
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 1304ff94ba6..9d66c4aa9e1 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -16,6 +16,20 @@
#
# WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT.
#
+# PYTHON_LTO: Controls whether Python is built with Link-Time Optimization (LTO).
+#
+# Link-Time Optimization uses MD5 checksums during the compilation process to verify
+# object files and intermediate representations. In FIPS-compliant environments, MD5
+# is blocked as it's not an approved cryptographic algorithm (see FIPS 140-2/140-3).
+# This can cause Python builds with LTO to fail when FIPS mode is enabled.
+#
+# When building FIPS-compliant images, set this to "false" to disable LTO:
+# docker build --build-arg PYTHON_LTO="false" ...
+#
+# Default: "true" (LTO enabled for better performance)
+#
+# Related: https://github.com/apache/airflow/issues/58337
+ARG PYTHON_LTO="true"
ARG BASE_IMAGE="debian:bookworm-slim"
##############################################################################################
@@ -44,6 +58,7 @@ if [[ "$#" != 1 ]]; then
fi
AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}
+PYTHON_LTO=${PYTHON_LTO:-true}
GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
if [[ "${1}" == "runtime" ]]; then
@@ -329,9 +344,17 @@ function install_python() {
EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer";
LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
LDFLAGS="${LDFLAGS:--Wl},--strip-all"
+ # Link-Time Optimization (LTO) uses MD5 checksums for object file verification during
+ # compilation. In FIPS mode, MD5 is blocked as a non-approved algorithm, causing builds
+ # to fail. The PYTHON_LTO variable allows disabling LTO for FIPS-compliant builds.
+ # See: https://github.com/apache/airflow/issues/58337
+ local lto_option=""
+ if [[ "${PYTHON_LTO:-true}" == "true" ]]; then
+ lto_option="--with-lto"
+ fi
./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip --build="$gnuArch" \
--enable-loadable-sqlite-extensions --enable-option-checking=fatal \
- --enable-shared --with-lto
+ --enable-shared ${lto_option}
make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python
make -s -j "$(nproc)" install
@@ -1528,9 +1551,11 @@ ARG AIRFLOW_PYTHON_VERSION="3.12.12"
ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION}
ENV GOLANG_MAJOR_MINOR_VERSION="1.25.4"
+ARG PYTHON_LTO
+
COPY --from=scripts install_os_dependencies.sh /scripts/docker/
-RUN bash /scripts/docker/install_os_dependencies.sh ci
+RUN PYTHON_LTO=${PYTHON_LTO} bash /scripts/docker/install_os_dependencies.sh ci
# In case system python is installed, setting LD_LIBRARY_PATH prevents any case the system python
# libraries will be accidentally used before the library installed from sources (which is newer and
diff --git a/airflow-core/newsfragments/58337.feature.rst b/airflow-core/newsfragments/58337.feature.rst
new file mode 100644
index 00000000000..e2720735819
--- /dev/null
+++ b/airflow-core/newsfragments/58337.feature.rst
@@ -0,0 +1 @@
+Enable FIPS Support by making Python LTO configurable via ``PYTHON_LTO`` build argument
diff --git a/docker-stack-docs/build.rst b/docker-stack-docs/build.rst
index cabbdfff6b2..e22c393f91c 100644
--- a/docker-stack-docs/build.rst
+++ b/docker-stack-docs/build.rst
@@ -862,6 +862,30 @@ vetted by the security teams. It is also the most complex way of building the im
expert of building and using Dockerfiles in order to use it and have to have specific needs of security if
you want to follow that route.
+.. _image-build-fips:
+
+Build images in FIPS-compliant environments
+...........................................
+
+If you are building images in a FIPS-compliant environment, you might encounter issues with the default
+build process. For example, the default build process uses ``--with-lto`` (Link Time Optimization) when
+building Python, which might fail in FIPS mode because LTO uses MD5 checksums to verify object files
+during compilation, and MD5 is blocked in FIPS mode.
+
+In order to build the image in FIPS-compliant environment, you can use ``PYTHON_LTO`` build argument
+and set it to ``false``.
+
+.. code-block:: bash
+
+ docker build . --build-arg PYTHON_LTO="false" --tag my-image:my-tag
+
+.. note::
+
+ While disabling LTO is necessary for FIPS compliance during the build process, it is not sufficient
+ to make the image fully FIPS compliant. There might be other reasons for FIPS incompatibility
+ (for example usage of non-FIPS compliant algorithms in the software installed in the image).
+ You should verify the compliance of the image yourself.
+
This builds below builds the production image with packages and constraints used from the local
``docker-context-files`` rather than installed from PyPI or GitHub. It also disables MySQL client
installation as it is using external installation method.
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index 0c1dcdb15f1..de8c0cabf24 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -704,6 +704,7 @@ filesystems
filetype
finalizers
findall
+fips
firebase
Firehose
Firestore
diff --git a/scripts/docker/install_os_dependencies.sh b/scripts/docker/install_os_dependencies.sh
index 0488f83ec89..75151bb1f0e 100644
--- a/scripts/docker/install_os_dependencies.sh
+++ b/scripts/docker/install_os_dependencies.sh
@@ -26,6 +26,7 @@ if [[ "$#" != 1 ]]; then
fi
AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}
+PYTHON_LTO=${PYTHON_LTO:-true}
GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
if [[ "${1}" == "runtime" ]]; then
@@ -311,9 +312,17 @@ function install_python() {
EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer";
LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
LDFLAGS="${LDFLAGS:--Wl},--strip-all"
+ # Link-Time Optimization (LTO) uses MD5 checksums for object file verification during
+ # compilation. In FIPS mode, MD5 is blocked as a non-approved algorithm, causing builds
+ # to fail. The PYTHON_LTO variable allows disabling LTO for FIPS-compliant builds.
+ # See: https://github.com/apache/airflow/issues/58337
+ local lto_option=""
+ if [[ "${PYTHON_LTO:-true}" == "true" ]]; then
+ lto_option="--with-lto"
+ fi
./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip --build="$gnuArch" \
--enable-loadable-sqlite-extensions --enable-option-checking=fatal \
- --enable-shared --with-lto
+ --enable-shared ${lto_option}
make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python
make -s -j "$(nproc)" install