You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2025/11/26 14:57:05 UTC
(airflow) branch main updated: Check docker and docker compose at startup of task sdk integration tests (#58725)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c65241ff455 Check docker and docker compose at startup of task sdk integration tests (#58725)
c65241ff455 is described below
commit c65241ff455852550bd21d1513d37645aa122e7b
Author: Amogh Desai <am...@gmail.com>
AuthorDate: Wed Nov 26 20:26:58 2025 +0530
Check docker and docker compose at startup of task sdk integration tests (#58725)
---
.../tests/task_sdk_tests/conftest.py | 143 +++++++++++++++++++++
.../tests/task_sdk_tests/constants.py | 2 +
2 files changed, 145 insertions(+)
diff --git a/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py b/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py
index 756473badb3..f0ac9c167ed 100644
--- a/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py
+++ b/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py
@@ -18,6 +18,7 @@ from __future__ import annotations
import os
import platform
+import re
import subprocess
import sys
from collections.abc import Callable
@@ -25,6 +26,7 @@ from pathlib import Path
import pytest
import requests
+from packaging import version
from tenacity import (
retry,
retry_if_exception_type,
@@ -37,6 +39,8 @@ from task_sdk_tests import console
from task_sdk_tests.constants import (
AIRFLOW_ROOT_PATH,
DOCKER_IMAGE,
+ MIN_DOCKER_COMPOSE_VERSION,
+ MIN_DOCKER_VERSION,
TASK_SDK_HOST_PORT,
TASK_SDK_INTEGRATION_DOCKER_COMPOSE_FILE_PATH,
TASK_SDK_INTEGRATION_ENV_FILE,
@@ -46,6 +50,142 @@ from task_sdk_tests.constants import (
from tests_common.test_utils.fernet import generate_fernet_key_string
+def compare_gte(a: str, b: str) -> bool:
+ """Compare two versions for greater than equal to. Validates a>=b"""
+ try:
+ return version.parse(a) >= version.parse(b)
+ except Exception:
+ return False
+
+
+def check_docker_is_running() -> bool:
+ try:
+ result = subprocess.run(
+ ["docker", "info"],
+ capture_output=True,
+ text=False,
+ check=False,
+ timeout=10,
+ )
+ return result.returncode == 0
+ except Exception:
+ return False
+
+
+def check_docker_version() -> tuple[bool, str]:
+ try:
+ result = subprocess.run(
+ ["docker", "version", "--format", "{{.Client.Version}}"],
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=10,
+ )
+
+ if result.returncode != 0:
+ return False, "unknown"
+
+ docker_version_str = result.stdout.strip()
+ # extract version number (e.g., "25.0.0" from "25.0.0" or "25.0.0-ce")
+ version_pattern = re.compile(r"(\d+)\.(\d+)\.(\d+)")
+ version_match = version_pattern.search(docker_version_str)
+
+ if version_match:
+ docker_version_str = ".".join(version_match.groups())
+ is_valid = compare_gte(docker_version_str, MIN_DOCKER_VERSION)
+ return is_valid, docker_version_str
+
+ return False, "unknown"
+ except Exception:
+ return False, "unknown"
+
+
+def check_docker_compose_version() -> tuple[bool, str]:
+ try:
+ result = subprocess.run(
+ ["docker", "compose", "version"],
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=10,
+ )
+
+ if result.returncode != 0:
+ return False, "unknown"
+
+ docker_compose_output = result.stdout.strip()
+ # extract version number from output like "Docker Compose version v2.20.2"
+ version_pattern = re.compile(r"(\d+)\.(\d+)\.(\d+)")
+ version_match = version_pattern.search(docker_compose_output)
+
+ if version_match:
+ compose_version_str = ".".join(version_match.groups())
+ is_valid = compare_gte(compose_version_str, MIN_DOCKER_COMPOSE_VERSION)
+ return is_valid, compose_version_str
+
+ return False, "unknown"
+ except Exception:
+ return False, "unknown"
+
+
+def check_docker_requirements():
+ """
+ Verify docker and docker Compose meet minimum version crieteria.
+
+ 1. Check if Docker is running
+ 2. Check Docker version
+ 3. Check Docker Compose version
+ """
+ console.print("[yellow]Checking Docker requirements...[/]")
+
+ if not check_docker_is_running():
+ console.print(
+ "[red]Docker is not running.[/]\n"
+ "[yellow]Please make sure Docker is installed and running.[/]\n"
+ "Installation instructions: https://docs.docker.com/engine/install/"
+ )
+ sys.exit(1)
+
+ valid, docker_version_str = check_docker_version()
+ if not valid:
+ if docker_version_str == "unknown":
+ console.print(
+ f"[red]Could not determine Docker version.[/]\n"
+ f"[yellow]Please make sure Docker is at least version {MIN_DOCKER_VERSION}.[/]\n"
+ "Installation instructions: https://docs.docker.com/engine/install/"
+ )
+ else:
+ console.print(
+ f"[red]Docker version {docker_version_str} is too old.[/]\n"
+ f"[yellow]Minimum required version: {MIN_DOCKER_VERSION}[/]\n"
+ "Installation instructions: https://docs.docker.com/engine/install/"
+ )
+ sys.exit(1)
+
+ console.print(f"[green]Docker version {docker_version_str} meets requirements[/]")
+
+ valid, compose_version_str = check_docker_compose_version()
+ if not valid:
+ if compose_version_str == "unknown":
+ console.print(
+ "[red]Could not determine Docker Compose version.[/]\n"
+ "[yellow]You either do not have docker-compose or have docker-compose v1 installed.[/]\n"
+ "[yellow]Breeze does not support docker-compose v1 any more as it has been replaced by v2.[/]\n"
+ f"[yellow]Minimum required version: {MIN_DOCKER_COMPOSE_VERSION}[/]\n"
+ "Follow https://docs.docker.com/compose/migrate/ to migrate to v2\n"
+ "Installation instructions: https://docs.docker.com/compose/install/"
+ )
+ else:
+ console.print(
+ f"[red]Docker Compose version {compose_version_str} is too old.[/]\n"
+ f"[yellow]Minimum required version: {MIN_DOCKER_COMPOSE_VERSION}[/]\n"
+ "Installation instructions: https://docs.docker.com/compose/install/"
+ )
+ sys.exit(1)
+
+ console.print(f"[green]Docker Compose version {compose_version_str} meets requirements[/]")
+
+
def print_diagnostics(compose, compose_version, docker_version):
"""Print diagnostic information when test fails."""
console.print("[red]=== DIAGNOSTIC INFORMATION ===[/]")
@@ -193,6 +333,9 @@ def docker_compose_setup(tmp_path_factory):
def pytest_sessionstart(session):
"""Install Task SDK at the very start of the pytest session."""
+ # Check Docker requirements first (similar to what Breeze does)
+ check_docker_requirements()
+
console.print("[yellow]Installing apache-airflow-task-sdk via pytest_sessionstart...")
task_sdk_version = os.environ.get("TASK_SDK_VERSION")
diff --git a/task-sdk-integration-tests/tests/task_sdk_tests/constants.py b/task-sdk-integration-tests/tests/task_sdk_tests/constants.py
index 1de133d5856..a5b913879c6 100644
--- a/task-sdk-integration-tests/tests/task_sdk_tests/constants.py
+++ b/task-sdk-integration-tests/tests/task_sdk_tests/constants.py
@@ -23,6 +23,8 @@ AIRFLOW_ROOT_PATH = Path(__file__).resolve().parents[3]
TASK_SDK_INTEGRATION_TESTS_ROOT = AIRFLOW_ROOT_PATH / "task-sdk-integration-tests"
DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.10"
+MIN_DOCKER_VERSION = "25.0.0"
+MIN_DOCKER_COMPOSE_VERSION = "2.20.2"
DEFAULT_DOCKER_IMAGE = f"ghcr.io/apache/airflow/main/prod/python{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}:latest"
DOCKER_IMAGE = os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE