You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2026/09/21 06:29:19 UTC
(airflow) branch main updated: Add a Modal backend for the sandbox toolset (#72910)
This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 51876b066a0 Add a Modal backend for the sandbox toolset (#72910)
51876b066a0 is described below
commit 51876b066a0b0406992233b224e3cd5a856b40ff
Author: Kaxil Naik <ka...@apache.org>
AuthorDate: Mon Sep 21 07:29:08 2026 +0100
Add a Modal backend for the sandbox toolset (#72910)
---
docs/spelling_wordlist.txt | 3 +
providers/common/ai/README.rst | 1 +
providers/common/ai/docs/choosing_a_toolset.rst | 81 +-
providers/common/ai/docs/index.rst | 10 +-
providers/common/ai/docs/operators/agent.rst | 16 +-
providers/common/ai/docs/sandbox.rst | 821 ++++++++++++++
providers/common/ai/docs/toolsets.rst | 330 +-----
providers/common/ai/provider.yaml | 3 +
providers/common/ai/pyproject.toml | 4 +
.../ai/example_dags/example_sandbox_toolset.py | 282 +++++
.../providers/common/ai/get_provider_info.py | 5 +
.../providers/common/ai/sandbox/__init__.py | 14 +
.../airflow/providers/common/ai/sandbox/base.py | 21 +-
.../airflow/providers/common/ai/sandbox/modal.py | 868 +++++++++++++++
.../providers/common/ai/toolsets/sandbox.py | 52 +-
.../common/ai/example_sandbox_toolset_modal.py | 258 +++++
.../ai/tests/unit/common/ai/sandbox/fake_modal.py | 322 ++++++
.../ai/tests/unit/common/ai/sandbox/test_base.py | 17 +
.../ai/tests/unit/common/ai/sandbox/test_modal.py | 1120 ++++++++++++++++++++
.../tests/unit/common/ai/toolsets/test_sandbox.py | 107 ++
uv.lock | 120 ++-
21 files changed, 4095 insertions(+), 360 deletions(-)
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index 67308f30971..104a9ecb98c 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -774,6 +774,7 @@ gsuite
gte
Gunicorn
gunicorn
+gVisor
gz
Gzip
gzipped
@@ -1335,6 +1336,7 @@ pretify
prev
Proc
proc
+procfs
productionalize
ProductSearchClient
profiler
@@ -1582,6 +1584,7 @@ SlackHook
SlackResponse
slas
smtp
+SNI
SnowflakeHook
Snowpark
snowpark
diff --git a/providers/common/ai/README.rst b/providers/common/ai/README.rst
index b7b02d9ddb2..4f9511b64ee 100644
--- a/providers/common/ai/README.rst
+++ b/providers/common/ai/README.rst
@@ -91,6 +91,7 @@ Extra Dependencies
``openai`` ``pydantic-ai-slim[openai]>=2.23.0``
``mcp`` ``pydantic-ai-slim[mcp]>=2.23.0``
``typesafe`` ``typesafe-sdk>=0.6.0``
+``modal`` ``modal>=1.5.0``
``code-mode`` ``pydantic-ai-harness[codemode]>=0.3.0``
``shields`` ``pydantic-ai-shields>=0.3.4``
``skills`` ``apache-airflow-providers-git>=0.4.0``, ``pydantic-ai-skills>=1.2.0``
diff --git a/providers/common/ai/docs/choosing_a_toolset.rst b/providers/common/ai/docs/choosing_a_toolset.rst
index e764e52c204..3ac32937e33 100644
--- a/providers/common/ai/docs/choosing_a_toolset.rst
+++ b/providers/common/ai/docs/choosing_a_toolset.rst
@@ -375,8 +375,10 @@ and any script execution happen on the worker.
**Choose it when** the work is running code the model wrote, rather than calling
a tool you picked in advance — exploratory analysis, installing a package for one
-task, producing a file. Every other route on this page answers "call this thing";
-this one answers "here is somewhere to work".
+task, a script the model writes, runs, and fixes from its own traceback. Every
+other route on this page answers "call this thing"; this one answers "here is
+somewhere to work". It has its own page, :doc:`sandbox`, with worked scenarios
+and the limitations to read before designing a Dag around it.
Before following this row, check whether the actual need is narrower than that:
``code_mode=True`` is a flag on ``AgentOperator``, not a toolset on this table
@@ -384,7 +386,7 @@ Before following this row, check whether the actual need is narrower than that:
code to call several of them instead of emitting one call per step. It does not
give the agent somewhere to run arbitrary code of its own, and it avoids the
``sbx`` backend's production-readiness, network-isolation, and reclamation
-caveats below — but not the reachability one: the generated code runs in
+caveats below and needs no backend at all — but not the reachability one: the generated code runs in
Monty's deny-by-default sandbox, but the tools it calls still run in the
worker, so a credential-bearing toolset on the same agent stays within reach
whether or not code mode is on. See :ref:`code-mode` and
@@ -392,20 +394,23 @@ whether or not code mode is on. See :ref:`code-mode` and
**What it cannot do**
-- The only backend that ships is not built for production. ``SbxSandboxBackend``
- drives Docker Sandboxes, and its own documentation in this provider says to use
- it for local development, calling a worker-driven run "off-label use". It wants
- the ``sbx`` binary on the host, an authenticated Docker account, a one-time
- ``sbx policy init``, and on Linux KVM or nested virtualization — which an
- unprivileged container cannot provide. Anything beyond local development means
- writing another backend behind
- :class:`~airflow.providers.common.ai.sandbox.SandboxBackend`.
+- Only one of its two backends runs on Kubernetes. ``SbxSandboxBackend`` drives
+ Docker Sandboxes on the worker host, and its own documentation says to use it
+ for local development: it wants the ``sbx`` binary on the host, an
+ authenticated Docker account, a one-time ``sbx policy init``, and on Linux KVM
+ or nested virtualization — which an unprivileged container cannot provide.
+ Production and Kubernetes use
+ :class:`~airflow.providers.common.ai.sandbox.modal.ModalSandboxBackend`, a
+ hosted backend behind the ``modal`` extra that installs nothing on the worker
+ and reclaims a sandbox at its own lifetime if the worker dies. Both implement
+ :class:`~airflow.providers.common.ai.sandbox.SandboxBackend`, and a third
+ vendor can too.
- It does not contain the agent. Only what these tools do runs in the sandbox;
the agent loop, the model calls, and every other toolset on the same agent stay
in the worker with the worker's credentials. It contains model-written code, so
pairing it with a credential-bearing toolset on the same agent puts the
credential back within reach. :ref:`sandbox-boundaries` sets this out in full.
-- It cannot enforce network isolation on its own. The backend applies
+- The ``sbx`` backend cannot enforce network isolation on its own. It applies
``allow_egress_to`` as a per-sandbox policy rule, but only on top of a
``deny-all`` host policy, since a local rule can narrow egress and never
widen it; ``block_network`` has no per-sandbox enforcement at all. Ask for
@@ -413,26 +418,40 @@ whether or not code mode is on. See :ref:`code-mode` and
backend raises rather than silently leaving the sandbox less restricted
than the spec asked for. ``block_network`` defaults to ``True``, so a bare
``SandboxSpec()`` with no arguments already asks for it and is refused
- under the default ``host_network_policy="unknown"``.
-- It does not guarantee reclamation. A failed teardown is logged as a warning
+ under the default ``host_network_policy="unknown"``. On Modal the same default
+ maps onto the sandbox's own ``block_network`` and is enforced exactly; a
+ hostname allowlist there is matched on the TLS handshake name and has to be
+ opted into, for the reasons set out on :doc:`sandbox`.
+- Reclamation depends on the backend. A failed teardown is logged as a warning
rather than raised, deliberately, so that a teardown blip cannot fail a
- finished run. Nothing else picks up the slack: the backend that ships has no
- server-side TTL to fall back on, so a worker killed outright leaves the microVM
- and its workspace directory behind. They are named ``airflow-sandbox-*`` so an
- operator can find and remove them — budget for that sweep.
-
-**A real example.** ``tests/system/common/ai/example_sandbox_toolset_sbx.py``
-in this provider is a runnable Dag against a real ``sbx`` host — the only
-toolset here with a system test. It is reachable from the System Tests entry in
-the sidebar.
+ finished run. On ``sbx`` nothing else picks up the slack: there is no
+ server-side TTL, so a worker killed outright leaves the microVM and its
+ workspace directory behind, named ``airflow-sandbox-*`` so an operator can find
+ and remove them. On Modal the sandbox ends at its own ``sandbox_timeout``
+ whatever became of the worker.
+- Nothing survives the run, and a file the agent built can leave only through
+ the model's context, which is text-only and capped. Producing an artifact for a
+ downstream task is a ``@task`` driving a backend directly today, not the agent;
+ :doc:`sandbox` has the example.
+
+**A real example.** ``example_sandbox_toolset.py`` in this provider's example
+Dags has an agent investigating a revenue anomaly on the Modal backend beside a
+``SQLToolset``, the same agent shape on ``sbx`` for a laptop, and a ``@task``
+producing a file through a sandbox; all three are on :doc:`sandbox`. The two
+system tests, ``example_sandbox_toolset_sbx.py`` and
+``example_sandbox_toolset_modal.py``, run against a real backend and are
+reachable from the System Tests entry in the sidebar.
**Credentials and where it runs.** This route does not end at an
Airflow connection. Airflow puts none of its context, connections, variables or
worker environment into the sandbox; only what you pass through
-:class:`~airflow.providers.common.ai.sandbox.SandboxSpec` goes in. Authorization
-is host-level instead — ``sbx login`` and ``sbx policy init``, performed on the
-machine, outside anything Airflow can see. Work runs in a per-session microVM
-on the worker host. Its tool calls act as barriers, as they do for the other
+:class:`~airflow.providers.common.ai.sandbox.SandboxSpec` goes in, and the
+credential that provisions the sandbox never enters it. Authorization to the
+backend sits outside Airflow — ``sbx login`` and ``sbx policy init`` on the
+machine for ``sbx``, or ``MODAL_TOKEN_ID`` and ``MODAL_TOKEN_SECRET`` on the
+worker for Modal. Work runs in a per-run microVM on the worker host with
+``sbx``, or off the worker entirely in Modal's infrastructure. Its tool calls
+act as barriers, as they do for the other
routes that build their own tools; see :ref:`toolset-call-barriers`.
``BaseManagedAgentToolset``
@@ -512,14 +531,16 @@ making on purpose rather than inheriting.
- Worker process
* - ``SandboxToolset``
- None from Airflow. Host-level ``sbx login`` and ``sbx policy init``
- - A microVM on the worker host
+ for ``sbx``; ambient Modal token on the worker for Modal
+ - A microVM on the worker host (``sbx``), or Modal's infrastructure off
+ the worker (Modal)
* - ``BaseManagedAgentToolset``
- Undefined by the base class; the subclass decides
- The vendor's infrastructure
Two rows are worth pausing on. ``SandboxToolset`` deliberately takes no Airflow
-credential — that is the whole point of it, and it substitutes a host-level
-boundary for the connection-level one. ``BaseManagedAgentToolset`` does not
+credential — that is the whole point of it, and it substitutes a backend-level
+boundary, on the host or at the vendor, for the connection-level one. ``BaseManagedAgentToolset`` does not
substitute anything; it simply leaves the question to whoever writes the
subclass. Neither is a defect, but in both cases the access decision has moved
somewhere Airflow cannot see it, and somebody has to make that decision again in
diff --git a/providers/common/ai/docs/index.rst b/providers/common/ai/docs/index.rst
index 29b6c9e211e..ee64ca6a9f2 100644
--- a/providers/common/ai/docs/index.rst
+++ b/providers/common/ai/docs/index.rst
@@ -54,9 +54,10 @@ tools aren't locked out either: pydantic-ai ships ``pydantic_ai.ext.langchain.La
upstream, which wraps LangChain tools for a common.ai agent, and the provider's own
:func:`~airflow.providers.common.ai.toolsets.langchain_bridge.airflow_toolset_to_langchain_tools`
converts the other way — Airflow-managed toolsets into LangChain tools (see :doc:`toolsets`).
-The AI step is orchestrated by Airflow: the model calls, the agent loop, and any tools all
-run in the Airflow worker, where they get retries, logging, and observability like any other
-task.
+The AI step is orchestrated by Airflow: the model calls, the agent loop, and any tools
+run in the Airflow worker by default, where they get retries, logging, and observability like
+any other task. The exception is :ref:`SandboxToolset <sandbox-limitations>`, which exists so
+that code the *model* writes runs somewhere else.
Use it when a Dag needs:
@@ -65,6 +66,7 @@ Use it when a Dag needs:
* **Branching on a model's decision** — :doc:`LLMBranchOperator <operators/llm_branch>`.
* **Agents with tools** — :doc:`AgentOperator <operators/agent>` runs a multi-turn agent loop
in the worker, calling Airflow-defined :doc:`toolsets <toolsets>` (SQL, hooks, MCP servers,
+ a sandboxed shell and filesystem,
:ref:`Agent Skills <agent-skills>`), optionally collapsed into a single sandboxed
:ref:`code mode <code-mode>` call, with optional human-in-the-loop review and durable step
replay — if the task retries after a failure, completed steps are replayed from cache
@@ -159,6 +161,7 @@ See the Optional dependencies table below for the exact package each extra insta
LlamaIndex connection <connections/llamaindex>
Hooks <hooks/index>
Toolsets <toolsets>
+ Sandboxed execution <sandbox>
Choosing a toolset <choosing_a_toolset>
Operators <operators/index>
Examples <examples>
@@ -278,6 +281,7 @@ Extra Dependencies
``openai`` ``pydantic-ai-slim[openai]>=2.23.0``
``typesafe`` ``typesafe-sdk>=0.6.0``
``mcp`` ``pydantic-ai-slim[mcp]>=2.23.0``
+``modal`` ``modal>=1.5.0``
``code-mode`` ``pydantic-ai-harness[codemode]>=0.3.0``
``shields`` ``pydantic-ai-shields>=0.3.4``
``skills`` ``apache-airflow-providers-git>=0.4.0``, ``pydantic-ai-skills>=1.2.0``
diff --git a/providers/common/ai/docs/operators/agent.rst b/providers/common/ai/docs/operators/agent.rst
index 2bec8d4d329..0c040a26c56 100644
--- a/providers/common/ai/docs/operators/agent.rst
+++ b/providers/common/ai/docs/operators/agent.rst
@@ -437,10 +437,18 @@ mode:
expose a hook method that returns the processed result (``HookToolset``). The
tool runs in the full worker environment with all its dependencies, and code
mode just orchestrates it.
-- **Use a container-based execution environment** (e.g. Docker or E2B via
- pydantic-ai-harness) instead of the in-process Monty sandbox. These support
- third-party packages but pay a per-run container cost and a larger security
- surface, so reach for them only when inline library code is genuinely required.
+- **Give the agent a real environment**, with
+ :class:`~airflow.providers.common.ai.toolsets.sandbox.SandboxToolset`. It hands
+ the model a shell and a filesystem in a disposable sandbox off the worker, so
+ third-party packages, a real interpreter and installed binaries are all
+ available. It costs a sandbox per run and a second or so to provision, against
+ well under a millisecond for Monty, so reach for it when inline library code is
+ genuinely required rather than by default. See :doc:`../toolsets` for the
+ backends and their limitations.
+
+The two are not exclusive: ``code_mode=True`` and a ``SandboxToolset`` can be
+enabled together, and the file tools fold into ``run_code`` while ``run_command``
+stays a tool of its own.
Requires the ``code-mode`` extra::
diff --git a/providers/common/ai/docs/sandbox.rst b/providers/common/ai/docs/sandbox.rst
new file mode 100644
index 00000000000..d88731f1d10
--- /dev/null
+++ b/providers/common/ai/docs/sandbox.rst
@@ -0,0 +1,821 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Sandboxed execution for agents
+==============================
+
+An agent that is asked to do open-ended work writes code, and then something has
+to run that code. By default that something is the Airflow worker: a skill
+script, a hand-written tool that shells out, or generated glue in
+:ref:`code mode <code-mode>` all execute on the host that also holds your
+connections, your filesystem and your network position, and the code they run
+was generated a moment ago from inputs you do not control.
+
+:class:`~airflow.providers.common.ai.toolsets.sandbox.SandboxToolset` gives the
+model a disposable workspace for that code instead. It exposes four tools:
+
+.. list-table::
+ :widths: 25 75
+ :header-rows: 1
+
+ * - Tool
+ - What it does
+ * - ``run_command``
+ - Runs a shell command. Pipes, redirection, ``&&`` and globs work. A
+ non-zero exit is reported as output, not raised, so the model reads
+ ``stderr`` and corrects itself.
+ * - ``read_file``
+ - Reads a text file, head-first, and reports the next ``offset`` so the
+ model can page through a long file.
+ * - ``write_file``
+ - Writes text to a file, creating parent directories.
+ * - ``list_directory``
+ - Lists a directory. Directories are shown with a trailing ``/``.
+
+The sandbox is provisioned by a
+:class:`~airflow.providers.common.ai.sandbox.SandboxBackend` on the model's first
+tool call and destroyed when the agent run ends. Two backends ship: a hosted one on
+`Modal <https://modal.com/docs/guide/sandbox>`__ for production and Kubernetes,
+and a local microVM one on `Docker Sandboxes <https://docs.docker.com/ai/sandboxes/>`__
+for development. The four tool names and shapes match pydantic-ai's own sandbox
+capabilities, so a model that has seen one already knows this one.
+
+**Adding this toolset gives the agent shell and file operations in a separate
+workspace. Every other tool keeps its existing permissions and runs where it ran
+before.** Whether that is a new capability depends on what the agent already had:
+an agent with a skill script or a shell tool already ran generated code on the
+worker, and this is where that code should run instead. An agent that only had
+named database operations is being granted a general shell for the first time, in
+a workspace with less authority than the worker, and you are choosing to grant it.
+
+.. _sandbox-quick-start:
+
+Quick start
+-----------
+
+An agent investigating a revenue anomaly. The warehouse is reached through a
+:class:`~airflow.providers.common.ai.toolsets.sql.SQLToolset`, so the credential
+stays in the task and the model only ever sees rows. The arithmetic happens in a
+hosted sandbox with pandas baked into the image, so the sandbox needs no network,
+and a bad pivot or a runaway loop costs a small sandbox rather than a worker slot:
+
+.. exampleinclude:: /../../ai/src/airflow/providers/common/ai/example_dags/example_sandbox_toolset.py
+ :language: python
+ :start-after: [START howto_sandbox_agent_investigation]
+ :end-before: [END howto_sandbox_agent_investigation]
+
+The same toolset on the local ``sbx`` backend, for developing on a laptop. Here
+the agent maps a vendor file whose columns drift onto a staging schema: it writes a
+script, runs it, reads the traceback and fixes it, which is the loop nobody can
+write down in advance. Swapping ``SbxSandboxBackend`` for ``ModalSandboxBackend``
+is the only change between this Dag and production:
+
+.. exampleinclude:: /../../ai/src/airflow/providers/common/ai/example_dags/example_sandbox_toolset.py
+ :language: python
+ :start-after: [START howto_sandbox_agent_local]
+ :end-before: [END howto_sandbox_agent_local]
+
+Install the Modal extra and authenticate as you would for Modal's own CLI. On a
+worker, set ``MODAL_TOKEN_ID`` and ``MODAL_TOKEN_SECRET`` in the environment
+instead; nothing is read until the first sandbox is created, so a Dag file that
+constructs the backend parses without credentials present:
+
+.. code-block:: bash
+
+ pip install 'apache-airflow-providers-common-ai[modal]'
+ modal token new # writes ~/.modal.toml
+
+What it is for, in practice
+---------------------------
+
+The rule is: a model inside a running task writes code you cannot name in advance,
+and that code needs a real shell, a real interpreter or packages. These are the
+jobs that rule describes, each paired with the version where a sandbox is the wrong
+call.
+
+**A vendor drops a CSV nobody has a schema for.** Three date formats, an encoding
+error at row 41,000, pandas needed. Every step depends on the previous step's
+output, and a bad read of the whole file into memory should exhaust a small
+sandbox, not a worker slot shared with other tasks. If the schema is stable and
+the mapping is known, this is a plain task with no agent.
+
+**Yesterday's revenue moved and nobody knows why.** The agent queries through a
+SQL toolset with a table allowlist, writes the rows into the sandbox, and pivots
+with pandas rather than doing arithmetic in tokens. The warehouse credential never
+enters the sandbox. If it is always the same five queries, that is
+:class:`~airflow.providers.common.ai.operators.llm.LLMOperator` over a fixed SQL
+task; if the rows number in the millions, compute in the warehouse rather than
+carrying them through the model's context.
+
+**Migrate forty dbt models between warehouse dialects.** The model files go into
+the workspace, the image carries ``sqlglot`` and ``sqlfluff``, and the loop of
+convert, lint, read errors, fix runs for as long as it takes, fully offline. The
+rewritten files are the deliverable, and getting forty files out is the gap
+described under :ref:`Getting a result out <sandbox-results>`.
+
+**A failing task's traceback, handed to an agent to propose a fix.** The agent
+reproduces the crash against a sample, tries a fix, reruns, and posts a diff for a
+person to review. That code ran with the worker's credentials the first time; the
+reproduction should not. A written diagnosis with no execution is
+``LLMOperator`` reading the log.
+
+**Not a sandbox: flag 200 customers with an open ticket** using two existing
+tools. That is :ref:`code mode <code-mode>`: a short loop over registered tools,
+no packages, no shell, sub-millisecond start.
+
+**Not a sandbox: run a vendor's PII-scanner image over an S3 prefix.** No model
+writes anything. That is ``KubernetesPodOperator``, a fixed payload with templated
+arguments in its own pod.
+
+**Not only a sandbox: one team's agent tasks must run off the shared workers.**
+That is a where-does-the-task-run requirement, so ``KubernetesExecutor`` with a
+pod override, and the sandbox toolset inside the pod if those agents also write
+code. The pod protects the cluster from the task; the sandbox protects the task
+from its model.
+
+.. _sandbox-boundaries:
+
+Choosing the boundary
+---------------------
+
+"Sandboxed" means different things at different layers, and picking the wrong
+layer is the most common way to end up with less protection than you think. Four
+boundaries exist, from smallest to largest:
+
+.. list-table::
+ :widths: 22 33 45
+ :header-rows: 1
+
+ * - Boundary
+ - What moves inside it
+ - What that protects you from
+ * - **A tool call**
+ - What ``run_command`` and the file tools do. **This toolset.**
+ - Model-written code damaging the worker host, reading its files, or
+ reaching the network from it.
+ * - **The glue between tools**
+ - Generated orchestration code, via :ref:`code mode <code-mode>` and the
+ Monty interpreter.
+ - Generated code touching anything other than the tools you registered.
+ Monty runs it in a worker subprocess on the worker host, a language-level
+ sandbox rather than an OS one, and every tool it calls executes in the
+ task process with the task's authority.
+ * - **The agent process**
+ - The whole agent loop, its LLM credentials and its message history.
+ - The agent's own credentials leaking, and any *other* toolset on the same
+ agent. Not available today.
+ * - **The whole task**
+ - The complete Airflow task, supervisor included, as
+ ``KubernetesExecutor`` does.
+ - Everything outside the task. Not the task from its own code: the
+ agent and what it runs are both inside this boundary.
+
+``SandboxToolset`` is the first row. It relocates exactly its four tools and
+nothing else. Boundary size is not a security level on its own: the image, the
+credentials you inject, the network policy and the resource limits decide the
+actual isolation. Choose the smallest boundary that fits, then configure it.
+
+.. list-table::
+ :widths: 48 52
+ :header-rows: 1
+
+ * - The agent needs to
+ - Reach for
+ * - Call operations you can name in advance
+ - :class:`~airflow.providers.common.ai.toolsets.sql.SQLToolset` or
+ :class:`~airflow.providers.common.ai.toolsets.hook.HookToolset`. Narrow,
+ bounded, and the credential stays in the worker rather than reaching the
+ model. If you can name the operations, name them.
+ * - Chain those tools with glue logic
+ - :ref:`code mode <code-mode>`. Monty confines generated code to the tools
+ you registered and starts in well under a millisecond, against roughly a
+ second to provision a hosted sandbox. Reach past it for a sandbox when the
+ model needs something Monty does not have: a shell, ``pip install``, a
+ compiled library.
+ * - Write and run open-ended code: reshape data with no known schema, install
+ a package, or fix its own failing script by reading the traceback
+ - ``SandboxToolset``
+ * - Produce a large artifact for a downstream task
+ - Today, a ``@task`` driving a backend directly, not the agent. That is a
+ missing seam in the toolset rather than a recommendation; see
+ :ref:`Getting a result out <sandbox-results>`.
+ * - A whole task's worth of untrusted work isolated, with no agent involved
+ - ``KubernetesPodOperator``
+ * - Airflow's own credentials kept away from the agent
+ - Not this. Scope the connections the task can see, and give the agent only
+ the toolsets it needs. See :ref:`sandbox-security`.
+
+**Code mode and the sandbox compose.** With both enabled, the three file tools
+fold into ``run_code`` as callables the generated code can loop over, and
+``run_command`` stays a tool the model calls directly, so a shell command is
+written as a shell command rather than quoted inside generated Python. Both paths
+reach the same sandbox: there is one per agent run whichever way a call arrives.
+
+Why not ``KubernetesPodOperator`` or the ``KubernetesExecutor``?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Because they answer a different question. Both isolate a task from the rest of
+your platform. Neither puts a boundary between an agent and the code that agent
+writes.
+
+Run an ``AgentOperator`` under the ``KubernetesExecutor`` and the whole task,
+agent loop included, gets its own pod. Model-written code still executes inside
+that pod, with its service account, its mounted secrets, its connections and its
+network position. The pod protects the cluster from the task; it does not protect
+the task from what the model decided to run, because from the pod's point of view
+that code is the task.
+
+``KubernetesPodOperator`` does contain the work, and its image, command, arguments
+and environment are all templated, so it can launch a different payload per run,
+and that payload can be a complete agent. The difference is what each one is. The
+pod is the environment that hosts the workload, and everything inside it runs with
+the pod's authority. ``SandboxToolset`` is a separate workspace that an
+already-running workload reaches into repeatedly, with less authority than the
+workload itself. Expressing the workspace as a pod means one pod per turn, with
+scheduling latency and a lost filesystem between each, or letting the model run
+whatever it wants inside the pod that also holds your credentials.
+
+.. list-table::
+ :widths: 30 35 35
+ :header-rows: 1
+
+ * -
+ - ``KubernetesExecutor`` / ``KubernetesPodOperator``
+ - ``SandboxToolset``
+ * - Protects
+ - The cluster, from the task
+ - The task and its credentials, from the model's output
+ * - Boundary sits
+ - Around the whole task, agent included
+ - Between the agent and the code it writes
+ * - What runs inside is decided
+ - Before the process starts
+ - By the model, mid-run
+ * - Credentials inside
+ - The task's own: service account, secrets, connections
+ - Only what ``SandboxSpec.env`` names, which is nothing by default
+
+Running an agent in a pod *and* giving it a sandbox is the setup a Kubernetes
+deployment usually wants: the pod bounds the task, the sandbox bounds the model.
+The ``sbx`` backend cannot be the second half of that, because it needs KVM on the
+worker host and an unprivileged pod cannot provide it; that is what the hosted
+backend is for.
+
+.. _sandbox-configuring:
+
+Configuring a sandbox
+---------------------
+
+:class:`~airflow.providers.common.ai.sandbox.SandboxSpec` says what a sandbox is
+provisioned with. The default denies outbound network access and injects no
+environment, and a backend that cannot enforce a field it was given **raises
+instead of ignoring it**, so a spec never gives you a false sense of a restriction
+being in force.
+
+.. code-block:: python
+
+ from airflow.providers.common.ai.sandbox import SandboxSpec, SbxSandboxBackend
+ from airflow.providers.common.ai.toolsets import SandboxToolset
+
+ SandboxToolset(
+ SbxSandboxBackend(host_network_policy="allow-all"),
+ spec=SandboxSpec(
+ env={"HF_TOKEN": "..."}, # only what the generated code legitimately needs
+ block_network=False, # this sandbox may reach the internet
+ ),
+ )
+
+**Network.** Three modes exist. ``block_network=True``, the default, drops all
+outbound traffic including name resolution. On Modal it maps onto the sandbox's
+own ``block_network`` flag. ``sbx`` has no per-sandbox enforcement of it at all:
+egress there is a host-level ``sbx policy``, so the backend honors the default by
+refusing to provision unless the Deployment Manager has declared
+``host_network_policy="deny-all"``, which means a bare ``SandboxSpec()`` is refused
+under the default ``host_network_policy="unknown"`` rather than silently getting
+an open sandbox. ``allow_egress_to`` names hosts the sandbox may reach, and both
+backends refuse it until you have said something that makes it enforceable:
+``sbx`` applies it as a per-sandbox rule on top of that ``deny-all`` host policy,
+since a local rule can narrow egress and never widen it, and Modal matches
+hostnames in the TLS handshake, which is weaker than it sounds and has to be
+opted into with ``egress_enforcement="sni"`` (see
+:ref:`the Modal backend <sandbox-backend-modal>` for exactly what it does and does
+not stop). ``block_network=False`` opens outbound access; on Modal the cloud
+metadata endpoint and private address ranges stay unreachable even then.
+
+**Packages.** A default sandbox has the Python standard library and no network,
+so an agent that reaches for ``pip install`` gets a DNS failure in a few seconds.
+Bake what it needs into the image, which needs no egress at all:
+
+.. code-block:: python
+
+ import modal
+
+ ModalSandboxBackend(
+ image=modal.Image.from_registry("python:3.12-slim").pip_install("pandas", "pyarrow"),
+ )
+
+Or, on Modal, allow exactly the two hosts a Python install needs and accept the
+handshake-name caveats: with ``pypi.org`` and ``files.pythonhosted.org`` allowed,
+installing pandas took about ten seconds when measured and ``github.com`` stayed
+blocked. Tell the model in its instructions what the image already has; the tool
+description says whether the sandbox has network access and where its working
+directory is, but not which packages are installed.
+
+**Resources.** A default Modal sandbox is a fraction of a core (``nproc`` reports
+1) with the standard library and no ``curl``, ``git`` or ``wget``. Set ``cpu`` for
+anything heavier. ``memory`` on Modal is a scheduling request, not a ceiling: a
+sandbox created with ``memory=512`` allocated 1.5 GiB without complaint when
+measured, so do not reach for it to bound what model-written code can consume.
+``sbx`` gives a sandbox every host CPU by default and enforces its ``memory`` as a
+limit.
+
+**Timeouts.** Three clocks apply. A command has a budget: ``default_command_timeout``
+(60 s) when the model does not ask, up to ``max_command_timeout`` (300 s) when it
+does. The sandbox has a lifetime: ``sandbox_timeout`` on Modal, 3600 s by default,
+and the clock runs through the model's thinking time between tool calls, so size it
+against the whole agent run. And Modal can reclaim an idle sandbox after
+``idle_timeout``, which defaults to ``None`` because a gap for model generation
+looks idle and would take the agent's files with it.
+
+**Two sandboxes on one agent** need ``tool_prefix``, since tool names must be
+unique within an agent:
+
+.. code-block:: python
+
+ toolsets = [
+ SandboxToolset(
+ SbxSandboxBackend(image="python:3.12-slim", host_network_policy="deny-all"),
+ tool_prefix="py",
+ ),
+ SandboxToolset(
+ SbxSandboxBackend(image="node:22-slim", host_network_policy="deny-all"),
+ tool_prefix="node",
+ ),
+ ]
+
+That yields ``py_run_command``, ``node_run_command`` and so on. Without a prefix
+on at least one of them the run fails at startup. Give the model instructions on
+which one to use for what, or it will guess.
+
+.. _sandbox-credentials:
+
+Credentials
+-----------
+
+``SandboxSpec.env`` is the only way in. Airflow never populates it: no connection,
+variable or worker environment variable reaches a sandbox unless you name it
+there, and the credential that *provisions* the sandbox never enters it either.
+Modal's token stays on the worker and is used by the client, so code running
+inside cannot call Modal as you or create further sandboxes.
+
+Before you put a real secret in ``env``, four things are true of it.
+
+**The model can read it.** Model-generated code can print the environment, and a
+tool result travels into the model's context, the task log and whatever the agent
+returns to XCom. Treat a credential given to a sandbox as disclosed to the model
+and to everything that records the run, and scope it to that one job. If the model
+only needs to *use* a system rather than hold its credential, a
+:class:`~airflow.providers.common.ai.toolsets.sql.SQLToolset` or
+:class:`~airflow.providers.common.ai.toolsets.hook.HookToolset` runs the hook in
+the task and returns only the result, which is the better shape whenever the
+operations can be named.
+
+**The value is fixed when the Dag file is parsed.** ``toolsets`` is not a
+templated field, so a spec cannot carry a Jinja expression, and a connection
+lookup written beside it would run in the Dag processor on every parse. An agent's
+sandbox therefore cannot take a credential from an Airflow connection or a secrets
+backend today. If it has to, the job cannot be an agent's sandbox: drive a backend
+from a ``@task``, where you are in ordinary Python at run time and the lookup runs
+where it should.
+
+.. code-block:: python
+
+ @task
+ def run_with_credential():
+ from airflow.providers.common.ai.sandbox import ModalSandboxBackend, SandboxSpec
+ from airflow.providers.common.compat.sdk import BaseHook
+
+ token = BaseHook.get_connection("my_api").password # resolved in the task, not at parse
+ backend = ModalSandboxBackend()
+ sandbox = backend.create(spec=SandboxSpec(env={"API_TOKEN": token}, block_network=False))
+ try:
+ ... # write_file, run_command, read_file
+ finally:
+ backend.destroy(sandbox)
+
+**On** ``sbx`` **the value is written to the guest filesystem.** The CLI has no
+create-time environment flag, so the backend appends ``export`` lines to
+``/etc/profile`` inside the microVM, readable by anything in that sandbox for its
+whole life. Modal passes the environment at creation and writes no file. Prefer the
+hosted backend when the sandbox needs a secret at all.
+
+**Non-string values are refused** at provisioning, with the offending keys named,
+rather than failing somewhere less obvious.
+
+.. _sandbox-results:
+
+Getting a result out
+--------------------
+
+An agent's result is whatever the model returns: findings, a mapping, a table
+small enough to read. That reaches XCom through ``output_type`` like any other
+agent output, and the examples above end that way.
+
+What does not come out is a *file* the agent built. The sandbox is destroyed when
+the run ends; ``read_file`` is text-only, replacing any byte it cannot decode, so a
+parquet or an image cannot survive the round trip at any size; and everything the
+model reads is capped, 50 KiB per stream for ``run_command`` and ``max_read_bytes``
+(5 MiB) per ``read_file``. Those caps are a budget rather than a transport limit:
+a 200 MB file reads out of a live sandbox in under ten seconds on the same path,
+but raising the cap costs roughly three times the file size in worker memory to
+show the model 50 KiB of it. The toolset does not yet have a seam for an author to
+collect what the agent produced; that is a known gap rather than the intended
+design.
+
+When the deliverable is a file and the Dag already knows the job, do not use an
+agent for it. Drive a backend from a ``@task``: the input goes in through
+``write_file``, the output comes back through ``read_file`` with a budget the
+worker can afford rather than a model's, and it lands in object storage with only
+the location passed downstream. The sandbox needs no network and no credentials
+for this, and the task owns its teardown:
+
+.. exampleinclude:: /../../ai/src/airflow/providers/common/ai/example_dags/example_sandbox_toolset.py
+ :language: python
+ :start-after: [START howto_sandbox_task_artifact]
+ :end-before: [END howto_sandbox_task_artifact]
+
+If the agent genuinely has to be the one producing the file, the shape that works
+is to give the sandboxed code egress to your bucket and a scoped credential in the
+spec, and have it write the object itself, passing the key back through its
+output. That widens the default deliberately and is worth a sentence in the Dag
+saying why.
+
+.. _sandbox-lifecycle:
+
+Lifecycle, failures and retries
+-------------------------------
+
+The sandbox is created lazily on the first tool call, shared by every call in that
+agent run, and destroyed when the run ends. A run that never calls a tool never
+provisions one, and concurrent runs never share a sandbox. Files written by one
+call are visible to later calls in the same run; each ``run_command`` is a fresh
+shell, so shell variables and background jobs do not survive between calls.
+
+**A failed command is output, not a failure.** A non-zero exit comes back to the
+model with both streams and ``[exit code: N]`` appended, and the model corrects
+itself. A recoverable sandbox error, such as a missing path or a file over the
+read budget, becomes a bounded retry prompt. Only a terminal failure, such as
+rejected credentials or a sandbox that has gone, fails the task, so Airflow's own
+retry handles it rather than the model burning its retry budget.
+
+**A command timeout** reports ``[timed out after Ns]`` with the budget that was
+actually applied. On Modal the sandbox survives and files written earlier are
+still there for the model to inspect. On ``sbx`` the backend has to destroy the
+sandbox to be certain the command stopped, and the result then says
+``[sandbox was replaced; files from earlier calls are gone]`` so the model knows
+to start over. Any other failure that loses the sandbox is reported the same way.
+
+**A run that outlives its sandbox fails the task.** When ``sandbox_timeout`` or
+``idle_timeout`` passes mid-run, the next tool call reaches a sandbox that is gone,
+which is terminal. This is the normal end of any run longer than its sandbox's
+lifetime. Human output review is not part of the run: it starts after the agent
+has finished and the sandbox has been destroyed, so a review pause costs no
+sandbox time and keeps no files.
+
+**Nothing survives the run.** A task retry starts from an empty sandbox, and so
+does every other attempt. Two operator features assume otherwise and must not be
+combined with a sandbox today, because neither is rejected:
+
+- ``durable=True`` caches each tool result and replays it on a retry without
+ calling the backend, so a replayed ``write_file`` reports success while no
+ sandbox exists, and the first call that misses the cache runs against a fresh
+ empty one. The model is handed a filesystem that does not match what it was just
+ told, and nothing raises.
+- ``enable_hitl_review=True`` regenerates after reviewer feedback by starting a
+ second agent run, and the first run's sandbox was destroyed when that run ended.
+ The regenerated agent gets an empty sandbox while its own history describes
+ files it wrote earlier.
+
+Cost and operations
+-------------------
+
+**One sandbox spans a whole agent run**, so you pay for the model's thinking time
+between tool calls, not only for the seconds your commands run. A default Modal
+sandbox is around two cents an hour, so this rarely matters; a data-sized one
+(``cpu=8, memory=32768``) is around two dollars an hour, and Modal bills the larger
+of what you requested and what you used, which is the second reason ``memory`` is
+worth setting deliberately. With ``idle_timeout`` unset, ``sandbox_timeout`` is
+the worst-case bill for one run. Fan-out is not the thing to worry about: 25
+sandboxes created concurrently from one process all came up in under three
+seconds.
+
+**Teardown runs inside the task.** A graceful end and an ordinary exception both
+destroy the sandbox. A worker killed outright, an OOM or a lost node never run that
+code, and what happens next is the backend's business alone: Modal reclaims the
+sandbox at ``sandbox_timeout`` whatever became of the worker; ``sbx`` has no
+server-side lifetime, so the microVM and its workspace directory survive with no
+owner. A teardown that fails is logged with the sandbox identity and does not fail
+the task.
+
+**Finding what a Dag left behind.** Modal sandboxes are named ``airflow-sandbox-*``
+and carry an ``airflow_sandbox`` tag plus whatever ``tags`` you pass, and
+``modal.Sandbox.list(app_id=..., tags={"dag_id": "my_dag"})`` returns exactly those
+still running. Tags are fixed when the backend is constructed, so every sandbox in
+a mapped task carries the same ones. When checking whether anything is actually
+still running, trust ``Sandbox.list``, which returns only live sandboxes, or
+``poll()`` on one you hold: ``None`` means running, and any integer means gone,
+including ``137``, which is a stopped sandbox rather than a sick one. Modal's
+dashboard lists stopped sandboxes alongside live ones, and the ``Tasks`` count in
+``modal app list`` lags by up to about a minute. ``sbx`` sandboxes are named the
+same way for an operator sweep.
+
+Backends
+--------
+
+.. _sandbox-backend-modal:
+
+Modal (hosted)
+^^^^^^^^^^^^^^
+
+:class:`~airflow.providers.common.ai.sandbox.modal.ModalSandboxBackend` runs each
+sandbox in Modal, provisioned over the API. Of the backends that ship with the
+provider, **this is the one to use in production**, and the only one that runs on
+Kubernetes: nothing has to be installed on the worker, model-written code never
+executes on the worker host, and Modal reclaims a sandbox at its own lifetime
+whether or not the worker survives. It needs the ``modal`` extra and ambient
+credentials, as under :ref:`Quick start <sandbox-quick-start>`.
+
+Constructor parameters:
+
+- ``image``: Registry tag for the sandbox image, or a prepared ``modal.Image``
+ carrying pre-installed packages. Default ``"python:3.12-slim"``.
+- ``app_name``: Modal app the sandboxes are created under. Default
+ ``"airflow-sandbox"``.
+- ``create_app_if_missing``: Create that app if it does not exist. Default ``True``.
+- ``sandbox_timeout``: Maximum lifetime of a sandbox in seconds. Default ``3600``.
+ Modal's own default is 300, which is below a plausible agent run.
+- ``idle_timeout``: Seconds of inactivity after which Modal reclaims the sandbox,
+ or ``None`` to rely on ``sandbox_timeout`` alone. Default ``None``; see
+ :ref:`Configuring a sandbox <sandbox-configuring>`.
+- ``workdir``: Working directory for commands, created if the image lacks it.
+ Default ``"/workspace"``. It is a starting directory, not a jail: absolute paths
+ and ``..`` are passed through, and commands run as root, so the model can read
+ and write anywhere in the sandbox filesystem. The sandbox boundary is what
+ contains that.
+- ``cpu``, ``memory``, ``gpu``, ``region``, ``cloud``: passed through to Modal.
+ ``None`` lets Modal choose. An unrecognized ``region`` or ``cloud`` fails the
+ task rather than falling back.
+- ``tags``: Extra Modal tags on every sandbox, e.g. ``{"dag_id": "my_dag"}``.
+- ``egress_enforcement``: ``"strict"`` (default) or ``"sni"``. See below.
+
+**Network policy.** ``block_network=True`` maps exactly onto Modal's own
+``block_network``, which drops all outbound traffic including DNS. A spec that
+names ``allow_egress_to`` is **refused by default**, because Modal cannot combine
+an allowlist with ``block_network`` at all, and its hostname allowlist is enforced
+by matching the name in the TLS handshake, which means:
+
+- TLS on port 443 to a listed host connects; any other host is refused at once.
+- **Non-TLS traffic is dropped, not refused.** A plain HTTP connection to a listed
+ host stalls until the client gives up, around two minutes of TCP retries for one
+ address, so with the 60 s default command budget the model reads
+ ``[timed out after 60s]`` and concludes its command was slow, never that the
+ network stopped it.
+- **The destination address is not part of the decision.** A connection opened to
+ an unrelated address while presenting a listed name is routed to the listed host
+ and answered by it: connecting to ``8.8.8.8:443`` with ``pypi.org`` in the
+ handshake returns the certificate and content of ``pypi.org``. The allowlist is a
+ name-routed egress proxy, not a filter on where packets may go.
+- **DNS resolution stays open for every hostname**, listed or not, against
+ authoritative servers outside Modal. A freshly generated label under a domain the
+ operator controls resolves and returns its answer, so this is a two-way channel.
+- A host that shares a **TLS endpoint** with a listed one can be reached by
+ presenting the listed name in the handshake and the other in the request. With
+ ``pypi.org`` as the only allowed host, a TLS session opened to
+ ``files.pythonhosted.org`` while presenting ``pypi.org`` was allowed through and
+ answered. Other tenants of the same CDN returned ``421 Misdirected Request``; it is
+ co-tenancy of the same TLS endpoint that matters, which you cannot check from
+ outside and which can change without notice.
+
+So the allowlist says which name a TLS session may be routed to, and nothing else.
+Pass ``ModalSandboxBackend(egress_enforcement="sni")`` to say you accept that and
+have the allowlist applied:
+
+.. code-block:: python
+
+ SandboxToolset(
+ ModalSandboxBackend(egress_enforcement="sni"),
+ spec=SandboxSpec(block_network=True, allow_egress_to=["pypi.org", "files.pythonhosted.org"]),
+ )
+
+Entries must be bare hostnames or one leading ``*.`` label; a URL, a ``host:port``,
+an address or a single-label name is refused, because Modal applies the list without
+checking it and any of those would silently match nothing.
+
+**What the image needs.** ``write_file`` and ``list_directory`` use Modal's own
+filesystem API, served by a helper Modal injects into the sandbox, so they need
+nothing from the image. ``read_file`` deliberately does not: Modal's read API takes
+no length, so it cannot honor a read budget, and a single call on a file that
+streams without end (``/dev/zero``, a FIFO, a procfs entry) would pull it into
+worker memory unbounded. That tool runs the base class's shell implementation,
+which caps the read inside the guest, and the image needs ``stat``, ``head`` and
+``base64``. Any Debian or Ubuntu based image, including ``python:*-slim``, has
+them.
+
+**Symlinks.** Because ``write_file`` goes through the native API, writing to a path
+that is a symlink replaces the link with a regular file and leaves the original
+target untouched, where a shell redirect would follow the link.
+
+sbx (Docker Sandboxes, local)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:class:`~airflow.providers.common.ai.sandbox.SbxSandboxBackend` runs each sandbox
+in a Docker Sandboxes microVM by driving the ``sbx`` CLI. Each sandbox is a real
+microVM with its own kernel.
+
+.. warning::
+
+ **Use this backend for local development, not production.** Docker Sandboxes
+ is built for running coding agents against a checkout on your own machine, and
+ driving it from an Airflow worker is off-label use. A production worker would
+ need the ``sbx`` binary on the host, an authenticated Docker account
+ (``sbx login``), a one-time ``sbx policy init``, and on Linux, KVM or nested
+ virtualization, which a worker in an unprivileged container cannot provide.
+
+ **Orphans are not reclaimed.** There is no server-side lifetime. If the worker is
+ killed outright, the microVM and its workspace directory survive; sandboxes are
+ named ``airflow-sandbox-*`` so an operator can find and remove them.
+
+Installing the CLI is a Deployment Manager prerequisite (``brew install
+docker/tap/sbx`` or ``winget install Docker.sbx``); the backend needs no Python
+dependency. The template image must provide GNU coreutils ``timeout``, ``base64``,
+``stat``, ``head``, ``find``, ``mkdir`` and ``dirname``, which any Debian or Ubuntu
+based image has.
+
+Constructor parameters:
+
+- ``image``: Container image for the sandbox. Default ``"python:3.12-slim"``.
+- ``memory``: Memory limit in binary units. ``sbx`` enforces a 1 GiB minimum.
+ Default ``"2g"``.
+- ``cpus``: CPUs to allocate. ``None`` (default) uses the ``sbx`` default, which
+ is every host CPU.
+- ``sbx_path``: Path to the ``sbx`` binary. Default ``"sbx"``.
+- ``create_timeout``: Seconds allowed for provisioning; a first-run microVM boot
+ plus an image pull can be slow. Default ``600``.
+- ``host_network_policy``: What ``sbx policy`` is set to on this host.
+ ``"unknown"`` (default) makes ``create`` refuse any spec asking for a network
+ guarantee this backend cannot make, and since ``block_network`` defaults to
+ ``True`` that includes a bare ``SandboxSpec()``. Set ``"deny-all"`` after running
+ ``sbx policy init deny-all``, or ``"allow-all"`` to state that egress is open
+ and pass ``SandboxSpec(block_network=False)`` to match.
+
+What differs between the two
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Swapping the backend is one constructor argument, and tool names, spec and prompt
+do not change. Four behaviours do, so read them before assuming the same Dag
+behaves identically in both places:
+
+- **CPU.** ``sbx`` gives a sandbox every host CPU; Modal defaults to a fraction of
+ one, so set ``cpu``.
+- **Egress allowlists.** ``sbx`` enforces ``allow_egress_to`` at the host policy
+ layer; Modal matches TLS handshake names, which is weaker and has to be opted
+ into.
+- **Command timeouts.** A timeout destroys an ``sbx`` sandbox and its files; a
+ Modal sandbox survives with its files intact.
+- **Symlinks.** ``write_file`` through a symlink follows the link on ``sbx`` and
+ replaces it on Modal.
+
+Bringing your own backend
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Any vendor that can create a sandbox, run a command in it and destroy it can plug
+in. Subclass :class:`~airflow.providers.common.ai.sandbox.SandboxBackend` in your
+own package and pass an instance to ``SandboxToolset``.
+
+**Three methods are required**: ``create``, ``run_command`` and ``destroy``. The
+three file operations ship as defaults implemented over ``run_command``, because
+reading, writing and listing a file are all expressible as shell commands.
+Override them only when the vendor has a native file API:
+
+.. code-block:: python
+
+ from airflow.providers.common.ai.sandbox import (
+ SandboxBackend,
+ SandboxExecResult,
+ SandboxSpec,
+ )
+
+
+ class AcmeSandboxBackend(SandboxBackend):
+ name = "acme"
+
+ def create(self, *, spec: SandboxSpec | None = None) -> str:
+ return acme_sdk.create_sandbox().id
+
+ def run_command(self, sandbox, command, *, timeout, max_output_bytes):
+ r = acme_sdk.exec(sandbox, command, timeout=timeout)
+ return SandboxExecResult(exit_code=r.exit_code, stdout=r.stdout, stderr=r.stderr)
+
+ def destroy(self, sandbox) -> None:
+ acme_sdk.delete_sandbox(sandbox)
+
+ # Optional: inherited from SandboxBackend unless the vendor has
+ # something better than shelling out.
+ def read_file(self, sandbox, path, *, max_bytes) -> bytes:
+ return acme_sdk.download(sandbox, path, limit=max_bytes)
+
+Four rules for an implementation:
+
+- Constructors run at Dag-parse time, so resolve credentials lazily, on first use.
+- ``destroy`` must be idempotent; destroying an already-gone sandbox is not an error.
+- Raise ``SandboxTerminalError`` when retrying cannot help and ``SandboxError``
+ when it might. The first fails the task for Airflow to retry; the second
+ becomes a bounded prompt back to the model.
+- If you cannot enforce something the ``SandboxSpec`` asks for, **raise**. Never
+ provision a weaker sandbox than the Dag author asked for.
+
+.. _sandbox-limitations:
+
+Limitations
+-----------
+
+These apply to every backend. Each is explained in the section it belongs to; this
+is the list to read before designing a Dag around an agent with a sandbox.
+
+- **Nothing survives the run**, including across task retries.
+ :ref:`Lifecycle <sandbox-lifecycle>`.
+- **A file the agent built cannot leave** except through the model's context,
+ which is text-only and capped. :ref:`Getting a result out <sandbox-results>`.
+- **A credential cannot come from a connection or a secrets backend**; the spec is
+ fixed at parse time, and anything injected is readable by the model.
+ :ref:`Credentials <sandbox-credentials>`.
+- **Do not combine with** ``durable=True`` **or** ``enable_hitl_review=True``.
+ Neither is rejected today. :ref:`Lifecycle <sandbox-lifecycle>`.
+- **A run that outlives** ``sandbox_timeout`` **fails the task.**
+ :ref:`Lifecycle <sandbox-lifecycle>`.
+- **The hostname allowlist is a weak control** and refused unless opted into.
+ :ref:`Modal <sandbox-backend-modal>`.
+- **Commands run as root and** ``workdir`` **is not a jail.**
+ :ref:`Modal <sandbox-backend-modal>`.
+- **It does not contain the agent.** :ref:`sandbox-security`.
+
+Parameters
+----------
+
+- ``backend``: The backend that provisions and drives the sandbox.
+- ``spec``: What to provision it with. Defaults to no environment and no egress.
+- ``default_command_timeout``: Seconds for a ``run_command`` the model did not
+ put a timeout on. Default ``60``.
+- ``max_command_timeout``: Hard ceiling for any single command, including a
+ model-supplied ``timeout_seconds``. Default ``300``.
+- ``max_output_lines`` / ``max_output_bytes``: Caps per output stream and per
+ file read, whichever is hit first. Defaults ``2000`` and 50 KiB. Command
+ output keeps the **tail**, where errors and the exit status live; file reads
+ keep the head and report a continuation offset.
+- ``max_read_bytes``: Largest file ``read_file`` will transfer. Default 5 MiB;
+ larger files are refused with a hint to slice them in the shell.
+- ``tool_prefix``: Prefix for the four tool names. Needed when one agent has
+ more than one ``SandboxToolset``.
+
+.. _sandbox-security:
+
+Security boundary
+-----------------
+
+**The sandbox does not contain the agent.** The agent loop, the model calls and
+every other toolset on the same agent still run in the worker process with the
+worker's credentials. An agent that has ``SandboxToolset`` *and* a toolset that
+can reach connections has a contained code tool sitting beside a credential path
+that is not contained: a prompt injection arriving in a queried row can steer the
+model into calling the credentialed tool, and that call executes from the worker
+with the connection's full grants. Moving the sandbox to a hosted backend changes
+none of this.
+
+The controls for that risk are different controls. Give the agent only the
+toolsets it needs. Point every connection at a least-privilege role. Prefer a
+capability toolset, which lets the model *use* a connection without ever holding
+it, over a credential in ``SandboxSpec.env``, which the model can read. And treat
+a table allowlist as a guardrail that contains intent rather than a boundary that
+contains access.
+
+What the sandbox does contain, verified against a live worker environment carrying
+sentinel credentials: nothing Airflow-shaped reaches it. No connection, variable,
+Fernet key or cloud credential appeared in the guest's environment, in any process
+environment or on its filesystem, and the provisioning token is not usable from
+inside even when the vendor's API is on the egress allowlist. Sandboxes cannot reach each
+other, so two sandboxes on one agent are isolated from one another as well as from
+the worker.
diff --git a/providers/common/ai/docs/toolsets.rst b/providers/common/ai/docs/toolsets.rst
index 103cb032a93..2db6edc5b5b 100644
--- a/providers/common/ai/docs/toolsets.rst
+++ b/providers/common/ai/docs/toolsets.rst
@@ -39,7 +39,7 @@ package root:
connections.
- :class:`~airflow.providers.common.ai.toolsets.sandbox.SandboxToolset` — give
the agent a shell and a filesystem inside an isolated sandbox, off the
- Airflow worker. See :ref:`which boundary that is <sandbox-boundaries>`.
+ Airflow worker. See :doc:`sandbox`.
- :class:`~airflow.providers.common.ai.toolsets.managed_agent.BaseManagedAgentToolset`
— base class that provider packages subclass to expose a **vendor-managed
agent**, one whose reasoning loop runs on a cloud provider's infrastructure.
@@ -562,326 +562,31 @@ and removes any cloned directories when the ``with`` block exits.
------------------
:class:`~airflow.providers.common.ai.toolsets.sandbox.SandboxToolset` gives the
-agent a shell and a filesystem inside a disposable sandbox, provisioned by a
-:class:`~airflow.providers.common.ai.sandbox.SandboxBackend` and running off the
-Airflow worker process. It exposes four tools:
+agent a shell and a filesystem inside a disposable sandbox, provisioned off the
+Airflow worker by a :class:`~airflow.providers.common.ai.sandbox.SandboxBackend`
+and destroyed when the agent run ends. It exposes four tools, ``run_command``,
+``read_file``, ``write_file`` and ``list_directory``, and ships with a hosted
+backend on Modal for production and a local microVM backend for development.
-.. list-table::
- :widths: 25 75
- :header-rows: 1
-
- * - Tool
- - What it does
- * - ``run_command``
- - Runs a shell command. Pipes, redirection, ``&&`` and globs work. A
- non-zero exit is reported as output, not raised, so the model reads
- ``stderr`` and corrects itself.
- * - ``read_file``
- - Reads a text file, head-first, and reports the next ``offset`` so the
- model can page through a long file.
- * - ``write_file``
- - Writes text to a file, creating parent directories.
- * - ``list_directory``
- - Lists a directory. Directories are shown with a trailing ``/``.
-
-These are the same four names and shapes that pydantic-ai's own sandbox
-capabilities use, so a model that has seen one already knows this one, and a
-vendor that has written an adapter for one is close to having written this one.
-
-.. _sandbox-boundaries:
-
-Which boundary this is
-^^^^^^^^^^^^^^^^^^^^^^
-
-"Sandboxed" means different things at different layers, and picking the wrong
-layer is the most common way to end up with less protection than you think.
-Four boundaries exist, from smallest to largest:
-
-.. list-table::
- :widths: 22 33 45
- :header-rows: 1
-
- * - Boundary
- - What moves inside it
- - What that protects you from
- * - **A tool call**
- - What ``run_command`` and the file tools do. **This toolset.**
- - Model-written code damaging the worker host, reading its files, or
- reaching the network from it.
- * - **The glue between tools**
- - Generated orchestration code, via :ref:`code mode <code-mode>` and the
- Monty interpreter.
- - Generated code touching anything other than the tools you registered.
- It still runs in the worker process.
- * - **The agent process**
- - The whole agent loop, its LLM credentials and its message history.
- - The agent's own credentials leaking, and any *other* toolset on the same
- agent. Not available today.
- * - **The whole task**
- - The complete Airflow task, supervisor included, as
- ``KubernetesExecutor`` does.
- - Everything, including Airflow's own worker context and connections.
-
-``SandboxToolset`` is the first row. Being precise about what that means:
-
-- Airflow does not put its context, connections, variables, or worker
- environment into the sandbox. Only what you pass through
- :class:`~airflow.providers.common.ai.sandbox.SandboxSpec` goes in.
-- **It does not contain the agent.** The agent loop, the model calls, and every
- other toolset on the same agent still run in the worker process with the
- worker's credentials. An agent that has ``SandboxToolset`` *and* a toolset
- that can reach connections has a contained code tool sitting beside a
- credential path that is not contained. The sandbox does not change what those other
- tools can do.
-- Boundary size is not a security level on its own. The image, the credentials
- you inject, the network policy, and the resource limits decide the actual
- isolation. Choose the smallest boundary that fits, then configure the runtime.
-
-If you need the whole task isolated, that is
-``KubernetesPodOperator`` or ``KubernetesExecutor`` today, not this.
-
-When to reach for it
-^^^^^^^^^^^^^^^^^^^^
-
-.. list-table::
- :widths: 45 55
- :header-rows: 1
-
- * - You want
- - Use
- * - The agent to analyze data, install a package, run a script it wrote, or
- produce a file, without any of that touching the worker
- - ``SandboxToolset``
- * - Fewer model round-trips when chaining tools you already trust
- - :ref:`code mode <code-mode>`. It is not a sandbox for your data; it
- restricts generated glue code to the tools you registered
- * - The agent to query a database with guardrails
- - :class:`~airflow.providers.common.ai.toolsets.sql.SQLToolset`, which
- enforces table allowlists. A sandbox does not help here
- * - A whole task's worth of untrusted work isolated, with no agent involved
- - ``KubernetesPodOperator``
- * - Airflow's own credentials kept away from the agent
- - Not this. Scope the connections the task can see, and give the agent only
- the toolsets it needs
-
-Both ``code_mode=True`` and ``SandboxToolset`` can be enabled together.
-``run_command`` deliberately stays a normal tool in that setup rather than being
-folded into ``run_code``, so the model writes Monty code that calls the sandbox,
-never a shell script quoted inside a Python string. The three file tools *are*
-folded in, where they are more useful as callables.
-
-Lifecycle
-^^^^^^^^^
-
-The sandbox is created lazily on the first tool call, shared by every call in
-that agent run, and destroyed when the run ends. A run that never calls a tool
-never provisions one, and concurrent runs never share a sandbox.
-
-Files written by one call are visible to later calls in the same run. Each
-``run_command`` is a fresh shell, so shell variables and background jobs do not
-survive between calls; write state to a file if you need it later.
-
-If a command outlives its budget and the backend cannot confirm it stopped, the
-backend destroys the sandbox and the tool result says so. The next call gets a
-fresh sandbox, and files from earlier calls are gone.
-
-A recoverable failure becomes a bounded retry the model can react to. Only a
-terminal failure -- credentials rejected, daemon unreachable, sandbox gone --
-fails the task, so Airflow's own retry handles it rather than the model burning
-its retry budget.
-
-Controlling what the sandbox gets
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-:class:`~airflow.providers.common.ai.sandbox.SandboxSpec` says what a sandbox is
-provisioned with. The default denies outbound network access and injects no
-environment.
-
-.. code-block:: python
-
- from airflow.providers.common.ai.sandbox import SandboxSpec, SbxSandboxBackend
- from airflow.providers.common.ai.toolsets import SandboxToolset
-
- SandboxToolset(
- SbxSandboxBackend(host_network_policy="allow-all"),
- spec=SandboxSpec(
- env={"HF_TOKEN": "..."}, # only what the generated code legitimately needs
- block_network=False, # this sandbox may reach the internet
- ),
- )
-
-Anything in ``env`` is readable by model-generated code, so scope it to that one
-sandbox's job rather than passing the worker's environment through.
-
-A backend that cannot enforce a field it was given **raises instead of ignoring
-it**, so a spec never gives you a false sense of a restriction being in force.
-The ``sbx`` backend applies ``allow_egress_to`` as a per-sandbox policy rule,
-but only on top of a ``deny-all`` host policy, since a local rule can narrow
-egress and never widen it; ``block_network`` has no per-sandbox enforcement at
-all. Ask for either against a host policy that is not already ``deny-all`` and
-it refuses rather than silently leaving the sandbox less restricted than the
-spec asked for.
-
-Using more than one sandbox
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Tool names must be unique within an agent, so two ``SandboxToolset`` instances
-need ``tool_prefix``:
-
-.. code-block:: python
-
- toolsets = [
- SandboxToolset(
- SbxSandboxBackend(image="python:3.12-slim", host_network_policy="deny-all"),
- tool_prefix="py",
- ),
- SandboxToolset(
- SbxSandboxBackend(image="node:22-slim", host_network_policy="deny-all"),
- tool_prefix="node",
- ),
- ]
-
-That yields ``py_run_command``, ``node_run_command`` and so on. Without a
-prefix on at least one of them, the run fails at startup with a duplicate tool
-name. Give the model instructions on which one to use for what, or it will guess.
-
-sbx backend (Docker Sandboxes)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-:class:`~airflow.providers.common.ai.sandbox.SbxSandboxBackend` runs each sandbox
-in a `Docker Sandboxes <https://docs.docker.com/ai/sandboxes/>`__ microVM by
-driving the ``sbx`` CLI. Each sandbox is a real microVM with its own kernel, so
-agent code is isolated by a hardware boundary rather than a shared kernel.
-
-.. warning::
-
- **Use this backend for local development, not production.** Docker Sandboxes
- is built for running coding agents against a checkout on your own machine:
- ``sbx create`` takes an agent name (``claude``, ``codex``, ``shell`` and so
- on) and bind-mounts host paths into the sandbox, and its stock network
- profile is described as "typical development traffic ... AI services and
- package registries". Driving it from an Airflow worker is off-label use.
-
- Concretely, a production worker would need all of: the ``sbx`` binary on the
- host, an authenticated Docker account (``sbx login``), a one-time
- ``sbx policy init``, and on Linux, KVM or nested virtualization. A worker in
- an unprivileged container -- the normal Kubernetes deployment -- generally
- cannot satisfy the last one at all.
-
- Treat it as the backend you develop and test a sandboxed agent against, then
- run something else in production. A hosted backend plugs in through
- :class:`~airflow.providers.common.ai.sandbox.SandboxBackend`, but none ships
- with the provider yet.
-
- **Orphans are not reclaimed automatically.** There is no server-side TTL. If
- the worker is killed outright, the microVM and its workspace directory
- survive. Sandboxes are named ``airflow-sandbox-*`` so an operator can find and
- remove them; budget for that sweep.
-
-Installing the CLI is a Deployment Manager prerequisite
-(``brew install docker/tap/sbx`` or ``winget install Docker.sbx``); the backend
-needs no Python dependency. The template image must provide GNU coreutils
-``timeout``, ``base64``, ``stat`` and ``ls``, which any Debian or Ubuntu based
-image, including ``python:*-slim``, does.
+It has its own page: :doc:`sandbox`, covering when to reach for it against code
+mode, ``KubernetesPodOperator`` and the ``KubernetesExecutor``, how to configure
+egress, packages and credentials, how a result gets out, what it costs, and the
+limitations to read before designing a Dag around it.
.. code-block:: python
from airflow.providers.common.ai.operators.agent import AgentOperator
- from airflow.providers.common.ai.sandbox import SbxSandboxBackend
+ from airflow.providers.common.ai.sandbox import ModalSandboxBackend
from airflow.providers.common.ai.toolsets import SandboxToolset
AgentOperator(
task_id="sandboxed_analyst",
prompt="Estimate pi with a Monte Carlo simulation of one million points.",
llm_conn_id="pydanticai_default",
- toolsets=[SandboxToolset(SbxSandboxBackend(host_network_policy="deny-all"))],
- )
-
-Constructor parameters:
-
-- ``image``: Container image for the sandbox. Default ``"python:3.12-slim"``.
-- ``memory``: Memory limit in binary units. ``sbx`` enforces a 1 GiB minimum.
- Default ``"2g"``.
-- ``cpus``: CPUs to allocate. ``None`` (default) uses the ``sbx`` default, which
- is every host CPU.
-- ``sbx_path``: Path to the ``sbx`` binary. Default ``"sbx"``.
-- ``create_timeout``: Seconds allowed for provisioning; a first-run microVM boot
- plus an image pull can be slow. Default ``600``.
-- ``host_network_policy``: What ``sbx policy`` is set to on this host.
- ``"unknown"`` (default) makes ``create`` refuse any spec asking for a network
- guarantee this backend cannot make. Set ``"deny-all"`` after running
- ``sbx policy init deny-all``, or ``"allow-all"`` to state that egress is open.
-
-Bringing your own backend
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Any vendor that can create a sandbox, run a command in it and destroy it can
-plug in. Subclass
-:class:`~airflow.providers.common.ai.sandbox.SandboxBackend` in your own package
-and pass an instance to ``SandboxToolset``.
-
-**Three methods are required**: ``create``, ``run_command`` and ``destroy``. The
-three file operations ship as defaults implemented over ``run_command``, because
-reading, writing and listing a file are all expressible as shell commands.
-Override them only when the vendor has a native file API, which avoids base64
-expansion, the command-line length ceiling, and the guest needing coreutils:
-
-.. code-block:: python
-
- from airflow.providers.common.ai.sandbox import (
- SandboxBackend,
- SandboxExecResult,
- SandboxSpec,
+ toolsets=[SandboxToolset(ModalSandboxBackend())],
)
- class AcmeSandboxBackend(SandboxBackend):
- name = "acme"
-
- def create(self, *, spec: SandboxSpec | None = None) -> str:
- return acme_sdk.create_sandbox().id
-
- def run_command(self, sandbox, command, *, timeout, max_output_bytes):
- r = acme_sdk.exec(sandbox, command, timeout=timeout)
- return SandboxExecResult(exit_code=r.exit_code, stdout=r.stdout, stderr=r.stderr)
-
- def destroy(self, sandbox) -> None:
- acme_sdk.delete_sandbox(sandbox)
-
- # Optional: inherited from SandboxBackend unless the vendor has
- # something better than shelling out.
- def read_file(self, sandbox, path, *, max_bytes) -> bytes:
- return acme_sdk.download(sandbox, path, limit=max_bytes)
-
-Four rules for an implementation:
-
-- Constructors run at Dag-parse time, so resolve credentials lazily, on first use.
-- ``destroy`` must be idempotent; destroying an already-gone sandbox is not an error.
-- Raise ``SandboxTerminalError`` when retrying cannot help and ``SandboxError``
- when it might. The first fails the task for Airflow to retry; the second
- becomes a bounded prompt back to the model.
-- If you cannot enforce something the ``SandboxSpec`` asks for, **raise**. Never
- provision a weaker sandbox than the Dag author asked for.
-
-Parameters
-^^^^^^^^^^
-
-- ``backend``: The backend that provisions and drives the sandbox.
-- ``spec``: What to provision it with. Defaults to no environment and no egress.
-- ``default_command_timeout``: Seconds for a ``run_command`` the model did not
- put a timeout on. Default ``60``.
-- ``max_command_timeout``: Hard ceiling for any single command, including a
- model-supplied ``timeout_seconds``. Default ``300``.
-- ``max_output_lines`` / ``max_output_bytes``: Caps per output stream and per
- file read, whichever is hit first. Defaults ``2000`` and 50 KiB. Command
- output keeps the **tail**, where errors and the exit status live; file reads
- keep the head and report a continuation offset.
-- ``max_read_bytes``: Largest file ``read_file`` will transfer. Default 5 MiB;
- larger files are refused with a hint to slice them in the shell.
-- ``tool_prefix``: Prefix for the four tool names. Needed when one agent has
- more than one ``SandboxToolset``.
-
-
Working with LangChain
----------------------
@@ -1047,9 +752,12 @@ No single layer is sufficient — they work together.
the same agent still run in the worker with its credentials, so this does
not stop an agent reaching connections through some other tool. It also
does not sanitize what the code computes or returns. Custom images can
- carry secrets, a backend you add can expose its own identity, the ``sbx``
- backend leaks orphaned microVMs
- if the worker is killed, and its CPU allocation defaults to every host CPU.
+ carry secrets and a backend you add can expose its own identity. The
+ ``sbx`` backend leaks orphaned microVMs if the worker is killed, and its
+ CPU allocation defaults to every host CPU; the Modal backend reclaims its
+ sandboxes on a server-side timeout, and if you opt into
+ ``egress_enforcement="sni"`` its hostname allowlist is enforced at the TLS
+ layer and leaves DNS resolution open.
* - **pydantic-ai: tool call budget**
- pydantic-ai's ``max_result_retries`` and ``model_settings`` control
how many tool-call rounds the agent can make before stopping.
@@ -1219,7 +927,7 @@ existing hook and no new connection types are needed.
A subclass implements two members:
``agent_ref``
- Normalised identity of the remote agent — ``platform`` and ``name`` — logged
+ Normalized identity of the remote agent — ``platform`` and ``name`` — logged
on every call so a run can be audited for which agents it consulted.
``invoke_sync(prompt)``
diff --git a/providers/common/ai/provider.yaml b/providers/common/ai/provider.yaml
index b44d3299903..22d43c43319 100644
--- a/providers/common/ai/provider.yaml
+++ b/providers/common/ai/provider.yaml
@@ -68,6 +68,9 @@ integrations:
- integration-name: Docker Sandboxes
external-doc-url: https://docs.docker.com/ai/sandboxes/
tags: [software]
+ - integration-name: Modal
+ external-doc-url: https://modal.com/docs/guide/sandbox
+ tags: [service]
hooks:
- integration-name: Pydantic AI
diff --git a/providers/common/ai/pyproject.toml b/providers/common/ai/pyproject.toml
index ef66ea5b0ee..ecf8f7186c6 100644
--- a/providers/common/ai/pyproject.toml
+++ b/providers/common/ai/pyproject.toml
@@ -93,6 +93,10 @@ dependencies = [
# openai<3, so a higher floor makes the workspace unsatisfiable. The guide says 2.45+.
"typesafe" = ["typesafe-sdk>=0.6.0"]
"mcp" = ["pydantic-ai-slim[mcp]>=2.23.0"]
+# Modal-hosted sandboxes for SandboxToolset, the backend to use where the worker host
+# cannot run microVMs -- Kubernetes, most notably. Needs no host-level installation,
+# only Modal credentials.
+"modal" = ["modal>=1.5.0"]
# Code mode: collapse tool calls into a single `run_code` tool that the model
# drives by writing Python, executed in the Monty sandbox (pydantic-monty).
# Enables AgentOperator(code_mode=True). Monty is pre-1.0; pinned here as an
diff --git a/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_sandbox_toolset.py b/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_sandbox_toolset.py
new file mode 100644
index 00000000000..972c976bd8a
--- /dev/null
+++ b/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_sandbox_toolset.py
@@ -0,0 +1,282 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Example Dags for ``SandboxToolset``.
+
+Three shapes, each a job a data team actually runs:
+
+1. An agent investigates a revenue anomaly. It queries the warehouse through a
+ ``SQLToolset`` (the credential stays in the task, the model only sees rows) and
+ does the arithmetic in a Modal sandbox with pandas baked into the image, so a
+ bad pivot or a runaway loop costs a 0.125-core sandbox rather than a worker slot.
+2. An agent normalizes a vendor CSV whose columns drift every month, on the local
+ ``sbx`` backend. Same toolset, same prompt, one constructor argument different,
+ which is how the same Dag moves from a laptop to production.
+3. A plain ``@task`` drives a backend directly to convert a file the Dag already
+ knows how to convert. No model is involved. This is the shape for producing an
+ artifact today, because a file inside an agent's sandbox can only leave through
+ the model's context.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from pydantic import BaseModel
+
+from airflow.providers.common.ai.operators.agent import AgentOperator
+from airflow.providers.common.ai.sandbox import SandboxSpec, SbxSandboxBackend
+from airflow.providers.common.ai.toolsets import SandboxToolset
+from airflow.providers.common.compat.sdk import ObjectStoragePath, dag, task
+
+try:
+ from airflow.providers.common.ai.toolsets.sql import SQLToolset
+except Exception:
+ SQLToolset = None # type: ignore[assignment,misc]
+
+try:
+ import modal
+
+ from airflow.providers.common.ai.sandbox import ModalSandboxBackend
+except Exception:
+ modal = None # type: ignore[assignment]
+ ModalSandboxBackend = None # type: ignore[assignment,misc]
+
+
+# Pydantic output classes live at module scope so downstream tasks can re-import
+# them when they deserialize the XCom payload.
+class Findings(BaseModel):
+ """What the investigation concluded, in a shape a downstream task can act on."""
+
+ summary: str
+ suspected_cause: str
+ affected_segments: list[str]
+ confidence: str
+
+
+class ColumnMapping(BaseModel):
+ """How the vendor's columns map onto the staging schema."""
+
+ mapped: dict[str, str]
+ unmapped: list[str]
+ notes: str
+
+
+# ---------------------------------------------------------------------------
+# 1. Investigate a revenue anomaly: warehouse through a capability toolset,
+# arithmetic in a hosted sandbox.
+# ---------------------------------------------------------------------------
+
+if SQLToolset is not None and modal is not None:
+ # [START howto_sandbox_agent_investigation]
+ # Packages come from the image, so the sandbox needs no network at all. The
+ # image is built by Modal the first time it is used and cached after that.
+ ANALYSIS_IMAGE = modal.Image.from_registry("python:3.12-slim").pip_install("pandas")
+
+ @dag(
+ schedule=None,
+ start_date=datetime(2024, 1, 1, tzinfo=timezone.utc),
+ catchup=False,
+ tags=["example", "sandbox"],
+ )
+ def example_sandbox_agent_investigation():
+ """Work out why daily revenue moved, with the warehouse and the workspace kept apart."""
+ AgentOperator(
+ task_id="investigate",
+ prompt=(
+ "Revenue for {{ ds }} moved more than 10% against the trailing seven-day "
+ "average. Find out which region and channel explain the move, and how sure you are."
+ ),
+ system_prompt=(
+ "You are a revenue analyst. Use the SQL tools to pull daily aggregates by region "
+ "and channel for the two weeks up to and including the date in question. Write "
+ "the rows into the sandbox as CSV and use pandas there for pivots and "
+ "comparisons rather than doing arithmetic in your head. The sandbox has no "
+ "network access and pandas is already installed; do not try to install anything. "
+ "If a script fails, read the traceback, fix it and run it again."
+ ),
+ llm_conn_id="pydanticai_default",
+ output_type=Findings,
+ toolsets=[
+ # The warehouse credential is held by the hook, in the task process. The
+ # model calls named operations and sees rows; it never sees the password
+ # and cannot pass it into the sandbox except by typing rows it was shown.
+ SQLToolset(
+ db_conn_id="warehouse",
+ allowed_tables=["analytics.daily_revenue", "analytics.orders"],
+ max_rows=500,
+ ),
+ # Everything the model writes and runs happens here: off the worker, no
+ # egress, nothing of Airflow's inside, destroyed when the run ends.
+ SandboxToolset(
+ ModalSandboxBackend(image=ANALYSIS_IMAGE, cpu=1),
+ # The default spec already denies all egress and injects no
+ # environment. Stated here so the intent is visible in the Dag.
+ spec=SandboxSpec(block_network=True),
+ ),
+ ],
+ )
+
+ # [END howto_sandbox_agent_investigation]
+
+ example_sandbox_agent_investigation()
+
+
+# ---------------------------------------------------------------------------
+# 2. Normalize a vendor file whose columns drift, on the local backend.
+# ---------------------------------------------------------------------------
+
+VENDOR_SAMPLE = """\
+Order Ref,Customer,Order Dt,Amt (USD),Ship Country
+A-1001,Acme Ltd,2026-03-01,1250.00,GB
+A-1002,Globex,03/02/2026,980.50,US
+A-1003,Initech,2 Mar 2026,"1,100.00",DE
+"""
+
+
+# [START howto_sandbox_agent_local]
+@dag(
+ schedule=None,
+ start_date=datetime(2024, 1, 1, tzinfo=timezone.utc),
+ catchup=False,
+ tags=["example", "sandbox"],
+)
+def example_sandbox_agent_local():
+ """Map a vendor's drifting columns onto the staging schema, developing against ``sbx``."""
+ AgentOperator(
+ task_id="map_columns",
+ prompt=(
+ "Here is a sample of this month's vendor export:\n\n"
+ f"{VENDOR_SAMPLE}\n"
+ "Our staging table has the columns order_id, customer_name, ordered_at (ISO "
+ "date), amount_usd (decimal) and country_code. Write a Python script that reads "
+ "the sample and produces a row for each order in the staging shape, run it, and "
+ "fix it until every row parses. Then report the column mapping you settled on and "
+ "anything you could not map."
+ ),
+ system_prompt=(
+ "You have a sandbox with Python 3.12 and the standard library, no network, and "
+ "an empty working directory. Write the sample to a file first, then iterate on a "
+ "script until it runs cleanly. Read tracebacks and fix the code rather than "
+ "guessing."
+ ),
+ llm_conn_id="pydanticai_default",
+ output_type=ColumnMapping,
+ toolsets=[
+ # ``sbx`` runs a microVM on this machine and needs the ``sbx`` CLI, a Docker
+ # login and KVM on Linux, which makes it a laptop and dev-host backend. Swap
+ # it for ``ModalSandboxBackend()`` and nothing else in this Dag changes.
+ SandboxToolset(SbxSandboxBackend(host_network_policy="deny-all")),
+ ],
+ )
+
+
+# [END howto_sandbox_agent_local]
+
+example_sandbox_agent_local()
+
+
+# ---------------------------------------------------------------------------
+# 3. Produce a file from a sandbox when the Dag already knows the job.
+# ---------------------------------------------------------------------------
+
+MIB = 1024 * 1024
+
+CONVERT_SCRIPT = """\
+import sys
+import pandas as pd
+
+frame = pd.read_csv(sys.argv[1])
+frame.columns = [c.strip().lower().replace(" ", "_") for c in frame.columns]
+frame = frame.drop_duplicates()
+frame.to_parquet(sys.argv[2], index=False)
+print(f"{len(frame)} rows, {len(frame.columns)} columns")
+"""
+
+
+@dag(
+ schedule=None,
+ start_date=datetime(2024, 1, 1, tzinfo=timezone.utc),
+ catchup=False,
+ tags=["example", "sandbox"],
+ params={
+ # Point these at object storage (``s3://``, ``gs://``) in a real deployment.
+ # ``file://`` keeps the example runnable on a laptop.
+ "input_uri": "file:///tmp/airflow-sandbox-example/orders.csv",
+ "output_uri": "file:///tmp/airflow-sandbox-example/orders.parquet",
+ },
+)
+def example_sandbox_task_artifact():
+ """
+ Convert a CSV to parquet inside a sandbox and land the result in object storage.
+
+ No agent is involved. The Dag knows exactly what to run, so a model would add
+ nothing, and a file produced inside an *agent's* sandbox could only come back
+ through the model's context, which is text-only and capped. Driving the
+ backend from a task has neither limit: the bytes move through the worker, and
+ the caller owns the sandbox's lifetime.
+ """
+
+ # [START howto_sandbox_task_artifact]
+ @task
+ def convert(input_uri: str, output_uri: str) -> str:
+ # Task-only dependencies stay out of the Dag-parsing process.
+ import modal
+
+ from airflow.providers.common.ai.sandbox import ModalSandboxBackend, SandboxSpec
+
+ source = ObjectStoragePath(input_uri).read_bytes()
+
+ backend = ModalSandboxBackend(
+ image=modal.Image.from_registry("python:3.12-slim").pip_install("pandas", "pyarrow"),
+ )
+ # No egress: the input arrives through write_file and the output leaves
+ # through read_file, so the sandbox needs no network and no credentials.
+ sandbox = backend.create(spec=SandboxSpec(block_network=True))
+ try:
+ backend.write_file(sandbox, "/workspace/orders.csv", source)
+ backend.write_file(sandbox, "/workspace/convert.py", CONVERT_SCRIPT.encode())
+ result = backend.run_command(
+ sandbox,
+ "python /workspace/convert.py /workspace/orders.csv /workspace/orders.parquet",
+ timeout=300,
+ max_output_bytes=64 * 1024,
+ )
+ if result.exit_code:
+ raise RuntimeError(f"Conversion failed ({result.exit_code}): {result.stderr}")
+ # The budget here is what the worker can hold, not a model's context.
+ # Above it read_file raises SandboxFileTooLargeError rather than truncating.
+ parquet = backend.read_file(sandbox, "/workspace/orders.parquet", max_bytes=256 * MIB)
+ finally:
+ # A toolset does this for you at the end of the run. Driving the backend
+ # yourself means owning teardown, including when the block above raises.
+ backend.destroy(sandbox)
+
+ target = ObjectStoragePath(output_uri)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_bytes(parquet)
+ # Hand the location downstream, not the bytes: XCom is the wrong place for
+ # a file of the size this pattern exists to handle.
+ return str(target)
+
+ # [END howto_sandbox_task_artifact]
+
+ convert("{{ params.input_uri }}", "{{ params.output_uri }}")
+
+
+example_sandbox_task_artifact()
diff --git a/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py b/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py
index a5cf51b4826..8d9bfa1a8dd 100644
--- a/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py
+++ b/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py
@@ -70,6 +70,11 @@ def get_provider_info():
"external-doc-url": "https://docs.docker.com/ai/sandboxes/",
"tags": ["software"],
},
+ {
+ "integration-name": "Modal",
+ "external-doc-url": "https://modal.com/docs/guide/sandbox",
+ "tags": ["service"],
+ },
],
"hooks": [
{
diff --git a/providers/common/ai/src/airflow/providers/common/ai/sandbox/__init__.py b/providers/common/ai/src/airflow/providers/common/ai/sandbox/__init__.py
index 97155421c47..3d4ae8dccd4 100644
--- a/providers/common/ai/src/airflow/providers/common/ai/sandbox/__init__.py
+++ b/providers/common/ai/src/airflow/providers/common/ai/sandbox/__init__.py
@@ -29,6 +29,7 @@ from airflow.providers.common.ai.sandbox.base import (
from airflow.providers.common.ai.sandbox.sbx import SbxSandboxBackend
__all__ = [
+ "ModalSandboxBackend",
"SandboxBackend",
"SandboxError",
"SandboxExecResult",
@@ -37,3 +38,16 @@ __all__ = [
"SandboxTerminalError",
"SbxSandboxBackend",
]
+
+
+def __getattr__(name: str):
+ # Imported on demand so the package stays importable without the 'modal' extra:
+ # everything above needs no third-party dependency at all.
+ if name == "ModalSandboxBackend":
+ # The module raises AirflowOptionalProviderFeatureException itself when the extra
+ # is absent, so it is not re-wrapped here: catching ImportError around this would
+ # also swallow a genuine one raised from inside the module.
+ from airflow.providers.common.ai.sandbox.modal import ModalSandboxBackend
+
+ return ModalSandboxBackend
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/providers/common/ai/src/airflow/providers/common/ai/sandbox/base.py b/providers/common/ai/src/airflow/providers/common/ai/sandbox/base.py
index c2e9af50884..34a58ff1778 100644
--- a/providers/common/ai/src/airflow/providers/common/ai/sandbox/base.py
+++ b/providers/common/ai/src/airflow/providers/common/ai/sandbox/base.py
@@ -114,6 +114,13 @@ class SandboxExecResult:
dropped bytes while reading that stream, before any model-facing formatting.
``sandbox_terminated`` means the backend destroyed the sandbox to stop the
command, so the toolset must provision a fresh one before the next call.
+
+ ``applied_timeout`` is the deadline the backend actually gave the command,
+ when that differs from the one it was asked for -- a backend may have to
+ shorten it, for instance to fit what is left of a sandbox's life. ``None``
+ means the requested deadline was used as given. The toolset reports this
+ rather than the request, so a model that times out is told the budget it
+ really had and can ask for something that fits.
"""
exit_code: int
@@ -123,6 +130,7 @@ class SandboxExecResult:
stdout_truncated: bool = False
stderr_truncated: bool = False
sandbox_terminated: bool = False
+ applied_timeout: float | None = None
class SandboxBackend(ABC):
@@ -192,9 +200,10 @@ class SandboxBackend(ABC):
# needing coreutils at all.
# ------------------------------------------------------------------
- # Reserved exit status for "the path is not readable", distinct from any
- # status the guest's own command might return.
+ # Reserved exit statuses for "the path is not readable" and "the path is a
+ # directory", distinct from any status the guest's own command might return.
_MISSING_PATH_STATUS = 66
+ _IS_DIRECTORY_STATUS = 67
def read_file(self, sandbox: str, path: str, *, max_bytes: int) -> bytes:
"""
@@ -209,9 +218,13 @@ class SandboxBackend(ABC):
# against anything ``stat`` reports as zero-length -- character devices,
# FIFOs, procfs -- which stream without end when read. ``stat`` failing
# is an error in its own right: without the explicit exit, a missing
- # path yields an empty ``base64`` and reads back as an empty file.
+ # path yields an empty ``base64`` and reads back as an empty file. A
+ # directory needs its own check for the same reason: ``stat`` succeeds
+ # on it, ``head`` fails but the pipeline's status is ``base64``'s, so
+ # without it a directory reads back as an empty file too.
script = (
f"sz=$(stat -Lc %s -- {quoted} 2>/dev/null) || exit {self._MISSING_PATH_STATUS}; "
+ f"[ -d {quoted} ] && exit {self._IS_DIRECTORY_STATUS}; "
f'printf "%s\n" "$sz"; '
f"head -c {max_bytes + 1} -- {quoted} | base64"
)
@@ -222,6 +235,8 @@ class SandboxBackend(ABC):
)
if result.exit_code == self._MISSING_PATH_STATUS:
raise SandboxError(f"{path!r} does not exist in the sandbox, or is not readable.")
+ if result.exit_code == self._IS_DIRECTORY_STATUS:
+ raise SandboxError(f"{path!r} is a directory. Use list_directory to see what is in it.")
if result.exit_code:
raise SandboxError(result.stderr.strip() or f"Could not read {path!r}.")
reported, _, encoded = result.stdout.partition("\n")
diff --git a/providers/common/ai/src/airflow/providers/common/ai/sandbox/modal.py b/providers/common/ai/src/airflow/providers/common/ai/sandbox/modal.py
new file mode 100644
index 00000000000..a88519894d0
--- /dev/null
+++ b/providers/common/ai/src/airflow/providers/common/ai/sandbox/modal.py
@@ -0,0 +1,868 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Modal hosted-sandbox backend for the SandboxToolset."""
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import math
+import posixpath
+import re
+import threading
+import time
+from contextlib import suppress
+from typing import TYPE_CHECKING, Literal
+
+try:
+ import modal
+except ModuleNotFoundError as e:
+ from airflow.providers.common.compat.sdk import AirflowOptionalProviderFeatureException
+
+ # Importing this module directly without the extra installed is how the provider
+ # verifier walks every submodule, and how a Dag author reaching past the package
+ # __getattr__ gets here. Both want the optional-feature signal, not ImportError.
+ raise AirflowOptionalProviderFeatureException(e)
+
+from airflow.providers.common.ai.sandbox.base import (
+ SandboxBackend,
+ SandboxError,
+ SandboxExecResult,
+ SandboxTerminalError,
+ _new_sandbox_name,
+ _validate_positive_finite,
+)
+
+if TYPE_CHECKING:
+ from collections.abc import Mapping, Sequence
+
+ from airflow.providers.common.ai.sandbox.base import SandboxSpec
+
+log = logging.getLogger(__name__)
+
+DEFAULT_IMAGE = "python:3.12-slim"
+DEFAULT_APP_NAME = "airflow-sandbox"
+# Modal creates this directory if the image does not carry it, so the agent gets a
+# writable working directory rather than "/".
+DEFAULT_WORKDIR = "/workspace"
+DEFAULT_SANDBOX_TIMEOUT = 3600
+# Off by default, as it is in Modal. One sandbox serves a whole agent run and nothing
+# keeps it warm between tool calls, so a gap for model generation or another toolset's
+# work would look idle and reclaim it mid-run. Measured, that
+# does not cost the agent its files: it costs the task. The next tool call reaches a
+# sandbox that is gone, which is terminal, so the run fails rather than continuing
+# against a fresh one. ``sandbox_timeout`` already bounds what a leak can cost.
+DEFAULT_IDLE_TIMEOUT = None
+
+# Extra wall-clock beyond a command's own deadline before we stop waiting on the
+# stream readers. Modal enforces the deadline itself, so this only has to absorb
+# the round trip that reports it.
+_DRAIN_GRACE = 30.0
+# Bounded budget for the small helper commands behind the inherited file operations.
+_FILE_OP_TIMEOUT = 120.0
+# Modal reports a command that hit its deadline as this exit status, in place of
+# raising: modal/container_process.py's poll/wait catch ExecTimeoutError and set it.
+# No exit status a guest command can return collides with it.
+_DEADLINE_RETURNCODE = -1
+# Modal maps a signal death to 128 + signal, and a sandbox dying under a command surfaces
+# as one of these too, which is why they are worth a liveness question.
+_SIGNAL_EXIT_BASE = 128
+# 128 + SIGKILL. Either the guest was killed inside a living sandbox, or the deadline was
+# enforced by a kill rather than reported as -1; see _hit_the_deadline.
+_SIGKILL_EXIT = 137
+# Budget for that question. It runs `true`, so anything slower than this means the answer
+# is not going to arrive usefully either way.
+_LIVENESS_TIMEOUT = 10
+
+# A plain hostname, optionally with a single leading "*." wildcard label, which is what
+# Modal matches an SNI value against. Modal itself sends the strings on without
+# validating them, so anything else -- a URL, a host:port, a bare "*" -- would be
+# accepted here and then silently match nothing (or, for "*", everything).
+_ALLOWED_HOSTNAME = re.compile(
+ r"^(?:\*\.)?(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.(?!-)[A-Za-z0-9-]{1,63}(?<!-))*$"
+)
+
+EgressEnforcement = Literal["strict", "sni"]
+
+
+def _is_tls_hostname(value: object) -> bool:
+ """Whether ``value`` is a name a TLS handshake could actually present."""
+ if not isinstance(value, str) or not _ALLOWED_HOSTNAME.match(value):
+ return False
+ with suppress(ValueError):
+ ipaddress.ip_address(value)
+ # An address is not a name. A client sends no SNI for one, so allowlisting it
+ # would allow nothing while reading like a restriction.
+ return False
+ # A final label of digits is not a hostname either (RFC 1123), which also catches a
+ # bare port number written on its own.
+ if value.rsplit(".", 1)[-1].isdigit():
+ return False
+ # A single label cannot be the name a TLS handshake presents to a public endpoint,
+ # and the sandbox has no internal resolver to give one meaning, so 'localhost' and
+ # friends would read as a restriction while allowing nothing. Wildcards are checked
+ # on what they qualify: '*.com' is as meaningless as 'com'.
+ return "." in value.removeprefix("*.")
+
+
+class ModalSandboxBackend(SandboxBackend):
+ """
+ Sandbox backend that runs agent commands in a `Modal <https://modal.com>`__ sandbox.
+
+ Each sandbox is a gVisor-isolated container in Modal's infrastructure, provisioned
+ over the API. Nothing has to be installed on the Airflow worker and model-written
+ code never executes on the worker host, which makes this the backend to reach for
+ on Kubernetes, where :class:`~airflow.providers.common.ai.sandbox.SbxSandboxBackend`
+ cannot run at all.
+
+ Modal reclaims a sandbox at ``sandbox_timeout`` whatever happens to the worker, so a
+ worker killed outright cannot leak one. Sandboxes are also named ``airflow-sandbox-*``
+ and carry whatever ``tags`` you set, so they can be found in Modal's dashboard. Those
+ tags are fixed when the backend is constructed, which is Dag-parse time, so they can
+ identify the Dag but not the run, task or map index that leaked one.
+
+ **Credentials are ambient.** Modal is authenticated the same way its CLI is: run
+ ``modal token new`` once to write ``~/.modal.toml``, or set ``MODAL_TOKEN_ID`` and
+ ``MODAL_TOKEN_SECRET`` in the worker environment. Nothing is read until the first
+ sandbox is created, so a Dag file that constructs this backend parses without
+ credentials present.
+
+ **What the network policy can and cannot promise.** ``SandboxSpec(block_network=True)``
+ maps to Modal's own ``block_network``, which drops all outbound traffic including DNS.
+ A :class:`~airflow.providers.common.ai.sandbox.SandboxSpec` that also names
+ ``allow_egress_to`` is **refused by default**, because Modal cannot combine an
+ allowlist with ``block_network`` and its hostname allowlist is enforced by matching
+ the TLS SNI: it permits TLS on port 443 to the listed hosts, blocks other hosts, and
+ leaves DNS resolution open for every hostname. Sandbox code can therefore still reach
+ a non-listed host that shares a TLS endpoint with a listed one, and can still carry
+ data out through DNS queries. Pass ``egress_enforcement="sni"`` to accept that and
+ have the allowlist applied.
+
+ **A timeout does not cost you the sandbox.** Modal stops the command server-side and
+ the sandbox stays usable, so files written by earlier calls survive and the model can
+ inspect them. That differs from ``sbx``, which has to destroy the sandbox to be sure
+ a command stopped.
+
+ The image must provide ``sh``, and ``stat``, ``head`` and ``base64`` for
+ :meth:`read_file`, which deliberately stays on the base class's shell implementation
+ (see that method). Any Debian or Ubuntu based image, including ``python:*-slim``, does.
+
+ :param image: Registry tag for the sandbox image, or a prepared ``modal.Image``.
+ Default ``"python:3.12-slim"``. An image carrying the packages an agent needs is
+ the alternative to opening egress so it can install them:
+ ``modal.Image.from_registry("python:3.12-slim").pip_install("pandas")``.
+ :param app_name: Modal app the sandboxes are created under. Default
+ ``"airflow-sandbox"``.
+ :param create_app_if_missing: Create the Modal app when it does not exist yet.
+ Default ``True``.
+ :param sandbox_timeout: Maximum lifetime in seconds of a sandbox before Modal shuts
+ it down. Bounds the whole sandbox, where a ``run_command`` timeout bounds one
+ command. Default ``3600``; Modal's own default of 300 is below a plausible agent
+ run.
+ :param idle_timeout: Seconds of inactivity after which Modal reclaims the sandbox.
+ Default ``None``, meaning ``sandbox_timeout`` is the only bound. Set it only if
+ you want tighter cost control and know your agent's pace: one sandbox serves a
+ whole run, nothing keeps it warm between tool calls, and a gap for model
+ generation would be reclaimed with every file in it.
+ :param workdir: Working directory for commands, created if the image lacks it.
+ Default ``"/workspace"``. ``None`` uses the image's own default, in which case
+ the backend has to ask the sandbox where that is before its first file
+ operation, so prefer stating it.
+ :param cpu: CPU cores to request. ``None`` uses Modal's default. Takes effect: a
+ sandbox created with ``cpu=4`` reports 4 from ``nproc``, though ``/proc/cpuinfo``
+ still lists the host's cores.
+ :param memory: Memory in MiB to request. ``None`` uses Modal's default. **A request,
+ not a ceiling**: a sandbox created with ``memory=512`` allocated 1.5 GiB without
+ complaint when measured, so this schedules the sandbox somewhere with room and does
+ not bound what model-written code can take. Nothing inside the sandbox can see it
+ either -- ``/proc/meminfo`` reports the host's memory.
+ :param gpu: GPU specification, e.g. ``"A10G"``. ``None`` requests none. Billed at a
+ different rate, so set it deliberately.
+ :param region: Region or regions to run in. ``None`` lets Modal choose. Validated by
+ Modal, so an unrecognized region fails the task rather than falling back.
+ :param cloud: Cloud provider to run on. ``None`` lets Modal choose. Validated the same
+ way.
+ :param tags: Extra Modal tags to set on every sandbox, e.g. ``{"dag_id": "my_dag"}``.
+ You can query them, which is the point: ``modal.Sandbox.list(app_id=..., tags={"dag_id":
+ "my_dag"})`` returns exactly the sandboxes carrying them, which is how an operator
+ finds what a Dag left behind. ``airflow_sandbox`` is set by this backend and will
+ overwrite a key of that name.
+ :param egress_enforcement: ``"strict"`` (default) refuses a ``SandboxSpec`` that
+ names ``allow_egress_to``, because Modal cannot enforce a hostname allowlist
+ below TLS. ``"sni"`` accepts it and applies Modal's SNI-matched allowlist, with
+ the limits described above.
+ """
+
+ name = "modal"
+
+ def __init__(
+ self,
+ *,
+ image: str | modal.Image = DEFAULT_IMAGE,
+ app_name: str = DEFAULT_APP_NAME,
+ create_app_if_missing: bool = True,
+ sandbox_timeout: int = DEFAULT_SANDBOX_TIMEOUT,
+ idle_timeout: int | None = DEFAULT_IDLE_TIMEOUT,
+ workdir: str | None = DEFAULT_WORKDIR,
+ cpu: float | None = None,
+ memory: int | None = None,
+ gpu: str | None = None,
+ region: str | Sequence[str] | None = None,
+ cloud: str | None = None,
+ tags: Mapping[str, str] | None = None,
+ egress_enforcement: EgressEnforcement = "strict",
+ ) -> None:
+ if isinstance(image, str) and not image:
+ raise ValueError("image must not be empty.")
+ if not app_name:
+ raise ValueError("app_name must not be empty.")
+ _validate_positive_finite(sandbox_timeout, "sandbox_timeout")
+ if sandbox_timeout < _FILE_OP_TIMEOUT:
+ # The inherited file operations run helper commands with a fixed budget, and
+ # a command may not outlive its sandbox. A shorter lifetime would make
+ # read_file fail every time with advice the model cannot act on, since that
+ # tool has no timeout argument to shorten.
+ raise ValueError(
+ f"sandbox_timeout must be at least {_FILE_OP_TIMEOUT:g} seconds, the budget the "
+ f"file operations run with, got {sandbox_timeout}."
+ )
+ if idle_timeout is not None:
+ _validate_positive_finite(idle_timeout, "idle_timeout")
+ if idle_timeout > sandbox_timeout:
+ # An idle timeout past the sandbox's own lifetime could never fire.
+ raise ValueError(
+ f"idle_timeout ({idle_timeout}) must not exceed sandbox_timeout ({sandbox_timeout})."
+ )
+ if workdir is not None and not workdir.startswith("/"):
+ raise ValueError(f"workdir must be an absolute path, got {workdir!r}.")
+ if cpu is not None:
+ _validate_positive_finite(cpu, "cpu")
+ if memory is not None:
+ _validate_positive_finite(memory, "memory")
+ if egress_enforcement not in ("strict", "sni"):
+ raise ValueError(f"egress_enforcement must be 'strict' or 'sni', got {egress_enforcement!r}.")
+ self._image = image
+ self._app_name = app_name
+ self._create_app_if_missing = create_app_if_missing
+ self._sandbox_timeout = int(sandbox_timeout)
+ self._idle_timeout = None if idle_timeout is None else int(idle_timeout)
+ self._workdir = workdir
+ self._cpu = cpu
+ self._memory = None if memory is None else int(memory)
+ self._gpu = gpu
+ self._region = region
+ self._cloud = cloud
+ self._tags = dict(tags) if tags else {}
+ self._egress_enforcement: EgressEnforcement = egress_enforcement
+ # Keyed by sandbox handle, so one backend instance shared by concurrent agent
+ # runs never has two of them touching the same entry.
+ self._sandboxes: dict[str, modal.Sandbox] = {}
+ self._workdirs: dict[str, str] = {}
+ self._expiries: dict[str, float] = {}
+
+ # ------------------------------------------------------------------
+ # Lifecycle.
+ # ------------------------------------------------------------------
+
+ def create(self, *, spec: SandboxSpec | None = None) -> str:
+ # Both refuse a spec this backend cannot carry faithfully, before anything is
+ # provisioned, so a rejected spec never costs a sandbox.
+ network = self._network_kwargs(spec)
+ environment = self._environment(spec)
+ name = _new_sandbox_name()
+ try:
+ # Looked up per create rather than memoized on the instance: the lookup is
+ # idempotent server-side, and one round trip per sandbox is cheaper than the
+ # locking a shared cache would need, given the toolset shares one backend
+ # across concurrent runs.
+ app = modal.App.lookup(self._app_name, create_if_missing=self._create_app_if_missing)
+ # A registry tag is the common case; a prepared Image is how packages get
+ # into the sandbox without opening egress to fetch them.
+ image = modal.Image.from_registry(self._image) if isinstance(self._image, str) else self._image
+ sandbox = modal.Sandbox.create(
+ app=app,
+ image=image,
+ name=name,
+ tags={**self._tags, "airflow_sandbox": name},
+ timeout=self._sandbox_timeout,
+ idle_timeout=self._idle_timeout,
+ workdir=self._workdir,
+ env=environment,
+ cpu=self._cpu,
+ memory=self._memory,
+ gpu=self._gpu,
+ region=self._region,
+ cloud=self._cloud,
+ **network,
+ )
+ except modal.exception.Error as e:
+ error = self._as_sandbox_error(e)
+ if isinstance(error, SandboxTerminalError):
+ raise error from e
+ # Everything that escapes create is terminal, whatever its shape. The toolset
+ # provisions the sandbox outside the block that turns a recoverable error into
+ # a ModelRetry, so a recoverable label here is one the model never sees, and
+ # could not act on if it did: no prompt fixes a bad image tag or an
+ # unreachable control plane. Airflow's own retry is the right handler.
+ raise SandboxTerminalError(f"Could not create a Modal sandbox: {error}") from e
+ # Nothing is applied after Sandbox.create returns: the environment, the network
+ # policy, the working directory and the tags are all create-time arguments. So
+ # there is no window in which a live sandbox exists half-configured, and no
+ # cleanup block to go with it. Anything added here later needs one.
+ handle = sandbox.object_id
+ self._sandboxes[handle] = sandbox
+ # When this sandbox stops existing, so a command that could not finish inside its
+ # remaining life is refused up front instead of dying half way through.
+ self._expiries[handle] = time.monotonic() + self._sandbox_timeout
+ return handle
+
+ def destroy(self, sandbox: str) -> None:
+ """
+ Ask Modal to terminate the sandbox, without waiting for it to finish stopping.
+
+ The request is not blocking on purpose. Measured against modal 1.5.5 in September
+ 2026, ``terminate()`` returned in under 0.2s while ``terminate(wait=True)`` took 31
+ seconds, and this runs in the teardown path of every agent run, where that would be
+ pure added latency. The sandbox stops shortly afterwards and shows an exit status
+ within about 35 seconds; ``sandbox_timeout`` is the backstop if the request never
+ lands at all.
+ """
+ handle = self._sandboxes.pop(sandbox, None)
+ self._workdirs.pop(sandbox, None)
+ self._expiries.pop(sandbox, None)
+ try:
+ target = handle if handle is not None else modal.Sandbox.from_id(sandbox)
+ target.terminate()
+ except modal.exception.NotFoundError:
+ # Already gone, which is what destroy is for. Idempotent by contract.
+ pass
+ except modal.exception.Error:
+ # Terminating a sandbox that is already finished, or a control-plane blip.
+ # Modal reclaims it at sandbox_timeout or idle_timeout either way, so log
+ # and let the toolset finish its run.
+ log.warning("Could not terminate Modal sandbox %s", sandbox, exc_info=True)
+
+ # ------------------------------------------------------------------
+ # Commands.
+ # ------------------------------------------------------------------
+
+ def run_command(
+ self, sandbox: str, command: str, *, timeout: float, max_output_bytes: int
+ ) -> SandboxExecResult:
+ _validate_positive_finite(timeout, "timeout")
+ _validate_positive_finite(max_output_bytes, "max_output_bytes")
+ # Modal takes whole-second deadlines and reads 0 as "no timeout", so round up.
+ seconds = max(1, math.ceil(timeout))
+ if seconds > self._sandbox_timeout:
+ # Against the whole lifetime, which is a fact the Dag author chose and can
+ # change. Recoverable, because the model can also just ask for less.
+ raise SandboxError(
+ f"A {seconds}s command does not fit this sandbox's {self._sandbox_timeout}s "
+ "lifetime. Ask for a shorter timeout, or raise sandbox_timeout on the backend."
+ )
+ # Against what is LEFT, the answer is to shorten the deadline rather than refuse.
+ # Refusing here would be unactionable: the file operations inherited from the base
+ # class run on a fixed internal budget that neither the model nor the Dag author
+ # can lower, so once a sandbox had less of that budget left than the budget itself
+ # -- which happens to every sandbox eventually, whatever its lifetime -- read_file
+ # would fail for the rest of its life with advice nobody could follow. A clamped
+ # deadline runs the work that fits, and a command that then uses all of it has
+ # stood on the sandbox's expiry, which ``clamped`` records below.
+ clamped = False
+ remaining = self._remaining_lifetime(sandbox)
+ if remaining is not None and seconds > remaining:
+ log.debug(
+ "Shortening a %ss command to the %.0fs sandbox %s has left",
+ seconds,
+ remaining,
+ sandbox,
+ )
+ seconds = max(1, int(remaining))
+ clamped = True
+ handle = self._handle(sandbox)
+ started = time.monotonic()
+ try:
+ process = handle.exec("sh", "-c", command, timeout=seconds, text=False)
+ except modal.exception.Error as e:
+ raise self._as_sandbox_error(e, sandbox=sandbox) from e
+
+ cap = int(max_output_bytes)
+ stdout, stderr = bytearray(), bytearray()
+ out_truncated, err_truncated = [False], [False]
+ # Drained in threads, so a command that fills one stream while we read the other
+ # cannot wedge the call, and so a stream that never ends cannot outlive the
+ # deadline: the joins below are bounded and the threads are daemons.
+ drains = [
+ threading.Thread(
+ target=self._drain, args=(process.stdout, stdout, out_truncated, cap), daemon=True
+ ),
+ threading.Thread(
+ target=self._drain, args=(process.stderr, stderr, err_truncated, cap), daemon=True
+ ),
+ ]
+ for thread in drains:
+ thread.start()
+
+ try:
+ returncode = process.wait()
+ except modal.exception.ExecTimeoutError:
+ # Not raised by modal 1.5.x, which reports the deadline as an exit status
+ # instead, carrying its own note that it should probably raise. Handled so
+ # this keeps working if it starts to.
+ returncode = _DEADLINE_RETURNCODE
+ except modal.exception.Error as e:
+ raise self._as_sandbox_error(e, sandbox=sandbox) from e
+ # Measured from here rather than from the start of the command: the streams have
+ # only the round trip left to deliver, and a command that finished in a second
+ # must not leave a background writer holding the fd for the rest of its budget.
+ deadline = time.monotonic() + _DRAIN_GRACE
+ for thread in drains:
+ thread.join(timeout=max(0.0, deadline - time.monotonic()))
+ if any(thread.is_alive() for thread in drains):
+ # Something still holds a stream open. Report what arrived and mark it short
+ # rather than blocking past the caller's budget.
+ out_truncated[0] = err_truncated[0] = True
+
+ elapsed = time.monotonic() - started
+ # A sandbox that died under the command reports the same way an ordinary failure
+ # does, so ask. Only for a deadline or a signal exit, which is where a death can
+ # hide: an ordinary non-zero exit costs no round trip.
+ terminated = False
+ if returncode == _DEADLINE_RETURNCODE or returncode >= _SIGNAL_EXIT_BASE:
+ if clamped and self._hit_the_deadline(returncode, elapsed=elapsed, budget=seconds):
+ # The deadline WAS the sandbox's remaining lifetime, so reaching it means
+ # the lifetime is spent. Do not ask: Modal reclaims a sandbox some seconds
+ # after its deadline, and measured at this boundary the probe answers
+ # "alive" for a sandbox that dies moments later. Believing it told the
+ # model only that its command was slow, and the next call then failed the
+ # task. The clamp is the better evidence because it cannot race.
+ terminated = True
+ else:
+ terminated = not self._still_alive(sandbox)
+ if terminated:
+ # The handle is dead, so drop it rather than hand it to a later call.
+ self._sandboxes.pop(sandbox, None)
+ self._workdirs.pop(sandbox, None)
+ self._expiries.pop(sandbox, None)
+
+ return SandboxExecResult(
+ exit_code=returncode,
+ stdout=stdout.decode(errors="replace"),
+ stderr=stderr.decode(errors="replace"),
+ timed_out=self._hit_the_deadline(returncode, elapsed=elapsed, budget=seconds),
+ stdout_truncated=out_truncated[0],
+ stderr_truncated=err_truncated[0],
+ sandbox_terminated=terminated,
+ # Whole seconds, and shortened when the sandbox had less life left than the
+ # command asked for, so the model hears the budget it actually had.
+ applied_timeout=seconds,
+ )
+
+ @staticmethod
+ def _hit_the_deadline(returncode: int, *, elapsed: float, budget: int) -> bool:
+ """
+ Whether the command was stopped by its deadline rather than by something inside it.
+
+ Modal usually reports a deadline as ``-1``, but not always: the same ``sleep 30``
+ at a two second budget has been observed returning ``137`` instead, at the same
+ elapsed time, from the same SDK version. Depending on the status alone therefore
+ loses about a quarter of deadlines in the environments where the race shows, and
+ the model reads a bare ``[exit code: 137]`` for a command that simply ran long.
+
+ Elapsed time separates the two cleanly, which is what ``sbx`` already does with the
+ same ambiguity. Measured: a deadline lands at or just past its budget (2.02-2.97s
+ against 2s), while a guest killing itself lands nowhere near it (0.04s against 30s),
+ including ``sleep 3; kill -9`` at 3.03s of a 30s budget. What is left over is a real
+ SIGKILL that happens to arrive after the budget, an OOM at the very end of a long
+ command, which no signal available here can tell apart.
+
+ Worth knowing before trusting the second branch: the ``137`` deadline has been seen
+ three times, all inside one ten-minute window on a host whose Docker daemon was
+ half-wedged at the time, and 40 consecutive deadlines since have all been ``-1``,
+ including 16 in that same container image and Python version. Host load is the
+ plausible trigger, which is to say the race appears exactly when a worker is busy
+ -- so the branch is worth its two lines even though it cannot be summoned on
+ demand. It is unit-covered and costs nothing when it never fires.
+ """
+ if returncode == _DEADLINE_RETURNCODE:
+ return True
+ return returncode == _SIGKILL_EXIT and elapsed >= budget
+
+ def _still_alive(self, sandbox: str) -> bool:
+ """
+ Whether the sandbox can still run a command, asked by running the cheapest one.
+
+ ``poll()`` is the obvious question and the wrong one: measured against modal
+ 1.5.5, a sandbox terminated mid-command still polled ``None`` while a command in
+ it was already impossible. A trivial ``exec`` answers truthfully within about two
+ tenths of a second, either returning 0 or raising -- ``NotFoundError`` for a
+ terminated sandbox, ``ConflictError`` for one shutting down at its own timeout.
+ """
+ handle = self._sandboxes.get(sandbox)
+ if handle is None:
+ return False
+ try:
+ handle.exec("sh", "-c", "true", timeout=_LIVENESS_TIMEOUT, text=True).wait()
+ except modal.exception.Error:
+ return False
+ except Exception:
+ # Nothing here is worth failing a command that already produced its output.
+ log.debug("Could not check whether Modal sandbox %s is alive", sandbox, exc_info=True)
+ return True
+ return True
+
+ @staticmethod
+ def _drain(stream, buf: bytearray, flag: list[bool], cap: int) -> None:
+ """Read a stream to its end, retaining at most ``cap`` bytes from the tail."""
+ # Keep the tail: the model reads this to fix its own command, and a traceback
+ # plus the exit status live at the end. The toolset's formatter promises a tail
+ # too, so retaining the head would hand it a window that no longer has one.
+ try:
+ for chunk in stream:
+ buf.extend(chunk)
+ if len(buf) > cap:
+ del buf[: len(buf) - cap]
+ flag[0] = True
+ # Drop the leading partial line so no fragment is presented as a
+ # whole record -- but only when dropping it is cheap. One line longer
+ # than the budget has its newline at the very end, so dropping through
+ # it would leave nothing at all, and the toolset labels only a
+ # non-empty stream: the model would read "(no output)" for a command
+ # that produced megabytes. A long line followed by a short one is the
+ # same trade in miniature, handing back a few bytes of a 50 KiB
+ # window. The formatter already marks output as cut, so a partial
+ # first line misleads nobody, while throwing away most of the window
+ # to avoid one does. Half the window is the line between the two.
+ newline = buf.find(b"\n")
+ if newline != -1 and len(buf) - (newline + 1) >= cap // 2:
+ del buf[: newline + 1]
+ except Exception:
+ # The sandbox died mid-stream, or the deadline cut the connection. What the
+ # command actually did is decided by ``wait()``, which classifies the same
+ # failure with the exit status in hand, so record the loss and stop reading.
+ flag[0] = True
+ log.debug("Modal sandbox stream ended early", exc_info=True)
+
+ # ------------------------------------------------------------------
+ # File operations.
+ #
+ # ``read_file`` is deliberately NOT overridden. Modal's ``filesystem.read_bytes``
+ # takes no length, so honoring ``max_bytes`` through it means stat-then-read: the
+ # TOCTOU window the base class documents avoiding, and no protection at all against
+ # anything ``stat`` reports as zero-length. Measured against modal 1.5.5, a
+ # ``read_bytes("/dev/zero")`` never returned and ``stat`` called that device
+ # ``size=0``, while a 200 MB regular file came back whole with no server-side
+ # ceiling. The base class's ``head -c`` runs inside the guest and cannot be lied to.
+ # ------------------------------------------------------------------
+
+ def write_file(self, sandbox: str, path: str, content: bytes) -> None:
+ """
+ Override: write through Modal's filesystem API instead of a shell command.
+
+ The base implementation carries the payload in the command itself, so the guest's
+ command-line length caps it. ``write_bytes`` streams the content and creates
+ parent directories itself.
+
+ One behavioral difference this buys, worth knowing before pointing an agent at a
+ tree of symlinks: writing to a path that is a symlink **replaces the link with a
+ regular file** and leaves the original target untouched, where a shell redirect
+ (what ``sbx`` and the base class do) follows the link and writes through it.
+ """
+ handle = self._handle(sandbox)
+ target = self._absolute(sandbox, path)
+ try:
+ handle.filesystem.write_bytes(content, target)
+ except modal.exception.Error as e:
+ raise self._as_sandbox_error(e, sandbox=sandbox) from e
+ except ValueError as e:
+ raise self._as_malformed_reply_error(e, operation="write") from e
+
+ def list_directory(self, sandbox: str, path: str) -> list[tuple[str, bool]]:
+ """
+ Override: list through Modal's filesystem API instead of ``find``.
+
+ Returns structured entries, so nothing has to be parsed out of shell output and
+ the image needs nothing from GNU find. Modal's filesystem calls do run a helper binary
+ inside the sandbox, but Modal injects it rather than expecting it in the image.
+
+ A symlink reports ``is_dir()`` false whatever it points at, because Modal types it
+ as a symlink rather than as its target. That matches the base class's
+ ``find -printf '%y'``, which does not follow links either, so a directory reached by a link
+ lists without a trailing slash on both backends.
+ """
+ handle = self._handle(sandbox)
+ target = self._absolute(sandbox, path)
+ try:
+ entries = handle.filesystem.list_files(target)
+ except modal.exception.Error as e:
+ raise self._as_sandbox_error(e, sandbox=sandbox) from e
+ except ValueError as e:
+ raise self._as_malformed_reply_error(e, operation="list") from e
+ return [(entry.name, entry.is_dir()) for entry in entries]
+
+ # ------------------------------------------------------------------
+ # Internals.
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def _as_malformed_reply_error(error: ValueError, *, operation: str) -> SandboxError:
+ """
+ Translate an unparsable reply from the filesystem helper into a sandbox error.
+
+ The native file operations are served by a helper binary Modal injects into the
+ guest, and the SDK parses its output. If that output is not what the SDK expects
+ it raises out of ``json`` rather than out of ``modal.exception``, so it would
+ otherwise escape the toolset's error handling and fail the task with a traceback
+ the model never sees and cannot act on. Model-written code running as root can
+ cause this deliberately, and a helper that is broken or version-skewed causes it
+ by accident.
+
+ Recoverable rather than terminal: the shell is a separate path that does not go
+ through the helper, so the model can get the same work done with ``run_command``.
+ """
+ return SandboxError(
+ f"The sandbox filesystem helper returned a reply that could not be parsed "
+ f"during a {operation} operation ({type(error).__name__}). Retry using "
+ f"run_command instead, which does not use the helper."
+ )
+
+ def _network_kwargs(self, spec: SandboxSpec | None) -> dict[str, object]:
+ """
+ Map a spec's network policy onto ``Sandbox.create`` arguments, or refuse it.
+
+ Raises :class:`SandboxTerminalError` rather than provisioning something weaker
+ than was asked for: an unenforceable spec is a Dag-author or Deployment-Manager
+ fact that the model cannot see and cannot fix by trying again.
+ """
+ if spec is None:
+ # "No requirements stated" -- see SandboxBackend.create. The toolset always
+ # sends a concrete spec, so this is the direct-caller path.
+ return {}
+ if spec.allow_egress_to:
+ if not spec.block_network:
+ raise SandboxTerminalError(
+ "SandboxSpec names an egress allowlist but leaves block_network False, "
+ "which asks for an open network and an allowlist at the same time. Set "
+ "block_network=True to restrict egress to allow_egress_to, or drop "
+ "allow_egress_to to leave the network open."
+ )
+ if self._egress_enforcement != "sni":
+ raise SandboxTerminalError(
+ "SandboxSpec names an egress allowlist, which Modal can only enforce by "
+ "matching the TLS SNI: it allows TLS on port 443 to those hosts, but "
+ "leaves DNS open for every hostname and cannot stop a host that shares a "
+ "TLS endpoint with an allowed one from being reached. Pass "
+ "ModalSandboxBackend(egress_enforcement='sni') to accept that, or use "
+ "SandboxSpec(block_network=True) with no allowlist, which Modal enforces "
+ "exactly and which also blocks DNS."
+ )
+ # block_network is deliberately not set alongside this: Modal rejects the
+ # combination outright, and an allowlist on its own already blocks every
+ # host that is not on it.
+ return {"outbound_domain_allowlist": self._hostnames(spec.allow_egress_to)}
+ if spec.block_network:
+ return {"block_network": True}
+ return {}
+
+ @staticmethod
+ def _hostnames(allow_egress_to: Sequence[str]) -> list[str]:
+ """
+ Check that every entry is a hostname Modal can actually match, or refuse the spec.
+
+ Modal passes these strings to its API without validating them, and matches them
+ against the hostname in the TLS handshake. So a URL, a ``host:port``, or a path
+ would be accepted and then match nothing at all, and a bare ``"*"`` is a wildcard
+ that matches everything. Either way the Dag author would be told egress is
+ restricted while it is not restricted the way they wrote it, which is the belief
+ the contract exists to protect.
+ """
+ if isinstance(allow_egress_to, str):
+ # A str is a Sequence[str], so this would otherwise be read one character at a
+ # time, and a name without dots would pass every character check and then
+ # allow nothing at all.
+ raise SandboxTerminalError(
+ "SandboxSpec.allow_egress_to must be a sequence of hostnames, not one string: "
+ f"{allow_egress_to!r} would be read a character at a time. Wrap it in a list."
+ )
+ rejected = [host for host in allow_egress_to if not _is_tls_hostname(host)]
+ if rejected:
+ raise SandboxTerminalError(
+ f"SandboxSpec.allow_egress_to must contain bare hostnames, optionally with a "
+ f"leading '*.' wildcard label, because Modal matches them against the hostname "
+ f"in the TLS handshake. These entries are not hostnames and would silently match "
+ f"nothing, or everything: {rejected}. Write 'pypi.org' or '*.pythonhosted.org', "
+ f"not a URL, a host:port, a bare '*', or a single label such as 'localhost'."
+ )
+ return list(allow_egress_to)
+
+ @staticmethod
+ def _environment(spec: SandboxSpec | None) -> dict[str, str] | None:
+ """
+ Return the environment to inject, refusing anything Modal cannot carry.
+
+ ``SandboxSpec.env`` is typed as strings but nothing enforces that, and Modal's
+ environment is strings too, so an ``int`` would fail somewhere deeper and less
+ legibly. Refuse it here with the offending keys named.
+ """
+ if spec is None or not spec.env:
+ return None
+ rejected = sorted(
+ str(key)
+ for key, value in spec.env.items()
+ if not isinstance(key, str) or not isinstance(value, str)
+ )
+ if rejected:
+ raise SandboxTerminalError(
+ f"SandboxSpec.env must map strings to strings; these entries do not: {rejected}. "
+ "Convert the value at the Dag level, e.g. str(port), so what reaches the sandbox "
+ "is what you meant."
+ )
+ return dict(spec.env)
+
+ def _remaining_lifetime(self, sandbox: str) -> float | None:
+ """
+ Seconds left before Modal ends this sandbox, or ``None`` if we did not create it.
+
+ A handle from elsewhere has a lifetime this backend cannot know, so nothing is
+ claimed about it rather than a number being invented.
+ """
+ expiry = self._expiries.get(sandbox)
+ return None if expiry is None else max(0.0, expiry - time.monotonic())
+
+ def _handle(self, sandbox: str) -> modal.Sandbox:
+ """Return the live sandbox object for a handle, looking it up if not cached."""
+ cached = self._sandboxes.get(sandbox)
+ if cached is not None:
+ return cached
+ try:
+ # Reachable when the handle came from another backend instance, or from a
+ # caller that created the sandbox elsewhere.
+ found = modal.Sandbox.from_id(sandbox)
+ except modal.exception.Error as e:
+ raise self._as_sandbox_error(e, sandbox=sandbox) from e
+ self._sandboxes[sandbox] = found
+ return found
+
+ def _absolute(self, sandbox: str, path: str) -> str:
+ """
+ Resolve ``path`` for Modal's filesystem API, which only accepts absolute paths.
+
+ Resolved against the sandbox's working directory, which is also what the shell
+ behind ``run_command`` and the inherited ``read_file`` resolve against, so both
+ views of the filesystem agree on what a relative path means.
+
+ ``.`` and doubled separators are collapsed, since they mean the same thing to
+ everyone. ``..`` is deliberately left alone: resolving it here would be a lexical
+ answer, and a shell resolves it physically, so the two disagree the moment a
+ symlink is in the path. Leaving it in place hands one path to both, and if Modal
+ will not take it the author hears about it instead of writing to a file nobody
+ asked for.
+ """
+ joined = path if path.startswith("/") else posixpath.join(self._working_directory(sandbox), path)
+ return "/" + "/".join(segment for segment in joined.split("/") if segment not in ("", "."))
+
+ def _working_directory(self, sandbox: str) -> str:
+ if self._workdir is not None:
+ return self._workdir
+ cached = self._workdirs.get(sandbox)
+ if cached is not None:
+ return cached
+ # Only reachable when the backend was built with workdir=None, so the image
+ # decides. Asked once per sandbox and remembered.
+ result = self.run_command(sandbox, "pwd", timeout=_FILE_OP_TIMEOUT, max_output_bytes=4096)
+ resolved = result.stdout.strip().splitlines()[-1] if result.stdout.strip() else ""
+ if result.exit_code or not resolved.startswith("/"):
+ raise SandboxError(
+ "Could not determine the sandbox working directory, so a relative path "
+ "cannot be resolved. Use an absolute path, or set workdir on the backend."
+ )
+ self._workdirs[sandbox] = resolved
+ return resolved
+
+ def _as_sandbox_error(self, error: modal.exception.Error, *, sandbox: str | None = None) -> SandboxError:
+ """
+ Translate a Modal error into the contract's two kinds.
+
+ Recoverable errors reach the model as a retry; terminal ones fail the task so
+ Airflow's own retry handles them. Order matters here: ``ConflictError`` is a
+ subclass of ``InvalidError``, and ``SandboxTimeoutError`` shares a ``TimeoutError``
+ parent with ``ExecTimeoutError``, which means the opposite thing.
+ """
+ if isinstance(error, modal.exception.ExecTimeoutError):
+ # A command deadline, which run_command reports as output rather than an
+ # error. Only reachable if it escapes from somewhere that cannot report it.
+ return SandboxError(f"The command hit its deadline: {error}")
+ if isinstance(error, modal.exception.AuthError):
+ return SandboxTerminalError(
+ "Modal rejected the credentials. Run 'modal token new', or set "
+ f"MODAL_TOKEN_ID and MODAL_TOKEN_SECRET on the worker: {error}"
+ )
+ if isinstance(error, modal.exception.PermissionDeniedError):
+ # A sibling of AuthError rather than a subclass, and just as unfixable by
+ # anything the model could say: the workspace does not allow this.
+ return SandboxTerminalError(f"Modal refused the operation: {error}")
+ if isinstance(
+ error,
+ (
+ modal.exception.SandboxTerminatedError,
+ modal.exception.SandboxTimeoutError,
+ modal.exception.NotFoundError,
+ ),
+ ):
+ if sandbox is None:
+ # Reached from create, where no sandbox exists yet, so a missing app or
+ # image must not be reported as a sandbox that died.
+ return SandboxTerminalError(f"Modal could not find what the request named: {error}")
+ return SandboxTerminalError(
+ f"The Modal sandbox is gone: it was terminated, or it reached its sandbox_timeout. {error}"
+ )
+ if isinstance(error, modal.exception.SandboxFilesystemError):
+ # A bad path, a directory where a file was expected, a permission the guest
+ # does not have: all things the model can correct.
+ return SandboxError(str(error))
+ if isinstance(error, modal.exception.ConflictError):
+ # A subclass of InvalidError, so it has to be caught first, and it means
+ # something different: observed as "Modal Sandbox is shutting down", which is
+ # the sandbox going away rather than the request being wrong.
+ return SandboxTerminalError(f"The Modal sandbox is shutting down: {error}")
+ if isinstance(error, modal.exception.InvalidError):
+ # Modal rejected the request itself. Retrying the same call cannot help.
+ return SandboxTerminalError(f"Modal rejected the request: {error}")
+ # Everything else -- a connection blip, an unmapped server error -- is ambiguous
+ # on its own, so ask whether the sandbox is still there before deciding.
+ if sandbox is not None and self._has_finished(sandbox):
+ return SandboxTerminalError(f"The Modal sandbox is no longer running: {error}")
+ return SandboxError(str(error))
+
+ def _has_finished(self, sandbox: str) -> bool:
+ """
+ Whether the sandbox has stopped, used to classify an otherwise ambiguous error.
+
+ The answer can lag reality: measured against modal 1.5.5, a sandbox that has just
+ been terminated still polls as running for around 35 seconds. So an ambiguous error
+ arriving in that window is treated as recoverable, the model retries once, and the
+ next call gets the unambiguous ``SandboxTerminatedError`` and fails the task. One
+ wasted retry is the right trade against blocking to find out sooner. Where the
+ answer has to be right now -- deciding whether a command's sandbox died under it --
+ :meth:`_still_alive` asks by running a command instead, which does not lag.
+ """
+ handle = self._sandboxes.get(sandbox)
+ if handle is None:
+ return False
+ with suppress(Exception):
+ return handle.poll() is not None
+ return False
diff --git a/providers/common/ai/src/airflow/providers/common/ai/toolsets/sandbox.py b/providers/common/ai/src/airflow/providers/common/ai/toolsets/sandbox.py
index cb7283c5c4e..15870bb657c 100644
--- a/providers/common/ai/src/airflow/providers/common/ai/toolsets/sandbox.py
+++ b/providers/common/ai/src/airflow/providers/common/ai/toolsets/sandbox.py
@@ -107,7 +107,8 @@ _DESCRIPTIONS = {
"Run a shell command inside an isolated sandbox and return its output. Pipes, "
"redirection, && and globs work. A non-zero exit is reported, not raised, so read "
"stderr and fix your command. The sandbox is separate from the Airflow worker and "
- "is not given Airflow's connections, variables, or environment."
+ "is not given Airflow's connections, variables, or environment. Relative paths "
+ "resolve against the sandbox's working directory; run 'pwd' if you need to know it."
),
READ_FILE: (
"Read a text file from the sandbox. Long files are truncated and the result tells "
@@ -286,6 +287,29 @@ class SandboxToolset(AbstractToolset[Any]):
if self._create_task is create_task:
self._create_task = None
+ def _network_note(self) -> str:
+ """
+ Describe the sandbox's egress in the one place the model reliably reads.
+
+ Without it the model has to discover the policy by failing: measured, a
+ ``pip install`` in a sandbox that denies egress costs a turn and returns a DNS
+ error, and under an allowlist a reach for plain HTTP burns the whole command
+ budget and returns only a timeout, which reads as "my command was slow". The
+ spec is known here, so say it instead.
+ """
+ if not self._spec.block_network:
+ return "This sandbox has outbound network access."
+ allowed = list(self._spec.allow_egress_to or ())
+ if not allowed:
+ return (
+ "This sandbox has NO network access, including DNS, so installing packages "
+ "or downloading anything will fail. Work with what the image already has."
+ )
+ return (
+ "This sandbox reaches only these hosts, over HTTPS on port 443: "
+ f"{', '.join(allowed)}. Anything else, and plain HTTP to any host, will fail."
+ )
+
async def get_tools(self, ctx: RunContext[Any]) -> dict[str, ToolsetTool[Any]]:
tools: dict[str, ToolsetTool[Any]] = {}
for base, schema in _SCHEMAS.items():
@@ -297,9 +321,12 @@ class SandboxToolset(AbstractToolset[Any]):
# leaves it native instead of folding a shell surface into run_code.
extra = code_arg_kwargs("command", "shell") if base == RUN_COMMAND else {}
name = self._tool_name(base)
+ description = _DESCRIPTIONS[base]
+ if base == RUN_COMMAND:
+ description = f"{description} {self._network_note()}"
tool_def = ToolDefinition(
name=name,
- description=_DESCRIPTIONS[base],
+ description=description,
parameters_json_schema=schema,
sequential=True,
**return_schema_kwargs({"type": "string"}),
@@ -375,13 +402,22 @@ class SandboxToolset(AbstractToolset[Any]):
if result.stderr:
parts.append(f"[stderr]\n{self._truncate(result.stderr, result.stderr_truncated)}")
output = "\n".join(parts) if parts else "(no output)"
+ notes: list[str] = []
if result.timed_out:
- note = f"[timed out after {timeout:g}s]"
- if result.sandbox_terminated:
- note += " [sandbox was replaced; files from earlier calls are gone]"
- return f"{output}\n{note}"
- if result.exit_code:
- return f"{output}\n[exit code: {result.exit_code}]"
+ # What the command actually got, which a backend may have had to shorten.
+ # Telling the model the number it asked for would send it back with a bigger
+ # one when the real constraint was never its request.
+ applied = result.applied_timeout if result.applied_timeout is not None else timeout
+ notes.append(f"[timed out after {applied:g}s]")
+ elif result.exit_code:
+ notes.append(f"[exit code: {result.exit_code}]")
+ if result.sandbox_terminated:
+ # Said whatever else happened, not only after a timeout: a backend can lose
+ # the sandbox under an ordinary-looking failure too, and the model has to
+ # learn that its files are gone from somewhere.
+ notes.append("[sandbox was replaced; files from earlier calls are gone]")
+ if notes:
+ return f"{output}\n{' '.join(notes)}"
return output
def _truncate(self, text: str, already_truncated: bool) -> str:
diff --git a/providers/common/ai/tests/system/common/ai/example_sandbox_toolset_modal.py b/providers/common/ai/tests/system/common/ai/example_sandbox_toolset_modal.py
new file mode 100644
index 00000000000..97d024f56f5
--- /dev/null
+++ b/providers/common/ai/tests/system/common/ai/example_sandbox_toolset_modal.py
@@ -0,0 +1,258 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+End-to-end system test for SandboxToolset with the Modal backend.
+
+Needs Modal credentials on the worker: ``modal token new`` locally, or
+``MODAL_TOKEN_ID`` and ``MODAL_TOKEN_SECRET`` in the environment.
+
+Covers what only a live sandbox can show: that the four tools agree on one
+filesystem, that a non-zero exit is output rather than a failure, that a command
+hitting its deadline leaves the sandbox and its files intact (which is where Modal
+differs from ``sbx``), that the default spec really does deny egress, that none of
+Airflow's own environment crosses the boundary, and that teardown terminates the
+sandbox.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from datetime import datetime, timezone
+
+from airflow.providers.common.compat.sdk import dag as airflow_dag, task
+
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = f"common_ai_sandbox_toolset_modal_{ENV_ID}" if ENV_ID else "common_ai_sandbox_toolset_modal"
+
+MARKER = "boundary-ok"
+STATE_PATH = "state.txt"
+ABSOLUTE_STATE_PATH = "/workspace/state.txt"
+
+
+@airflow_dag(
+ dag_id=DAG_ID,
+ schedule="@once",
+ start_date=datetime(2024, 1, 1, tzinfo=timezone.utc),
+ catchup=False,
+ tags=["common.ai", "sandbox", "modal", "system_test"],
+)
+def example_sandbox_toolset_modal():
+ @task
+ def run_sandbox_agent() -> str:
+ # Keep task-only dependencies out of the Dag-parsing process.
+ from pydantic_ai import Agent
+ from pydantic_ai.messages import ModelMessage, ModelResponse, TextPart, ToolCallPart
+ from pydantic_ai.models.function import AgentInfo, FunctionModel
+
+ from airflow.providers.common.ai.sandbox import ModalSandboxBackend
+ from airflow.providers.common.ai.toolsets import SandboxToolset
+
+ class RecordingBackend(ModalSandboxBackend):
+ """Remembers the handles it created and destroyed, so teardown can be asserted."""
+
+ def __init__(self, **kwargs) -> None:
+ super().__init__(**kwargs)
+ self.created: list[str] = []
+ self.destroyed: list[str] = []
+
+ def create(self, *, spec=None) -> str:
+ handle = super().create(spec=spec)
+ self.created.append(handle)
+ return handle
+
+ def destroy(self, sandbox: str) -> None:
+ super().destroy(sandbox)
+ self.destroyed.append(sandbox)
+
+ def call(tool: str, call_id: str, **args) -> ModelResponse:
+ return ModelResponse(parts=[ToolCallPart(tool_name=tool, args=args, tool_call_id=call_id)])
+
+ def model_function(messages: list[ModelMessage], _info: AgentInfo) -> ModelResponse:
+ # Both kinds count: a tool that raises ModelRetry -- which is how the toolset
+ # reports a recoverable failure -- comes back as a retry-prompt part rather
+ # than a tool-return, and counting only the latter would re-issue the same
+ # call until the tool exhausted its retries.
+ returns = [
+ str(part.content)
+ for message in messages
+ for part in message.parts
+ if part.part_kind in ("tool-return", "retry-prompt")
+ ]
+
+ # 1. Write through the file tool, with a relative path: the native API takes
+ # absolute paths only, so this also proves the workdir resolution.
+ if not returns:
+ return call("write_file", "write", path=STATE_PATH, content=MARKER)
+ if "Wrote" not in str(returns[0]):
+ raise RuntimeError(f"Unexpected write_file result: {returns[0]!r}")
+
+ # 2. The shell sees the same file at the absolute path, and the image carries
+ # a real interpreter.
+ if len(returns) == 1:
+ return call(
+ "run_command",
+ "shell",
+ command=f"cat {ABSOLUTE_STATE_PATH} && python3 -c 'print(6 * 7)'",
+ )
+ shell_out = str(returns[1])
+ if MARKER not in shell_out or "42" not in shell_out:
+ raise RuntimeError(f"Unexpected run_command result: {shell_out!r}")
+
+ # 3. And the file tool reads back what the shell just saw.
+ if len(returns) == 2:
+ return call("read_file", "read", path=STATE_PATH)
+ if MARKER not in str(returns[2]):
+ raise RuntimeError(f"Unexpected read_file result: {returns[2]!r}")
+
+ # 4. A non-zero exit is output to react to, not an exception.
+ if len(returns) == 3:
+ return call("run_command", "fail", command="echo to-stderr >&2; exit 3")
+ failed = str(returns[3])
+ if "[exit code: 3]" not in failed or "to-stderr" not in failed:
+ raise RuntimeError(f"Unexpected failure result: {failed!r}")
+
+ # 5. A deadline is reported, and Modal does not have to destroy the sandbox
+ # to enforce it, so the toolset must not say the sandbox was replaced.
+ if len(returns) == 4:
+ return call("run_command", "slow", command="sleep 30", timeout_seconds=2)
+ timed_out = str(returns[4])
+ if "[timed out after 2s]" not in timed_out:
+ raise RuntimeError(f"Expected a timeout, got: {timed_out!r}")
+ if "sandbox was replaced" in timed_out:
+ raise RuntimeError("Modal should survive a command timeout, but the sandbox was replaced")
+
+ # 6. Which means the file from step 1 is still there afterwards.
+ if len(returns) == 5:
+ return call("run_command", "survived", command=f"cat {ABSOLUTE_STATE_PATH}")
+ if MARKER not in str(returns[5]):
+ raise RuntimeError(f"Files did not survive the timeout: {returns[5]!r}")
+
+ # 7. Airflow puts none of its own environment into the sandbox.
+ if len(returns) == 6:
+ return call(
+ "run_command",
+ "env",
+ command="python3 -c \"import os; print(sorted(k for k in os.environ if 'AIRFLOW' in k))\"",
+ )
+ leaked = str(returns[6])
+ if "[]" not in leaked:
+ raise RuntimeError(f"Airflow environment crossed the boundary: {leaked!r}")
+
+ # 8. The default spec denies egress, and Modal's block_network takes DNS with it.
+ if len(returns) == 7:
+ return call(
+ "run_command",
+ "egress",
+ command="python3 -c \"import socket; print(socket.gethostbyname('pypi.org'))\"",
+ )
+ egress = str(returns[7])
+ if "[exit code:" not in egress:
+ raise RuntimeError(f"Egress should have been denied, got: {egress!r}")
+
+ # 9. The listing sees the file too, and marks a directory as one.
+ if len(returns) == 8:
+ return call("run_command", "mkdir", command="mkdir -p /workspace/sub")
+ if len(returns) == 9:
+ return call("list_directory", "ls", path="/workspace")
+ listing = str(returns[9])
+ if "state.txt" not in listing or "sub/" not in listing:
+ raise RuntimeError(f"Unexpected listing: {listing!r}")
+
+ # 10. The documented symlink difference: a native write replaces a link rather
+ # than following it, where a shell redirect would write through it. Only a
+ # live sandbox can show which one this backend does.
+ if len(returns) == 10:
+ return call(
+ "run_command",
+ "link",
+ command="printf original > /workspace/target.txt && ln -s /workspace/target.txt /workspace/link.txt",
+ )
+ if len(returns) == 11:
+ return call("write_file", "through-link", path="link.txt", content="written-through-link")
+ if len(returns) == 12:
+ return call(
+ "run_command",
+ "link-check",
+ command="readlink /workspace/link.txt || echo NOT-A-LINK; cat /workspace/target.txt",
+ )
+ link_state = str(returns[12])
+ if "NOT-A-LINK" not in link_state or "original" not in link_state:
+ raise RuntimeError(
+ f"write_file should replace a symlink and leave its target alone: {link_state!r}"
+ )
+
+ # 11. And the reason read_file is not on the native API: a file that streams
+ # without end must be refused by the in-guest cap, not pulled into the
+ # worker. /dev/zero exists only in a real sandbox.
+ if len(returns) == 13:
+ return call("read_file", "streaming", path="/dev/zero")
+ streaming = str(returns[13])
+ if "read limit" not in streaming:
+ raise RuntimeError(f"/dev/zero should be refused by the read budget: {streaming!r}")
+
+ # 12. Reading a directory is an error, not an empty file. ``stat`` succeeds on a
+ # directory and the pipeline's status is base64's, so without the explicit
+ # check the model was told the directory was a file with no lines.
+ if len(returns) == 14:
+ return call("read_file", "directory", path="/workspace/sub")
+ directory = str(returns[14])
+ if "is a directory" not in directory:
+ raise RuntimeError(f"Reading a directory should be refused: {directory!r}")
+
+ return ModelResponse(parts=[TextPart(content="sandbox boundary e2e passed")])
+
+ # The default spec denies all egress, which Modal enforces exactly.
+ backend = RecordingBackend(app_name="airflow-sandbox-system-test", sandbox_timeout=900)
+ agent = Agent(
+ FunctionModel(model_function),
+ instructions="Use the sandbox tools as requested.",
+ toolsets=[SandboxToolset(backend, max_command_timeout=60.0)],
+ )
+ result = agent.run_sync("Run the sandbox boundary system test.")
+ if result.output != "sandbox boundary e2e passed":
+ raise RuntimeError(f"Unexpected agent output: {result.output!r}")
+
+ # Teardown: the toolset destroys the sandbox when the run ends. Modal's
+ # termination is asynchronous, and an exit status can take some seconds to
+ # show, so poll rather than assuming either way.
+ if backend.created != backend.destroyed:
+ raise RuntimeError(f"created {backend.created} but destroyed {backend.destroyed}")
+ import modal
+
+ handle = backend.created[0]
+ # Termination showed an exit status 32-33s after the request when measured, so this
+ # is a little over three times the observed value rather than under twice it.
+ deadline = time.monotonic() + 120
+ while time.monotonic() < deadline:
+ if modal.Sandbox.from_id(handle).poll() is not None:
+ break
+ time.sleep(5)
+ else:
+ raise RuntimeError(f"Sandbox {handle} was still running 60s after teardown")
+
+ return result.output
+
+ run_sandbox_agent()
+
+
+dag = example_sandbox_toolset_modal()
+
+from tests_common.test_utils.system_tests import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst)
+test_run = get_test_run(dag)
diff --git a/providers/common/ai/tests/unit/common/ai/sandbox/fake_modal.py b/providers/common/ai/tests/unit/common/ai/sandbox/fake_modal.py
new file mode 100644
index 00000000000..74810254e94
--- /dev/null
+++ b/providers/common/ai/tests/unit/common/ai/sandbox/fake_modal.py
@@ -0,0 +1,322 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+A stand-in for the ``modal`` SDK, faithful in the ways the backend depends on.
+
+The backend's error handling turns on ``isinstance`` checks whose ordering only matters
+because of how Modal's own classes are related, so the hierarchy here mirrors
+``modal==1.5.5`` exactly: ``ConflictError`` derives from ``InvalidError``,
+``SandboxTimeoutError`` and ``ExecTimeoutError`` share a ``TimeoutError`` parent while
+meaning opposite things, and every ``SandboxFilesystem*Error`` derives from
+``SandboxFilesystemError`` and from nothing else, so a missing file is never mistaken for
+a missing sandbox. A fake that flattened any of that would let a real bug through.
+
+What it cannot prove, and what the system test therefore has to: that a deadline arrives
+as returncode -1, that the filesystem API rejects relative paths, that ``write_bytes``
+replaces a symlink, and that ``terminate`` is not synchronous.
+"""
+
+from __future__ import annotations
+
+import types
+from typing import Any
+
+
+class Error(Exception):
+ """Root of Modal's error hierarchy."""
+
+
+class TimeoutError(Error):
+ pass
+
+
+class ExecTimeoutError(TimeoutError):
+ pass
+
+
+class SandboxTimeoutError(TimeoutError):
+ pass
+
+
+class SandboxTerminatedError(Error):
+ pass
+
+
+class ConnectionError(Error):
+ pass
+
+
+class InvalidError(Error):
+ pass
+
+
+class ConflictError(InvalidError):
+ pass
+
+
+class NotFoundError(Error):
+ pass
+
+
+class AuthError(Error):
+ pass
+
+
+class PermissionDeniedError(Error):
+ """A sibling of AuthError in the real SDK, not a subclass, which is the point."""
+
+
+class SandboxFilesystemError(Error):
+ pass
+
+
+class SandboxFilesystemNotFoundError(SandboxFilesystemError):
+ pass
+
+
+class SandboxFilesystemIsADirectoryError(SandboxFilesystemError):
+ pass
+
+
+class SandboxFilesystemPermissionError(SandboxFilesystemError):
+ pass
+
+
+class SandboxFilesystemFileTooLargeError(SandboxFilesystemError):
+ pass
+
+
+class FileInfo:
+ """
+ Mirrors ``modal.types.FileInfo`` in the parts the backend reads.
+
+ ``kind`` is Modal's three-valued ``FileType``, not a boolean, which matters: a symlink
+ is its own type, so ``is_dir()`` is false even for a link that points at a directory.
+ """
+
+ def __init__(self, name: str, *, kind: str = "file", size: int = 0) -> None:
+ if kind not in ("file", "directory", "symlink"):
+ raise ValueError(f"kind must be file, directory or symlink, got {kind!r}")
+ self.name = name
+ self.size = size
+ self.kind = kind
+
+ def is_dir(self) -> bool:
+ return self.kind == "directory"
+
+ def is_symlink(self) -> bool:
+ return self.kind == "symlink"
+
+
+class FakeProcess:
+ """A container process whose streams and exit status are scripted by a test."""
+
+ def __init__(
+ self,
+ *,
+ returncode: int = 0,
+ stdout: list[bytes] | None = None,
+ stderr: list[bytes] | None = None,
+ wait_error: Exception | None = None,
+ stream_error: Exception | None = None,
+ ) -> None:
+ self._returncode = returncode
+ self._wait_error = wait_error
+ self.stdout = _FakeStream(stdout or [], stream_error)
+ self.stderr = _FakeStream(stderr or [], None)
+ self.waited = False
+
+ def wait(self) -> int:
+ self.waited = True
+ if self._wait_error is not None:
+ raise self._wait_error
+ return self._returncode
+
+
+class _FakeStream:
+ """Yields byte chunks, optionally dying part-way through like a severed connection."""
+
+ def __init__(self, chunks: list[bytes], error: Exception | None) -> None:
+ self._chunks = chunks
+ self._error = error
+
+ def __iter__(self):
+ yield from self._chunks
+ if self._error is not None:
+ raise self._error
+
+
+class FakeFilesystem:
+ def __init__(self, sandbox: FakeSandbox) -> None:
+ self._sandbox = sandbox
+
+ def write_bytes(self, data: bytes, remote_path: str) -> None:
+ self._sandbox.calls.append(("write_bytes", remote_path, data))
+ if self._sandbox.filesystem_error is not None:
+ raise self._sandbox.filesystem_error
+ self._sandbox.files[remote_path] = data
+
+ def read_bytes(self, remote_path: str) -> bytes:
+ self._sandbox.calls.append(("read_bytes", remote_path))
+ if self._sandbox.filesystem_error is not None:
+ raise self._sandbox.filesystem_error
+ try:
+ return self._sandbox.files[remote_path]
+ except KeyError:
+ raise SandboxFilesystemNotFoundError(f"path does not exist: {remote_path}") from None
+
+ def list_files(self, remote_path: str) -> list[FileInfo]:
+ self._sandbox.calls.append(("list_files", remote_path))
+ if self._sandbox.filesystem_error is not None:
+ raise self._sandbox.filesystem_error
+ return self._sandbox.listing
+
+ def stat(self, remote_path: str) -> FileInfo:
+ self._sandbox.calls.append(("stat", remote_path))
+ if self._sandbox.filesystem_error is not None:
+ raise self._sandbox.filesystem_error
+ return FileInfo(remote_path.rsplit("/", 1)[-1])
+
+
+class FakeSandbox:
+ """One sandbox. Tests drive its behaviour by setting the attributes below."""
+
+ def __init__(self, object_id: str, **create_kwargs: Any) -> None:
+ self.object_id = object_id
+ self.create_kwargs = create_kwargs
+ self.calls: list[tuple] = []
+ self.files: dict[str, bytes] = {}
+ self.listing: list[FileInfo] = []
+ self.terminated = False
+ self.terminate_error: Exception | None = None
+ self.exec_error: Exception | None = None
+ # Consumed one per exec when set, so a test can let the command through and
+ # fail only the liveness probe that follows it.
+ self.exec_errors: list[Exception | None] = []
+ self.filesystem_error: Exception | None = None
+ self.poll_result: int | None = None
+ # Either one process for every exec, or a queue consumed in order.
+ self.process = FakeProcess()
+ self.processes: list[FakeProcess] = []
+
+ @property
+ def filesystem(self) -> FakeFilesystem:
+ return FakeFilesystem(self)
+
+ def exec(self, *args: str, **kwargs: Any) -> FakeProcess:
+ self.calls.append(("exec", args, kwargs))
+ error = self.exec_errors.pop(0) if self.exec_errors else self.exec_error
+ if error is not None:
+ raise error
+ if self.processes:
+ return self.processes.pop(0)
+ return self.process
+
+ def terminate(self, *, wait: bool = False) -> None:
+ self.calls.append(("terminate", wait))
+ if self.terminate_error is not None:
+ raise self.terminate_error
+ self.terminated = True
+
+ def poll(self) -> int | None:
+ return self.poll_result
+
+
+class FakeSandboxFactory:
+ """Stands in for ``modal.Sandbox``: records creations and hands out fakes."""
+
+ def __init__(self) -> None:
+ self.created: list[FakeSandbox] = []
+ self.create_error: Exception | None = None
+ self.from_id_error: Exception | None = None
+ self.by_id: dict[str, FakeSandbox] = {}
+ self.next_process: FakeProcess | None = None
+
+ def create(self, **kwargs: Any) -> FakeSandbox:
+ if self.create_error is not None:
+ raise self.create_error
+ sandbox = FakeSandbox(f"sb-{len(self.created)}", **kwargs)
+ if self.next_process is not None:
+ sandbox.process = self.next_process
+ self.created.append(sandbox)
+ self.by_id[sandbox.object_id] = sandbox
+ return sandbox
+
+ def from_id(self, object_id: str) -> FakeSandbox:
+ if self.from_id_error is not None:
+ raise self.from_id_error
+ try:
+ return self.by_id[object_id]
+ except KeyError:
+ raise NotFoundError(f"no such sandbox: {object_id}") from None
+
+
+class FakeApp:
+ def __init__(self, name: str) -> None:
+ self.name = name
+
+
+class FakeAppFactory:
+ def __init__(self) -> None:
+ self.lookups: list[tuple[str, bool]] = []
+
+ def lookup(self, name: str, *, create_if_missing: bool = False) -> FakeApp:
+ self.lookups.append((name, create_if_missing))
+ return FakeApp(name)
+
+
+class FakeImageFactory:
+ def __init__(self) -> None:
+ self.registries: list[str] = []
+
+ def from_registry(self, tag: str) -> str:
+ self.registries.append(tag)
+ return f"image:{tag}"
+
+
+def build_fake_modal() -> types.ModuleType:
+ """Assemble a module object shaped like ``modal``, with its exception hierarchy."""
+ exception = types.ModuleType("modal.exception")
+ for name, cls in (
+ ("Error", Error),
+ ("TimeoutError", TimeoutError),
+ ("ExecTimeoutError", ExecTimeoutError),
+ ("SandboxTimeoutError", SandboxTimeoutError),
+ ("SandboxTerminatedError", SandboxTerminatedError),
+ ("ConnectionError", ConnectionError),
+ ("InvalidError", InvalidError),
+ ("ConflictError", ConflictError),
+ ("NotFoundError", NotFoundError),
+ ("AuthError", AuthError),
+ ("PermissionDeniedError", PermissionDeniedError),
+ ("SandboxFilesystemError", SandboxFilesystemError),
+ ("SandboxFilesystemNotFoundError", SandboxFilesystemNotFoundError),
+ ("SandboxFilesystemIsADirectoryError", SandboxFilesystemIsADirectoryError),
+ ("SandboxFilesystemPermissionError", SandboxFilesystemPermissionError),
+ ("SandboxFilesystemFileTooLargeError", SandboxFilesystemFileTooLargeError),
+ ):
+ setattr(exception, name, cls)
+
+ module = types.ModuleType("modal")
+ for name, attr in (
+ ("exception", exception),
+ ("Sandbox", FakeSandboxFactory()),
+ ("App", FakeAppFactory()),
+ ("Image", FakeImageFactory()),
+ ):
+ setattr(module, name, attr)
+ return module
diff --git a/providers/common/ai/tests/unit/common/ai/sandbox/test_base.py b/providers/common/ai/tests/unit/common/ai/sandbox/test_base.py
index 7b44305e3f8..f6f50cf6582 100644
--- a/providers/common/ai/tests/unit/common/ai/sandbox/test_base.py
+++ b/providers/common/ai/tests/unit/common/ai/sandbox/test_base.py
@@ -180,6 +180,23 @@ class TestDefaultFileOperations:
with pytest.raises(SandboxError, match="does not exist"):
local.read_file("s", str(tmp_path / "nope.txt"), max_bytes=100)
+ def test_a_directory_is_an_error_not_an_empty_read(self, local, tmp_path):
+ # Regression: ``stat`` succeeds on a directory and the pipeline's exit
+ # status is base64's, so a directory used to read back as an empty file
+ # and the model was told the directory was a file with no lines.
+ (tmp_path / "sub").mkdir()
+ (tmp_path / "sub" / "inner.txt").write_text("x")
+
+ with pytest.raises(SandboxError, match="is a directory"):
+ local.read_file("s", str(tmp_path / "sub"), max_bytes=100)
+
+ def test_a_symlink_to_a_directory_is_an_error_too(self, local, tmp_path):
+ (tmp_path / "sub").mkdir()
+ (tmp_path / "link").symlink_to(tmp_path / "sub")
+
+ with pytest.raises(SandboxError, match="is a directory"):
+ local.read_file("s", str(tmp_path / "link"), max_bytes=100)
+
def test_oversized_file_is_refused(self, local, tmp_path):
(tmp_path / "big.bin").write_bytes(b"x" * 500)
diff --git a/providers/common/ai/tests/unit/common/ai/sandbox/test_modal.py b/providers/common/ai/tests/unit/common/ai/sandbox/test_modal.py
new file mode 100644
index 00000000000..11cea17ede3
--- /dev/null
+++ b/providers/common/ai/tests/unit/common/ai/sandbox/test_modal.py
@@ -0,0 +1,1120 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import builtins
+import importlib
+import json
+import sys
+import threading
+
+import pytest
+
+import airflow.providers.common.ai.sandbox as sandbox_package
+from airflow.providers.common.ai.sandbox.base import (
+ SandboxError,
+ SandboxSpec,
+ SandboxTerminalError,
+)
+from airflow.providers.common.compat.sdk import AirflowOptionalProviderFeatureException
+
+from unit.common.ai.sandbox.fake_modal import (
+ FakeProcess,
+ FileInfo,
+ build_fake_modal,
+)
+
+
+@pytest.fixture
+def modal_module(monkeypatch):
+ """
+ Install the fake ``modal`` and hand back a freshly imported backend module bound to it.
+
+ The backend imports ``modal`` at module scope, so the import has to happen while the
+ fake is in ``sys.modules``. Re-importing also keeps the tests honest about not sharing
+ state, and lets the suite run whether or not the real SDK is installed.
+ """
+ fake = build_fake_modal()
+ monkeypatch.setitem(sys.modules, "modal", fake)
+ monkeypatch.setitem(sys.modules, "modal.exception", fake.exception)
+ monkeypatch.delitem(sys.modules, "airflow.providers.common.ai.sandbox.modal", raising=False)
+ import airflow.providers.common.ai.sandbox.modal as backend_module
+
+ yield fake, backend_module
+ # The next test re-imports from scratch, so nothing here should linger.
+ sys.modules.pop("airflow.providers.common.ai.sandbox.modal", None)
+
+
+@pytest.fixture
+def fake(modal_module):
+ return modal_module[0]
+
+
+@pytest.fixture
+def backend_class(modal_module):
+ return modal_module[1].ModalSandboxBackend
+
+
+@pytest.fixture
+def backend(backend_class):
+ return backend_class(app_name="test-app", sandbox_timeout=600, idle_timeout=60)
+
+
+def _created(backend, fake, spec=None):
+ """Create one sandbox and return ``(handle, fake_sandbox)``."""
+ handle = backend.create(spec=spec)
+ return handle, fake.Sandbox.by_id[handle]
+
+
+class TestVendorContract:
+ """
+ Guards the calls the fake cannot: the fake accepts any argument, by design.
+
+ Skipped where the extra is not installed, which is most CI jobs, so this is a
+ tripwire for a Modal release that renames something rather than a gate.
+ """
+
+ @pytest.fixture
+ def real_modal(self):
+ return pytest.importorskip("modal", reason="needs the 'modal' extra")
+
+ def test_create_accepts_every_argument_the_backend_passes(self, real_modal):
+ import inspect
+
+ parameters = inspect.signature(real_modal.Sandbox.create).parameters
+ for name in (
+ "app",
+ "image",
+ "name",
+ "tags",
+ "timeout",
+ "idle_timeout",
+ "workdir",
+ "env",
+ "cpu",
+ "memory",
+ "gpu",
+ "region",
+ "cloud",
+ "block_network",
+ "outbound_domain_allowlist",
+ ):
+ assert name in parameters, f"Sandbox.create no longer takes {name!r}"
+
+ def test_the_calls_behind_the_tools_still_exist(self, real_modal):
+ import inspect
+
+ from modal.sandbox import _SandboxFilesystem
+
+ assert list(inspect.signature(_SandboxFilesystem.write_bytes).parameters)[1:3] == [
+ "data",
+ "remote_path",
+ ], "write_bytes takes data first; swapping them type-checks and corrupts every write"
+ for name in ("read_bytes", "list_files", "stat"):
+ assert hasattr(_SandboxFilesystem, name)
+ assert "timeout" in inspect.signature(real_modal.Sandbox.exec).parameters
+ assert hasattr(real_modal.Sandbox, "from_id")
+
+ def test_the_exception_hierarchy_the_classifier_relies_on_holds(self, real_modal):
+ exception = real_modal.exception
+ assert issubclass(exception.ConflictError, exception.InvalidError)
+ assert issubclass(exception.SandboxTimeoutError, exception.TimeoutError)
+ assert issubclass(exception.ExecTimeoutError, exception.TimeoutError)
+ assert not issubclass(exception.SandboxFilesystemNotFoundError, exception.NotFoundError), (
+ "a missing file must not be classified as a missing sandbox"
+ )
+ assert not issubclass(exception.PermissionDeniedError, exception.AuthError)
+
+
+class TestOptionalExtra:
+ """The module has to be importable without ``modal`` installed, not merely unusable."""
+
+ @staticmethod
+ def _without_modal(monkeypatch):
+ """Make ``import modal`` fail the way a missing extra does."""
+ for name in list(sys.modules):
+ if name == "modal" or name.startswith("modal."):
+ monkeypatch.delitem(sys.modules, name, raising=False)
+ monkeypatch.delitem(sys.modules, "airflow.providers.common.ai.sandbox.modal", raising=False)
+ real_import = builtins.__import__
+
+ def guarded(name, *args, **kwargs):
+ if name == "modal" or name.startswith("modal."):
+ raise ModuleNotFoundError(f"No module named {name!r}")
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "__import__", guarded)
+
+ def test_importing_the_module_directly_raises_the_optional_feature_error(self, monkeypatch):
+ # The provider verifier walks every submodule of the distribution and imports it
+ # directly, so this path is not reached through the package's __getattr__.
+ self._without_modal(monkeypatch)
+ with pytest.raises(AirflowOptionalProviderFeatureException):
+ importlib.import_module("airflow.providers.common.ai.sandbox.modal")
+ sys.modules.pop("airflow.providers.common.ai.sandbox.modal", None)
+
+ def test_package_attribute_raises_the_optional_feature_error(self, monkeypatch):
+ self._without_modal(monkeypatch)
+ with pytest.raises(AirflowOptionalProviderFeatureException):
+ getattr(sandbox_package, "ModalSandboxBackend")
+ sys.modules.pop("airflow.providers.common.ai.sandbox.modal", None)
+
+ def test_package_still_exports_everything_that_needs_no_extra(self, monkeypatch):
+ self._without_modal(monkeypatch)
+ assert sandbox_package.SandboxSpec is SandboxSpec
+ assert sandbox_package.SbxSandboxBackend.__name__ == "SbxSandboxBackend"
+
+ def test_unknown_attribute_still_raises_attribute_error(self):
+ with pytest.raises(AttributeError):
+ getattr(sandbox_package, "NoSuchBackend")
+
+
+class TestInit:
+ @pytest.mark.parametrize(
+ ("kwargs", "match"),
+ [
+ ({"image": ""}, "image"),
+ ({"app_name": ""}, "app_name"),
+ ({"sandbox_timeout": 0}, "sandbox_timeout"),
+ ({"sandbox_timeout": -5}, "sandbox_timeout"),
+ ({"idle_timeout": 0}, "idle_timeout"),
+ ({"idle_timeout": 7200}, "idle_timeout"),
+ ({"sandbox_timeout": 60}, "at least 120"),
+ ({"workdir": "relative/path"}, "absolute"),
+ ({"cpu": 0}, "cpu"),
+ ({"memory": -1}, "memory"),
+ ({"egress_enforcement": "nope"}, "egress_enforcement"),
+ ],
+ )
+ def test_rejects_invalid_configuration(self, backend_class, kwargs, match):
+ with pytest.raises(ValueError, match=match):
+ backend_class(**kwargs)
+
+ def test_defaults_leave_idle_reclamation_off(self, backend_class, fake):
+ """
+ One sandbox serves a whole run and nothing keeps it warm between tool calls.
+
+ Live against modal 1.5.5, a 100s gap with ``idle_timeout=60`` reclaimed the sandbox
+ and took the agent's files with it, so this is off unless a Dag author asks for it.
+ """
+ backend = backend_class()
+ _, sandbox = _created(backend, fake, SandboxSpec())
+
+ assert sandbox.create_kwargs["idle_timeout"] is None
+
+ def test_construction_touches_nothing_remote(self, backend_class, fake):
+ """Constructors run at Dag-parse time, so nothing may be looked up or authenticated."""
+ backend_class(image="python:3.13-slim", workdir="/srv")
+
+ assert fake.App.lookups == []
+ assert fake.Sandbox.created == []
+ assert fake.Image.registries == []
+
+
+class TestSpecEnforcement:
+ """A backend must refuse a restriction it cannot actually apply."""
+
+ def test_block_network_maps_exactly(self, backend, fake):
+ _, sandbox = _created(backend, fake, SandboxSpec())
+
+ assert sandbox.create_kwargs["block_network"] is True
+ assert "outbound_domain_allowlist" not in sandbox.create_kwargs
+
+ def test_open_network_sets_neither_flag(self, backend, fake):
+ _, sandbox = _created(backend, fake, SandboxSpec(block_network=False))
+
+ assert "block_network" not in sandbox.create_kwargs
+ assert "outbound_domain_allowlist" not in sandbox.create_kwargs
+
+ def test_allowlist_is_refused_by_default(self, backend, fake):
+ spec = SandboxSpec(block_network=True, allow_egress_to=["pypi.org"])
+
+ with pytest.raises(SandboxTerminalError, match="egress_enforcement='sni'"):
+ backend.create(spec=spec)
+ assert fake.Sandbox.created == [], "refused before anything was provisioned"
+
+ def test_allowlist_message_names_what_is_not_enforced(self, backend):
+ """The Dag author has to learn that DNS stays open, not just that this is approximate."""
+ with pytest.raises(SandboxTerminalError, match="DNS"):
+ backend.create(spec=SandboxSpec(block_network=True, allow_egress_to=["pypi.org"]))
+
+ def test_allowlist_under_sni_becomes_a_domain_allowlist(self, backend_class, fake):
+ backend = backend_class(egress_enforcement="sni")
+
+ _, sandbox = _created(
+ backend, fake, SandboxSpec(block_network=True, allow_egress_to=["a.example", "b.example"])
+ )
+
+ assert sandbox.create_kwargs["outbound_domain_allowlist"] == ["a.example", "b.example"]
+ # Modal rejects the two together, and an allowlist alone already denies the rest.
+ assert "block_network" not in sandbox.create_kwargs
+
+ def test_allowlist_without_block_network_is_contradictory(self, backend_class, fake):
+ backend = backend_class(egress_enforcement="sni")
+
+ with pytest.raises(SandboxTerminalError, match="block_network"):
+ backend.create(spec=SandboxSpec(block_network=False, allow_egress_to=["pypi.org"]))
+ assert fake.Sandbox.created == []
+
+ def test_empty_allowlist_with_block_network_means_no_egress(self, backend, fake):
+ """An empty allowlist is 'no egress at all' per the contract, not an allowlist."""
+ _, sandbox = _created(backend, fake, SandboxSpec(block_network=True, allow_egress_to=[]))
+
+ assert sandbox.create_kwargs["block_network"] is True
+
+ @pytest.mark.parametrize(
+ "host",
+ [
+ "*",
+ "https://pypi.org",
+ "pypi.org:443",
+ "pypi.org/simple",
+ "pypi .org",
+ "",
+ "*.*.pypi.org",
+ ],
+ )
+ def test_refuses_an_allowlist_entry_that_is_not_a_hostname(self, backend_class, fake, host):
+ """
+ Modal sends these strings on unvalidated and matches them against the TLS hostname.
+
+ A URL or a ``host:port`` would match nothing and a bare ``*`` would match
+ everything, either way telling the author egress is restricted when it is not
+ restricted the way they wrote it.
+ """
+ backend = backend_class(egress_enforcement="sni")
+
+ with pytest.raises(SandboxTerminalError, match="hostnames"):
+ backend.create(spec=SandboxSpec(block_network=True, allow_egress_to=[host]))
+ assert fake.Sandbox.created == []
+
+ @pytest.mark.parametrize("host", ["10.0.0.5", "169.254.169.254", "443", "::1"])
+ def test_refuses_an_address_where_a_name_belongs(self, backend_class, fake, host):
+ """A client sends no TLS hostname for an address, so allowlisting one allows nothing."""
+ backend = backend_class(egress_enforcement="sni")
+
+ with pytest.raises(SandboxTerminalError, match="hostnames"):
+ backend.create(spec=SandboxSpec(block_network=True, allow_egress_to=[host]))
+
+ @pytest.mark.parametrize("host", ["localhost", "internal", "*.com"])
+ def test_refuses_a_single_label_name(self, backend_class, fake, host):
+ """
+ A single label cannot be the name a TLS handshake presents to a public endpoint.
+
+ The sandbox has no internal resolver either, so an entry like ``localhost`` reads
+ as a restriction while allowing nothing at all. A wildcard is judged on what it
+ qualifies, so ``*.com`` is refused for the same reason ``com`` is.
+ """
+ backend = backend_class(egress_enforcement="sni")
+
+ with pytest.raises(SandboxTerminalError, match="hostnames"):
+ backend.create(spec=SandboxSpec(block_network=True, allow_egress_to=[host]))
+ assert fake.Sandbox.created == []
+
+ def test_refuses_a_bare_string_instead_of_a_list(self, backend_class, fake):
+ """
+ ``str`` satisfies ``Sequence[str]``, so this would be read one character at a time.
+
+ A name without dots would then pass every character check and allow nothing, which
+ is the silent mismatch this validation exists to prevent.
+ """
+ backend = backend_class(egress_enforcement="sni")
+
+ with pytest.raises(SandboxTerminalError, match="not one string"):
+ backend.create(spec=SandboxSpec(block_network=True, allow_egress_to="pypiorg"))
+ assert fake.Sandbox.created == []
+
+ @pytest.mark.parametrize("host", ["pypi.org", "*.pythonhosted.org", "a-b.example.co.uk"])
+ def test_accepts_hostnames_and_wildcard_labels(self, backend_class, fake, host):
+ backend = backend_class(egress_enforcement="sni")
+
+ _, sandbox = _created(backend, fake, SandboxSpec(block_network=True, allow_egress_to=[host]))
+
+ assert sandbox.create_kwargs["outbound_domain_allowlist"] == [host]
+
+ def test_no_spec_states_no_requirements(self, backend, fake):
+ _, sandbox = _created(backend, fake, None)
+
+ assert "block_network" not in sandbox.create_kwargs
+ assert sandbox.create_kwargs["env"] is None
+
+
+class TestEnvironment:
+ def test_env_reaches_the_sandbox(self, backend, fake):
+ _, sandbox = _created(backend, fake, SandboxSpec(env={"HF_TOKEN": "secret"}))
+
+ assert sandbox.create_kwargs["env"] == {"HF_TOKEN": "secret"}
+
+ def test_no_env_injects_nothing(self, backend, fake):
+ _, sandbox = _created(backend, fake, SandboxSpec())
+
+ assert sandbox.create_kwargs["env"] is None
+
+ @pytest.mark.parametrize("env", [{"PORT": 8080}, {"FLAG": True}, {"NOTHING": None}, {5: "x"}])
+ def test_refuses_a_non_string_env_entry(self, backend, fake, env):
+ """
+ ``SandboxSpec.env`` is typed as strings and nothing enforces it.
+
+ Refused here, with the key named, rather than failing deeper in Modal's own
+ serialization where the message would not say which entry was wrong.
+ """
+ with pytest.raises(SandboxTerminalError, match="strings"):
+ backend.create(spec=SandboxSpec(env=env)) # type: ignore[arg-type]
+ assert fake.Sandbox.created == [], "refused before anything was provisioned"
+
+
+class TestCreate:
+ def test_passes_configuration_through(self, backend_class, fake):
+ backend = backend_class(
+ image="python:3.13-slim",
+ app_name="my-app",
+ sandbox_timeout=900,
+ idle_timeout=120,
+ workdir="/srv",
+ cpu=2,
+ memory=4096,
+ gpu="A10G",
+ region="us-east",
+ cloud="aws",
+ tags={"dag_id": "my_dag"},
+ )
+
+ handle, sandbox = _created(backend, fake, SandboxSpec(env={"TOKEN": "x"}))
+
+ assert fake.App.lookups == [("my-app", True)]
+ assert fake.Image.registries == ["python:3.13-slim"]
+ assert sandbox.create_kwargs["timeout"] == 900
+ assert sandbox.create_kwargs["idle_timeout"] == 120
+ assert sandbox.create_kwargs["workdir"] == "/srv"
+ assert sandbox.create_kwargs["cpu"] == 2
+ assert sandbox.create_kwargs["memory"] == 4096
+ assert sandbox.create_kwargs["gpu"] == "A10G"
+ assert sandbox.create_kwargs["region"] == "us-east"
+ assert sandbox.create_kwargs["cloud"] == "aws"
+ assert sandbox.create_kwargs["env"] == {"TOKEN": "x"}
+ assert handle == sandbox.object_id
+
+ def test_accepts_a_prepared_image_as_well_as_a_registry_tag(self, backend_class, fake):
+ """
+ An image carrying the packages an agent needs is the alternative to opening egress.
+
+ A default sandbox has the standard library and no DNS, so installing a package at
+ run time cannot work; baking it into the image means it does not have to.
+ """
+ prepared = object()
+ backend = backend_class(image=prepared)
+
+ _, sandbox = _created(backend, fake, SandboxSpec())
+
+ assert sandbox.create_kwargs["image"] is prepared
+ assert fake.Image.registries == [], "a prepared image is not looked up as a tag"
+
+ def test_names_and_tags_the_sandbox_for_correlation(self, backend, fake):
+ _, sandbox = _created(backend, fake, SandboxSpec())
+
+ name = sandbox.create_kwargs["name"]
+ assert name.startswith("airflow-sandbox-")
+ assert sandbox.create_kwargs["tags"]["airflow_sandbox"] == name
+
+ def test_caller_tags_are_kept(self, backend_class, fake):
+ backend = backend_class(tags={"dag_id": "d", "task_id": "t"})
+
+ _, sandbox = _created(backend, fake, SandboxSpec())
+
+ assert sandbox.create_kwargs["tags"]["dag_id"] == "d"
+ assert sandbox.create_kwargs["tags"]["task_id"] == "t"
+
+ def test_app_is_looked_up_per_create_so_concurrent_runs_cannot_race(self, backend, fake):
+ """One backend instance is shared by concurrent runs, so no lookup result is memoized."""
+ handles: list[str] = []
+ threads = [threading.Thread(target=lambda: handles.append(backend.create())) for _ in range(4)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(timeout=10)
+
+ assert len(set(handles)) == 4, "each run gets its own sandbox"
+ assert len(fake.App.lookups) == 4
+ for handle in handles:
+ assert backend._sandboxes[handle].object_id == handle
+
+ def test_credential_failure_is_terminal(self, backend, fake):
+ fake.Sandbox.create_error = fake.exception.AuthError("bad token")
+
+ with pytest.raises(SandboxTerminalError, match="credentials"):
+ backend.create(spec=SandboxSpec())
+
+ @pytest.mark.parametrize("error_name", ["ConnectionError", "Error", "AuthError", "NotFoundError"])
+ def test_every_create_failure_is_terminal(self, backend, fake, error_name):
+ """
+ Nothing raised from create can reach the model, so nothing may be labelled as if it could.
+
+ The toolset provisions the sandbox outside the block that turns a recoverable error
+ into a ModelRetry (``toolsets/sandbox.py``), and no prompt fixes a bad image or an
+ unreachable control plane anyway. Airflow's own retry is the right handler.
+ """
+ fake.Sandbox.create_error = getattr(fake.exception, error_name)("nope")
+
+ with pytest.raises(SandboxTerminalError):
+ backend.create(spec=SandboxSpec())
+
+ def test_a_missing_app_is_not_reported_as_a_dead_sandbox(self, backend, fake):
+ fake.Sandbox.create_error = fake.exception.NotFoundError("no such image")
+
+ with pytest.raises(SandboxTerminalError, match="could not find"):
+ backend.create(spec=SandboxSpec())
+
+
+class TestRunCommand:
+ def test_runs_through_a_shell_and_returns_both_streams(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(returncode=0, stdout=[b"out\n"], stderr=[b"err\n"])
+
+ result = backend.run_command(handle, "echo out", timeout=30, max_output_bytes=1024)
+
+ assert (result.exit_code, result.stdout, result.stderr) == (0, "out\n", "err\n")
+ assert result.timed_out is False
+ exec_call = next(call for call in sandbox.calls if call[0] == "exec")
+ assert exec_call[1] == ("sh", "-c", "echo out")
+ assert exec_call[2]["timeout"] == 30
+ assert exec_call[2]["text"] is False, "bytes, so the cap is applied in bytes"
+
+ @pytest.mark.parametrize(
+ ("timeout", "expected"),
+ [(0.4, 1), (1.0, 1), (1.2, 2), (29.9, 30)],
+ )
+ def test_rounds_the_deadline_up_to_whole_seconds(self, backend, fake, timeout, expected):
+ """Modal takes integer seconds and reads 0 as 'no timeout', so rounding down would unbound it."""
+ handle, sandbox = _created(backend, fake)
+
+ backend.run_command(handle, "true", timeout=timeout, max_output_bytes=1024)
+
+ assert next(c for c in sandbox.calls if c[0] == "exec")[2]["timeout"] == expected
+
+ def test_deadline_is_reported_as_a_timeout_and_keeps_the_sandbox(self, backend, fake):
+ """Modal signals a deadline with returncode -1, not an exception, and does not tear down."""
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(returncode=-1)
+
+ result = backend.run_command(handle, "sleep 30", timeout=2, max_output_bytes=1024)
+
+ assert result.timed_out is True
+ assert result.sandbox_terminated is False, "files from earlier calls are still there"
+
+ def test_sigkill_well_inside_the_budget_is_not_a_timeout(self, backend, fake):
+ """A guest killing itself lands nowhere near its deadline: measured at 0.04s of 30s."""
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(returncode=137, stderr=[b"Killed\n"])
+
+ result = backend.run_command(handle, "hog", timeout=30, max_output_bytes=1024)
+
+ assert (result.exit_code, result.timed_out) == (137, False)
+
+ def test_a_deadline_reported_as_sigkill_is_still_a_timeout(self, backend, fake, monkeypatch):
+ """
+ Modal does not always report a deadline the same way.
+
+ The same ``sleep 30`` at a two second budget has been seen returning 137 instead of
+ -1, at the same elapsed time and from the same SDK version, in about a quarter of
+ runs in some environments. Reading the status alone would hand the model a bare
+ ``[exit code: 137]`` for a command that simply ran long, so elapsed time decides.
+ """
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(returncode=137)
+ ticking = [0.0]
+
+ def advancing_clock() -> float:
+ ticking[0] += 1.5
+ return ticking[0]
+
+ monkeypatch.setattr("airflow.providers.common.ai.sandbox.modal.time.monotonic", advancing_clock)
+
+ result = backend.run_command(handle, "sleep 30", timeout=1, max_output_bytes=1024)
+
+ assert result.timed_out is True
+ assert result.exit_code == 137
+
+ def test_exec_timeout_exception_is_still_handled(self, backend, fake):
+ """Not raised by modal 1.5.5, whose own comment says it eventually should be."""
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(wait_error=fake.exception.ExecTimeoutError("deadline"))
+
+ result = backend.run_command(handle, "sleep 30", timeout=2, max_output_bytes=1024)
+
+ assert result.timed_out is True
+
+ def test_nonzero_exit_is_output_not_an_error(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(returncode=3, stderr=[b"boom\n"])
+
+ result = backend.run_command(handle, "false", timeout=30, max_output_bytes=1024)
+
+ assert (result.exit_code, result.stderr) == (3, "boom\n")
+
+ def test_keeps_the_tail_when_output_exceeds_the_cap(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ chunks = [f"line-{i}\n".encode() for i in range(1, 1001)]
+ sandbox.process = FakeProcess(stdout=chunks)
+
+ result = backend.run_command(handle, "spew", timeout=30, max_output_bytes=40)
+
+ assert result.stdout_truncated is True
+ assert len(result.stdout.encode()) <= 40
+ assert result.stdout.strip().endswith("line-1000"), "the tail carries the error and the status"
+ assert "line-1\n" not in result.stdout
+
+ def test_caps_a_single_chunk_larger_than_the_whole_budget(self, backend, fake):
+ """One transport chunk can exceed the cap on its own, and must still be cut."""
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(stdout=[b"x" * 5000 + b"\ntail\n"])
+
+ result = backend.run_command(handle, "spew", timeout=30, max_output_bytes=64)
+
+ assert result.stdout_truncated is True
+ assert len(result.stdout.encode()) <= 64
+ assert result.stdout.strip().endswith("tail")
+
+ @pytest.mark.parametrize(
+ "chunk",
+ [
+ b"x" * 60000 + b"\n",
+ b"x" * 60000,
+ b'{"one":"very long line"}' * 5000 + b"\n",
+ ],
+ ids=["long_line_with_newline", "long_line_without_newline", "json_one_liner"],
+ )
+ def test_one_line_longer_than_the_budget_still_reaches_the_model(self, backend, fake, chunk):
+ """
+ Dropping the leading partial line must not empty the window.
+
+ A single line longer than the cap has its newline at the very end, so dropping
+ through it leaves nothing, and the toolset only labels a non-empty stream: the
+ model would be told "(no output)" for a command that produced megabytes. Hits
+ ``cat one_line.json``, ``base64 -w0``, minified JS, single-line CSV.
+ """
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(stdout=[chunk])
+
+ result = backend.run_command(handle, "cat big.json", timeout=30, max_output_bytes=51200)
+
+ assert result.stdout != ""
+ assert result.stdout_truncated is True
+ assert len(result.stdout.encode()) <= 51200
+ assert result.stdout.strip().endswith(chunk.strip()[-20:].decode())
+
+ def test_a_long_line_followed_by_a_short_one_keeps_the_window(self, backend, fake):
+ """
+ Dropping the partial head must not cost most of the budget either.
+
+ A 200 KB line then a short one puts the window's only newline near its end, so
+ dropping through it would hand the model 18 bytes of a 50 KiB budget. The
+ formatter already marks output as cut, so a partial first line misleads nobody.
+ """
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(stdout=[b"L" * 200_000 + b"\nshort tail\n"])
+
+ result = backend.run_command(handle, "spew", timeout=30, max_output_bytes=51200)
+
+ assert len(result.stdout.encode()) > 51200 // 2
+ assert result.stdout.strip().endswith("short tail")
+ assert "L" in result.stdout, "the long line's tail survives rather than being dropped"
+
+ def test_drops_a_leading_partial_line(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(stdout=[b"aaaaaaaaaa\nbbbb\ncccc\n"])
+
+ result = backend.run_command(handle, "spew", timeout=30, max_output_bytes=12)
+
+ assert not result.stdout.startswith("a"), "no fragment presented as a whole record"
+
+ def test_streams_are_capped_independently(self, backend, fake):
+ """A large stderr must not crowd stdout out of the result."""
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(stdout=[b"keep-me\n"], stderr=[b"e" * 4000 + b"\n"])
+
+ result = backend.run_command(handle, "spew", timeout=30, max_output_bytes=32)
+
+ assert result.stdout == "keep-me\n"
+ assert result.stdout_truncated is False
+ assert result.stderr_truncated is True
+
+ def test_invalid_utf8_is_replaced_not_raised(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(stdout=[b"\xff\xfe binary"])
+
+ result = backend.run_command(handle, "cat /bin/sh", timeout=30, max_output_bytes=1024)
+
+ assert "binary" in result.stdout
+
+ def test_stream_dying_mid_read_marks_truncation_and_still_reports(self, backend, fake):
+ """The exit status decides what happened; a severed stream only costs output."""
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(
+ returncode=0, stdout=[b"partial\n"], stream_error=fake.exception.ConnectionError("gone")
+ )
+
+ result = backend.run_command(handle, "spew", timeout=30, max_output_bytes=1024)
+
+ assert result.stdout == "partial\n"
+ assert result.stdout_truncated is True
+
+ def test_refuses_a_command_that_cannot_fit_the_sandbox_lifetime(self, backend, fake):
+ handle, _ = _created(backend, fake)
+
+ with pytest.raises(SandboxError, match="lifetime") as caught:
+ backend.run_command(handle, "sleep 9999", timeout=9999, max_output_bytes=1024)
+ assert not isinstance(caught.value, SandboxTerminalError), "the model can ask for less"
+
+ def test_shortens_a_command_to_the_life_the_sandbox_has_left(self, backend_class, fake, monkeypatch):
+ """
+ A command that fits the configured lifetime can still outlive an old sandbox.
+
+ Shortened rather than refused, because refusing is unactionable for the inherited
+ file operations: they run on a fixed internal budget that neither the model nor the
+ Dag author can lower, so once a sandbox has less of that budget left than the budget
+ itself -- which happens to every sandbox eventually -- ``read_file`` would fail for
+ the rest of its life with advice nobody could follow.
+ """
+ backend = backend_class(sandbox_timeout=600)
+ clock = [1000.0]
+ monkeypatch.setattr("airflow.providers.common.ai.sandbox.modal.time.monotonic", lambda: clock[0])
+ handle, sandbox = _created(backend, fake)
+ clock[0] += 500 # 100s of life left
+
+ backend.run_command(handle, "sleep 200", timeout=200, max_output_bytes=1024)
+
+ deadline = next(c for c in sandbox.calls if c[0] == "exec")[2]["timeout"]
+ assert deadline == 100, "shortened to what is left rather than refused"
+
+ def test_reports_the_deadline_it_actually_applied(self, backend_class, fake, monkeypatch):
+ """
+ A shortened command that times out must not be reported with the number it asked for.
+
+ Otherwise the model is told it had 200s, concludes the work needs longer, and comes
+ back asking for more -- when the real constraint was never its request.
+ """
+ backend = backend_class(sandbox_timeout=600)
+ clock = [1000.0]
+ monkeypatch.setattr("airflow.providers.common.ai.sandbox.modal.time.monotonic", lambda: clock[0])
+ handle, sandbox = _created(backend, fake)
+ clock[0] += 500 # 100s of life left
+ sandbox.process = FakeProcess(returncode=-1)
+
+ result = backend.run_command(handle, "sleep 200", timeout=200, max_output_bytes=1024)
+
+ assert result.timed_out is True
+ assert result.applied_timeout == 100
+
+ def test_the_inherited_file_budget_survives_a_short_lived_sandbox(self, backend_class, fake, monkeypatch):
+ """
+ The case that made refusing wrong: ``read_file`` runs a fixed 120s helper.
+
+ Measured live, a sandbox created at the 120s floor already reports 119s left by the
+ time the guard sees it, because provisioning takes a second or two, so a refusal
+ made the tool impossible for that sandbox's entire life.
+ """
+ backend = backend_class(sandbox_timeout=120)
+ clock = [1000.0]
+ monkeypatch.setattr("airflow.providers.common.ai.sandbox.modal.time.monotonic", lambda: clock[0])
+ handle, sandbox = _created(backend, fake)
+ clock[0] += 2 # provisioning ate two seconds
+ sandbox.process = FakeProcess(stdout=[b"7\ncGF5bG9hZA==\n"])
+
+ assert backend.read_file(handle, "/workspace/out.txt", max_bytes=1024) == b"payload"
+
+ def test_says_nothing_about_the_lifetime_of_a_sandbox_it_did_not_create(self, backend, fake):
+ """A handle from elsewhere has a lifetime this backend cannot know, so it invents none."""
+ handle, sandbox = _created(backend, fake)
+ other = type(backend)(sandbox_timeout=600)
+
+ assert other.run_command(handle, "true", timeout=590, max_output_bytes=1024).exit_code == 0
+
+ @pytest.mark.parametrize(
+ ("returncode", "probe_error", "expect_terminated"),
+ [
+ (137, None, False),
+ (137, "NotFoundError", True),
+ (128, "NotFoundError", True),
+ (-1, None, False),
+ (-1, "ConflictError", True),
+ (3, "NotFoundError", False),
+ ],
+ ids=[
+ "guest_sigkill_in_a_live_sandbox",
+ "sandbox_terminated_under_the_command",
+ "sandbox_destroyed_under_the_command",
+ "plain_deadline",
+ "sandbox_expired_at_its_own_timeout",
+ "ordinary_failure_asks_nothing",
+ ],
+ )
+ def test_detects_a_sandbox_that_died_under_the_command(
+ self, backend, fake, returncode, probe_error, expect_terminated
+ ):
+ """
+ A dead sandbox reports like an ordinary failure, so a signal exit or a deadline is
+ followed by the cheapest possible command to find out which it was.
+
+ Live against modal 1.5.5: a sandbox terminated mid-command returns 128 with its
+ plumbing text on stderr, one that hits its own ``sandbox_timeout`` returns -1, and
+ in both cases ``poll()`` still says None while an ``exec`` raises at once. An
+ ordinary non-zero exit asks nothing, so the common path costs no round trip.
+ """
+ handle, sandbox = _created(backend, fake)
+ sandbox.processes = [FakeProcess(returncode=returncode)]
+ # The command runs; the liveness probe after it is what fails.
+ sandbox.exec_errors = [None, getattr(fake.exception, probe_error)("gone") if probe_error else None]
+
+ result = backend.run_command(handle, "work", timeout=30, max_output_bytes=1024)
+
+ assert result.sandbox_terminated is expect_terminated
+ assert result.exit_code == returncode
+ if expect_terminated:
+ assert handle not in backend._sandboxes, "a dead handle is not kept"
+
+ @pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan")])
+ def test_rejects_an_impossible_timeout(self, backend, fake, timeout):
+ handle, _ = _created(backend, fake)
+
+ with pytest.raises(ValueError, match="timeout"):
+ backend.run_command(handle, "true", timeout=timeout, max_output_bytes=1024)
+
+ @pytest.mark.parametrize(
+ ("error_name", "terminal"),
+ [
+ ("AuthError", True),
+ ("NotFoundError", True),
+ ("SandboxTerminatedError", True),
+ ("SandboxTimeoutError", True),
+ ("InvalidError", True),
+ ("ConflictError", True),
+ ("SandboxFilesystemNotFoundError", False),
+ ("SandboxFilesystemPermissionError", False),
+ ("ConnectionError", False),
+ ("Error", False),
+ ],
+ )
+ def test_classifies_failures(self, backend, fake, error_name, terminal):
+ """
+ Terminal fails the task; recoverable reaches the model as a retry.
+
+ ``ConflictError`` derives from ``InvalidError`` and ``SandboxTimeoutError`` shares a
+ parent with ``ExecTimeoutError``, so this is where the ordering of those checks is
+ pinned down.
+ """
+ handle, sandbox = _created(backend, fake)
+ sandbox.exec_error = getattr(fake.exception, error_name)("failed")
+
+ with pytest.raises(SandboxError) as caught:
+ backend.run_command(handle, "true", timeout=30, max_output_bytes=1024)
+ assert isinstance(caught.value, SandboxTerminalError) is terminal
+
+ def test_a_shutting_down_sandbox_says_so(self, backend, fake):
+ """
+ ``ConflictError`` is a subclass of ``InvalidError`` and means something else.
+
+ Observed live as "Modal Sandbox is shutting down" on a sandbox reaching the end of
+ its life, which is not the request being wrong.
+ """
+ handle, sandbox = _created(backend, fake)
+ sandbox.exec_error = fake.exception.ConflictError("Modal Sandbox is shutting down.")
+
+ with pytest.raises(SandboxTerminalError, match="shutting down"):
+ backend.run_command(handle, "true", timeout=30, max_output_bytes=1024)
+
+ def test_ambiguous_failure_on_a_dead_sandbox_is_terminal(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ sandbox.exec_error = fake.exception.ConnectionError("unreachable")
+ sandbox.poll_result = 137
+
+ with pytest.raises(SandboxTerminalError, match="no longer running"):
+ backend.run_command(handle, "true", timeout=30, max_output_bytes=1024)
+
+ def test_looks_up_a_handle_it_did_not_create(self, backend, backend_class, fake):
+ """A handle can outlive the instance that made it, so the cache cannot be the only path."""
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(stdout=[b"ok\n"])
+ other = backend_class()
+
+ assert other.run_command(handle, "true", timeout=30, max_output_bytes=1024).stdout == "ok\n"
+
+ def test_unknown_handle_is_terminal(self, backend, fake):
+ with pytest.raises(SandboxTerminalError, match="gone"):
+ backend.run_command("sb-does-not-exist", "true", timeout=30, max_output_bytes=1024)
+
+
+class TestFileOperations:
+ def test_write_file_uses_the_native_api(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+
+ backend.write_file(handle, "/workspace/out.txt", b"payload")
+
+ assert ("write_bytes", "/workspace/out.txt", b"payload") in sandbox.calls
+ assert not any(call[0] == "exec" for call in sandbox.calls), "no shell round trip"
+
+ @pytest.mark.parametrize(
+ ("given", "expected"),
+ [
+ ("out.txt", "/workspace/out.txt"),
+ ("./out.txt", "/workspace/out.txt"),
+ ("nested/out.txt", "/workspace/nested/out.txt"),
+ ("/abs/out.txt", "/abs/out.txt"),
+ ("/abs/../out.txt", "/abs/../out.txt"),
+ ("//workspace//nested///out.txt", "/workspace/nested/out.txt"),
+ ],
+ )
+ def test_resolves_paths_against_the_working_directory(self, backend_class, fake, given, expected):
+ """Modal's filesystem API takes absolute paths only, and must agree with the shell's cwd."""
+ backend = backend_class(workdir="/workspace")
+ handle, sandbox = _created(backend, fake)
+
+ backend.write_file(handle, given, b"x")
+
+ assert ("write_bytes", expected, b"x") in sandbox.calls
+
+ def test_asks_the_sandbox_for_its_workdir_once_when_unset(self, backend_class, fake):
+ backend = backend_class(workdir=None)
+ handle, sandbox = _created(backend, fake)
+ sandbox.processes = [FakeProcess(stdout=[b"/srv\n"])]
+
+ backend.write_file(handle, "a.txt", b"x")
+ backend.write_file(handle, "b.txt", b"y")
+
+ assert ("write_bytes", "/srv/a.txt", b"x") in sandbox.calls
+ assert ("write_bytes", "/srv/b.txt", b"y") in sandbox.calls
+ assert len([call for call in sandbox.calls if call[0] == "exec"]) == 1, "cached after the first ask"
+
+ def test_unusable_workdir_answer_is_recoverable(self, backend_class, fake):
+ backend = backend_class(workdir=None)
+ handle, sandbox = _created(backend, fake)
+ sandbox.processes = [FakeProcess(returncode=1, stderr=[b"pwd: not found\n"])]
+
+ with pytest.raises(SandboxError, match="working directory"):
+ backend.write_file(handle, "a.txt", b"x")
+
+ def test_list_directory_marks_directories(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ sandbox.listing = [FileInfo("sub", kind="directory"), FileInfo("file.txt")]
+
+ assert backend.list_directory(handle, "/workspace") == [("sub", True), ("file.txt", False)]
+
+ def test_a_symlink_is_not_reported_as_a_directory(self, backend, fake):
+ """
+ Modal types a symlink as a symlink whatever it points at, and so does ``find -printf``.
+
+ Reporting one as a directory would be a guess about its target that neither backend
+ makes, so a symlinked directory lists without a trailing slash on both.
+ """
+ handle, sandbox = _created(backend, fake)
+ sandbox.listing = [FileInfo("link-to-dir", kind="symlink")]
+
+ assert backend.list_directory(handle, "/workspace") == [("link-to-dir", False)]
+
+ def test_list_directory_uses_the_native_api(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+
+ backend.list_directory(handle, "/workspace")
+
+ assert ("list_files", "/workspace") in sandbox.calls
+ assert not any(call[0] == "exec" for call in sandbox.calls)
+
+ def test_read_file_stays_on_the_shell_path(self, backend, fake):
+ """
+ Not overridden on purpose: Modal's ``read_bytes`` takes no length, so it cannot honour
+ ``max_bytes``. The inherited implementation caps inside the guest with ``head -c``.
+ """
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(stdout=[b"7\ncGF5bG9hZA==\n"])
+
+ assert backend.read_file(handle, "/workspace/out.txt", max_bytes=1024) == b"payload"
+ assert any(call[0] == "exec" for call in sandbox.calls)
+ assert not any(call[0] == "read_bytes" for call in sandbox.calls)
+
+ def test_missing_file_is_recoverable(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ sandbox.filesystem_error = fake.exception.SandboxFilesystemNotFoundError("no such path")
+
+ with pytest.raises(SandboxError) as caught:
+ backend.list_directory(handle, "/workspace/nope")
+ assert not isinstance(caught.value, SandboxTerminalError)
+
+ def test_dead_sandbox_during_a_file_operation_is_terminal(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ sandbox.filesystem_error = fake.exception.SandboxTerminatedError("gone")
+
+ with pytest.raises(SandboxTerminalError):
+ backend.write_file(handle, "/workspace/out.txt", b"x")
+
+
+class TestClampedExpiry:
+ """
+ A command clamped to the sandbox's remaining life, which then hits that deadline, has
+ stood on the expiry itself.
+
+ Modal reclaims a sandbox some seconds after its deadline passes, so a liveness probe
+ taken at that instant can answer "alive" for a sandbox that dies moments later.
+ Measured against a live account it did exactly that, four runs out of four: the model
+ was told only that its command was slow, and the next tool call then failed the task.
+ The clamp is better evidence than the probe because it cannot race.
+ """
+
+ def test_a_clamped_deadline_reports_the_sandbox_gone_without_probing(
+ self, backend_class, fake, monkeypatch
+ ):
+ backend = backend_class(sandbox_timeout=600)
+ clock = [1000.0]
+ monkeypatch.setattr("airflow.providers.common.ai.sandbox.modal.time.monotonic", lambda: clock[0])
+ handle, sandbox = _created(backend, fake)
+ clock[0] += 500 # 100s of life left, so a 200s command is clamped to it
+ sandbox.process = FakeProcess(returncode=-1)
+ # Make the probe lie the way the live one did, so the test fails if we consult it.
+ sandbox.exec_errors = []
+
+ result = backend.run_command(handle, "sleep 200", timeout=200, max_output_bytes=1024)
+
+ assert result.timed_out is True
+ assert result.applied_timeout == 100
+ assert result.sandbox_terminated is True, "an expiry the clamp stood on must not need a probe"
+
+ def test_an_unclamped_deadline_still_consults_the_sandbox(self, backend_class, fake, monkeypatch):
+ """A command that merely overran its own budget leaves a live sandbox alone."""
+ backend = backend_class(sandbox_timeout=3600)
+ clock = [1000.0]
+ monkeypatch.setattr("airflow.providers.common.ai.sandbox.modal.time.monotonic", lambda: clock[0])
+ handle, sandbox = _created(backend, fake)
+ sandbox.process = FakeProcess(returncode=-1)
+
+ result = backend.run_command(handle, "sleep 5", timeout=2, max_output_bytes=1024)
+
+ assert result.timed_out is True
+ assert result.sandbox_terminated is False
+
+
+class TestMalformedHelperReply:
+ """
+ Modal serves the native file operations from a helper binary it injects into the
+ guest, and the SDK parses that helper's output. Code running as root inside the
+ sandbox can replace the binary, and a skewed or broken helper misbehaves without
+ anyone's help. Either way the SDK raises out of ``json``, not out of
+ ``modal.exception``, so the backend has to translate it or the toolset's error
+ handling never sees it and the task dies with a traceback the model cannot act on.
+ """
+
+ @staticmethod
+ def _malformed() -> json.JSONDecodeError:
+ return json.JSONDecodeError("Expecting value", "not json", 0)
+
+ def test_write_file_translates_a_malformed_reply(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ sandbox.filesystem_error = self._malformed()
+
+ with pytest.raises(SandboxError) as caught:
+ backend.write_file(handle, "/workspace/out.txt", b"data")
+
+ # Recoverable, not terminal: the shell does not go through the helper.
+ assert not isinstance(caught.value, SandboxTerminalError)
+ assert "run_command" in str(caught.value)
+
+ def test_list_directory_translates_a_malformed_reply(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+ sandbox.filesystem_error = self._malformed()
+
+ with pytest.raises(SandboxError) as caught:
+ backend.list_directory(handle, "/workspace")
+
+ assert not isinstance(caught.value, SandboxTerminalError)
+ assert "run_command" in str(caught.value)
+
+ def test_a_vendor_error_is_still_mapped_normally(self, backend, fake):
+ # The new branch must not shadow the existing one.
+ handle, sandbox = _created(backend, fake)
+ sandbox.filesystem_error = fake.exception.SandboxFilesystemNotFoundError(
+ "path does not exist: /workspace/missing"
+ )
+
+ with pytest.raises(SandboxError) as caught:
+ backend.list_directory(handle, "/workspace/missing")
+
+ assert "run_command" not in str(caught.value)
+
+
+class TestDestroy:
+ def test_terminates_without_waiting(self, backend, fake):
+ """``terminate(wait=True)`` took 31s when measured, and this runs in every teardown."""
+ handle, sandbox = _created(backend, fake)
+
+ backend.destroy(handle)
+
+ assert ("terminate", False) in sandbox.calls
+ assert sandbox.terminated is True
+
+ def test_is_idempotent(self, backend, fake):
+ handle, sandbox = _created(backend, fake)
+
+ backend.destroy(handle)
+ backend.destroy(handle)
+
+ assert handle not in backend._sandboxes
+
+ def test_forgets_a_missing_sandbox_quietly(self, backend):
+ backend.destroy("sb-never-existed")
+
+ def test_survives_a_failed_termination(self, backend, fake, caplog):
+ """The model's work is already paid for, so a teardown blip must not fail the task."""
+ handle, sandbox = _created(backend, fake)
+ sandbox.terminate_error = fake.exception.ConnectionError("blip")
+
+ backend.destroy(handle)
+
+ assert "Could not terminate" in caplog.text
+
+ def test_can_be_called_while_a_command_is_outstanding(self, backend, fake):
+ """
+ A cancelled tool call leaves a thread inside a Modal call, and teardown runs anyway.
+
+ Threads are not preemptible, so the two overlap. Nothing may deadlock or raise.
+ """
+ handle, sandbox = _created(backend, fake)
+ released = threading.Event()
+ entered = threading.Event()
+
+ class BlockingStream:
+ def __iter__(self):
+ entered.set()
+ released.wait(timeout=5)
+ yield b"late\n"
+
+ process = FakeProcess(returncode=0)
+ process.stdout = BlockingStream()
+ sandbox.process = process
+ result: list = []
+ worker = threading.Thread(
+ target=lambda: result.append(
+ backend.run_command(handle, "slow", timeout=30, max_output_bytes=1024)
+ )
+ )
+ worker.start()
+ assert entered.wait(timeout=5)
+
+ backend.destroy(handle)
+ released.set()
+ worker.join(timeout=10)
+
+ assert not worker.is_alive()
+ assert sandbox.terminated is True
+ assert len(result) == 1
+ assert result[0].exit_code == 0
diff --git a/providers/common/ai/tests/unit/common/ai/toolsets/test_sandbox.py b/providers/common/ai/tests/unit/common/ai/toolsets/test_sandbox.py
index 9cd0b282c2c..e248047e5c6 100644
--- a/providers/common/ai/tests/unit/common/ai/toolsets/test_sandbox.py
+++ b/providers/common/ai/tests/unit/common/ai/toolsets/test_sandbox.py
@@ -170,6 +170,50 @@ class TestGetTools:
tools["run_command"].args_validator.validate_python(bad_args)
+class TestNetworkNote:
+ """
+ The model reads tool descriptions and nothing else, so the egress policy belongs
+ there. Measured without it: a ``pip install`` under the default deny costs a turn
+ and returns a DNS error, and under an allowlist a reach for plain HTTP is never
+ refused, so the model burns its whole command budget and reads a timeout, which
+ it takes to mean its command was slow.
+ """
+
+ @pytest.mark.asyncio
+ async def test_the_default_spec_tells_the_model_there_is_no_network(self):
+ tools = await SandboxToolset(_RecordingBackend()).get_tools(_ctx())
+
+ description = tools["run_command"].tool_def.description
+ assert "NO network access" in description
+ assert "DNS" in description
+
+ @pytest.mark.asyncio
+ async def test_an_allowlist_names_the_hosts_and_the_port(self):
+ toolset = SandboxToolset(
+ _RecordingBackend(),
+ spec=SandboxSpec(block_network=True, allow_egress_to=["pypi.org", "files.pythonhosted.org"]),
+ )
+
+ description = (await toolset.get_tools(_ctx()))["run_command"].tool_def.description
+ assert "pypi.org, files.pythonhosted.org" in description
+ # Plain HTTP is the trap worth naming, because nothing refuses it.
+ assert "plain HTTP" in description
+
+ @pytest.mark.asyncio
+ async def test_an_open_sandbox_says_so(self):
+ toolset = SandboxToolset(_RecordingBackend(), spec=SandboxSpec(block_network=False))
+
+ description = (await toolset.get_tools(_ctx()))["run_command"].tool_def.description
+ assert "has outbound network access" in description
+
+ @pytest.mark.asyncio
+ async def test_only_run_command_carries_the_note(self):
+ tools = await SandboxToolset(_RecordingBackend()).get_tools(_ctx())
+
+ for name in ("read_file", "write_file", "list_directory"):
+ assert "network" not in tools[name].tool_def.description
+
+
class TestRunCommand:
@pytest.mark.asyncio
async def test_labels_streams_and_reports_a_nonzero_exit(self):
@@ -251,6 +295,69 @@ class TestRunCommand:
assert [c[0] for c in backend.commands] == ["box-1", "box-2"]
+ @pytest.mark.asyncio
+ async def test_a_timeout_reports_the_budget_the_command_actually_had(self):
+ """
+ A backend may have to shorten a deadline, and then the request is the wrong number.
+
+ The Modal backend does this when a sandbox has less life left than the command asked
+ for. Reporting the request would send the model back asking for more time when the
+ constraint was never its request, and the sandbox may well have survived, so there is
+ no replacement note to explain the difference either.
+ """
+ backend = _RecordingBackend(
+ run_result=SandboxExecResult(
+ exit_code=-1, stdout="", stderr="", timed_out=True, applied_timeout=23
+ )
+ )
+ ts = SandboxToolset(backend, default_command_timeout=60, max_command_timeout=60)
+
+ async with ts:
+ result = await _call(ts, "run_command", {"command": "x"})
+
+ assert "[timed out after 23s]" in result
+ assert "60s" not in result
+
+ @pytest.mark.asyncio
+ async def test_a_timeout_falls_back_to_the_requested_budget(self):
+ """A backend that shortens nothing sets no applied_timeout, and `sbx` never does."""
+ backend = _RecordingBackend(
+ run_result=SandboxExecResult(exit_code=-1, stdout="", stderr="", timed_out=True)
+ )
+ ts = SandboxToolset(backend, default_command_timeout=5, max_command_timeout=5)
+
+ async with ts:
+ result = await _call(ts, "run_command", {"command": "x"})
+
+ assert "[timed out after 5s]" in result
+
+ @pytest.mark.asyncio
+ async def test_a_replaced_sandbox_is_announced_after_an_ordinary_failure_too(self):
+ """
+ A backend can lose the sandbox under a command that did not time out.
+
+ The Modal backend reports exactly that when a sandbox is terminated or reaches its
+ own lifetime mid-command: an exit status like any other, with the sandbox gone. The
+ model has to hear that its files went with it, or it will act on a filesystem that
+ no longer exists.
+ """
+ backend = _RecordingBackend(
+ run_result=SandboxExecResult(
+ exit_code=128,
+ stdout="starting\n",
+ stderr="waiting on pid 4: ... failed: EOF\n",
+ timed_out=False,
+ sandbox_terminated=True,
+ )
+ )
+ ts = SandboxToolset(backend)
+
+ async with ts:
+ result = await _call(ts, "run_command", {"command": "x"})
+
+ assert "[exit code: 128]" in result
+ assert "sandbox was replaced" in result
+
@pytest.mark.asyncio
async def test_backend_truncation_is_surfaced_to_the_model(self):
backend = _RecordingBackend(
diff --git a/uv.lock b/uv.lock
index 07a45a3b29f..88a620d2ded 100644
--- a/uv.lock
+++ b/uv.lock
@@ -4637,6 +4637,9 @@ llamaindex = [
mcp = [
{ name = "pydantic-ai-slim", extra = ["mcp"] },
]
+modal = [
+ { name = "modal" },
+]
openai = [
{ name = "pydantic-ai-slim", extra = ["openai"] },
]
@@ -4698,6 +4701,7 @@ requires-dist = [
{ name = "llama-index-core", marker = "extra == 'llamaindex'", specifier = ">=0.13.0" },
{ name = "llama-index-embeddings-openai", marker = "extra == 'llamaindex'", specifier = ">=0.6.0" },
{ name = "llama-index-llms-openai", marker = "extra == 'llamaindex'", specifier = ">=0.6.0" },
+ { name = "modal", marker = "extra == 'modal'", specifier = ">=1.5.0" },
{ name = "pyarrow", marker = "python_full_version >= '3.14' and extra == 'parquet'", specifier = ">=22.0.0" },
{ name = "pyarrow", marker = "python_full_version < '3.14' and extra == 'parquet'", specifier = ">=18.0.0" },
{ name = "pydantic-ai-harness", extras = ["codemode"], marker = "extra == 'code-mode'", specifier = ">=0.3.0" },
@@ -4714,7 +4718,7 @@ requires-dist = [
{ name = "sqlglot", marker = "extra == 'sql'", specifier = ">=30.0.0" },
{ name = "typesafe-sdk", marker = "extra == 'typesafe'", specifier = ">=0.6.0" },
]
-provides-extras = ["anthropic", "bedrock", "google", "openai", "typesafe", "mcp", "code-mode", "shields", "skills", "avro", "parquet", "sql", "common-sql", "langchain", "llamaindex", "pdf", "docx", "git"]
+provides-extras = ["anthropic", "bedrock", "google", "openai", "typesafe", "mcp", "modal", "code-mode", "shields", "skills", "avro", "parquet", "sql", "common-sql", "langchain", "llamaindex", "pdf", "docx", "git"]
[package.metadata.requires-dev]
dev = [
@@ -11037,6 +11041,62 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f5/4d/6b8460462012a52075ba97932197f8c141be71b8e5f7e918af08ee73424b/cattrs-26.2.0-py3-none-any.whl", hash = "sha256:c680f17cd1df3a1c038ad1db8d90a9c595568cc54c4e8098e7ceb8d7b9bd826b", size = 74830, upload-time = "2026-09-06T20:31:56.327Z" },
]
+[[package]]
+name = "cbor2"
+version = "6.1.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c6/14/b02446bacfe44351b1689c04937ade007588f44570431880a6937e525e6c/cbor2-6.1.4.tar.gz", hash = "sha256:01ecc79a28f33d17331943ce508fc1e21f4b06553c73f874f4c77120d72b2ef9", size = 90840, upload-time = "2026-08-01T20:41:39.797Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/66/7d/91e6b680e8ae313e53ae8da1d0b578c668d240c9f294be06d729fc014854/cbor2-6.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3cfb7eec57406b86aab47183f6dc90ece888e03721d37a4d33315e79e486898d", size = 415427, upload-time = "2026-08-01T20:40:24.371Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/43/3d8d53f22be9566cbb011bdaaf799a77f52e447b60a54efc9537f271f177/cbor2-6.1.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ae14eb3f2f251102d72625ab02a28c83a8d11adf3d0383bd6d7bd9d672e18119", size = 458280, upload-time = "2026-08-01T20:40:26.362Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/d8/0b2fb7e936b63395ab1acf91074dc0788c575a57e8d808d9e8489de631f4/cbor2-6.1.4-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:7b697c2a18eab9a3326447fe0f1c927db065a096febc57f064f0f5ed67fe96fc", size = 464737, upload-time = "2026-08-01T20:40:27.897Z" },
+ { url = "https://files.pythonhosted.org/packages/52/e1/f794155ec864c0e85a2df293adc589e7e45133ab15c0d21ac5ae3a51f804/cbor2-6.1.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:032ce71cbdc9267e9cec42132f6ac6c40f441ec27ebc87ca9dd209dcab6804ee", size = 524277, upload-time = "2026-08-01T20:40:29.475Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/7b/819912a5192ffdb0261b9c73e57b2ee77b4141132ffac77bf374c738e310/cbor2-6.1.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e8097e8259b78c8feb5dd5b833b23f83ed588e821ef44cf1455082cfc9434440", size = 533360, upload-time = "2026-08-01T20:40:30.922Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/f4/73aaf73837bae5c47f01e1a602214a9b9ece6c0687b7b1d6e4ecafe55b74/cbor2-6.1.4-cp310-cp310-win32.whl", hash = "sha256:41aae72e8863ac0d50705ea8290b72c49db90a6d953110feaa91360ae526fa8d", size = 282266, upload-time = "2026-08-01T20:40:32.576Z" },
+ { url = "https://files.pythonhosted.org/packages/3a/a9/a1a88346b44a7fbcd4b4aa4f527fbc5f46649bf0706e994d8e59aada4641/cbor2-6.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:deff027a56e04caebfee649029bfd1836f4fb468d2cf66d2a396ae1018f655db", size = 303781, upload-time = "2026-08-01T20:40:34.198Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/fb/969a945d3f3c0bd619d73e7eb375ef1800af33451d7ece2d56d881ef1732/cbor2-6.1.4-cp310-cp310-win_arm64.whl", hash = "sha256:c435720a25c1de9b241df883b5890b35aff3bf64365efc8037d4d26df085724e", size = 296316, upload-time = "2026-08-01T20:40:35.885Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/84/1e363301c06f509963d134f5479e82b3ade87fb1495ddacf9bf7ff24ac42/cbor2-6.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8156fdeb73c3ff6c8cf67ad414fb5c887cd708ff0af6d61f62629f41cb4c17b2", size = 414947, upload-time = "2026-08-01T20:40:37.405Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/96/d8e1ed3e79ea20a3423a96b5c89ce794fa02cb428e4429e601f8ebcbac7c/cbor2-6.1.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:e1fe2d62c50df290576280b18247ec63486f78be73e285bae269c2456c6ddff0", size = 457343, upload-time = "2026-08-01T20:40:38.868Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/0c/5796c2ed2dcd0696fc4abedf0ea0dfd5361b3f022a311481f977fa51b2b8/cbor2-6.1.4-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c204a75f91f8cd9ed0881f6b88ec395c59aeac9fcf4d08155e7f899db2a1c46e", size = 464314, upload-time = "2026-08-01T20:40:40.63Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/88/de524c6c2c91b740e5df6e6955a113fb616e979b26fd2e6a0693082d36e0/cbor2-6.1.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:28fa5db05a7eae8fd80709959988d8a7f12838c6d4e5c58ec951414058641195", size = 523053, upload-time = "2026-08-01T20:40:42.602Z" },
+ { url = "https://files.pythonhosted.org/packages/84/07/cb5fd92834633508d680a5b5695aeaf99d33ca0bdc5b844550d538f335b0/cbor2-6.1.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:316e217a496640418d3137483279d0e70053b000cdd4b52a4dbf20ea478bc40a", size = 532177, upload-time = "2026-08-01T20:40:44.058Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/19/be98721365edfe6fc23e6bcd1385afa0e960b247c5f0b50bb67f5d05e2d9/cbor2-6.1.4-cp311-cp311-win32.whl", hash = "sha256:4903f24e0f9087275a0b6606c8b0aa586277001d51e4844fcdbc5b7211330aa8", size = 281660, upload-time = "2026-08-01T20:40:45.761Z" },
+ { url = "https://files.pythonhosted.org/packages/16/23/d54f679d4b155918f5a0879dab78203ce4fd514d311b7cfeba27dafe480b/cbor2-6.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:5b99305d4013867e059f147752b95f728680682ab03d75a3f4dcfbb270d8dfe9", size = 303207, upload-time = "2026-08-01T20:40:47.293Z" },
+ { url = "https://files.pythonhosted.org/packages/53/3c/b3839d6213c88b249ba860525df05ff18b27bdc28ebc09cb1547790f001a/cbor2-6.1.4-cp311-cp311-win_arm64.whl", hash = "sha256:bd20ecc5c8ece24db952e48a91c8c47319eaa6358af707c85ac2bb388a79abc8", size = 296123, upload-time = "2026-08-01T20:40:48.808Z" },
+ { url = "https://files.pythonhosted.org/packages/2e/76/fb64293c19cafb860060310c57b768fd9cfb7cf592449660b756538cc116/cbor2-6.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1fc15061553e4494dc10883237501e3402c645fe509248dd698e1faf2460d68b", size = 404608, upload-time = "2026-08-01T20:40:50.219Z" },
+ { url = "https://files.pythonhosted.org/packages/96/ac/f58b3bafce7c86ada2ad8eaf189453136d2cf5bae526ea0540e1b9bc9d06/cbor2-6.1.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d9ada5a6ccfbb8ea7a3aa2aeb028421b52d8e0cd9323f0a2aeaa9c09d25fbce2", size = 449851, upload-time = "2026-08-01T20:40:51.725Z" },
+ { url = "https://files.pythonhosted.org/packages/f0/a5/10c6c126d59b07f2bd005094dd12a20afa46146f7e2673ed6f61a57641a7/cbor2-6.1.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:310f3dfb296ba48fe9b63c5cf26e691e3548a1eae6901d2f0c18e941d151f220", size = 461193, upload-time = "2026-08-01T20:40:53.446Z" },
+ { url = "https://files.pythonhosted.org/packages/15/e4/4445e6237088d1cca3b8536daeb90d6b4e23776de5609c9fa46773874757/cbor2-6.1.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e6c76004d674ad1c620660cb0bc5a8a0b72a5d8c7b70926d8e09e6d7e87332f", size = 516937, upload-time = "2026-08-01T20:40:54.952Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/87/9c0959510f7a402e5995c81ccfd82cb9f314140dc0cce88c12836e5b93f1/cbor2-6.1.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:32a4663425fbca4a4a7aa918eb5789d844c406439e58424cf34511f79f559242", size = 529229, upload-time = "2026-08-01T20:40:56.365Z" },
+ { url = "https://files.pythonhosted.org/packages/91/8e/6811e4ee84203ac657f6f461a37c7c9ba0287bde80eb83c7971e9b3fe156/cbor2-6.1.4-cp312-cp312-win32.whl", hash = "sha256:2310f07db3f9ba26f2a623774ff9f3dc7185af54f732ea119785a6b1bf7e1e7e", size = 278810, upload-time = "2026-08-01T20:40:57.76Z" },
+ { url = "https://files.pythonhosted.org/packages/da/27/87440788fc0d9513534c3c699238e2a9ca6010f8cb72e9c203b7af20a9f6/cbor2-6.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:cc8cd300e236e9797b2e1ce306109dc481fcccf78bfa2682bf36d99e6eab1ec6", size = 299971, upload-time = "2026-08-01T20:40:59.256Z" },
+ { url = "https://files.pythonhosted.org/packages/23/f9/77981e6e63092de19d7306a09a12b0eb3fd2907dc22c10dd5d389eb27faf/cbor2-6.1.4-cp312-cp312-win_arm64.whl", hash = "sha256:553a46bda7d09552631a714e22b91e6ff2c867ecd91511596ce290d8879b8d5b", size = 290662, upload-time = "2026-08-01T20:41:00.89Z" },
+ { url = "https://files.pythonhosted.org/packages/0d/17/0b20c88e76942ede86c98cdce138681690f95908c540c264fff847729cd4/cbor2-6.1.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c48a7c938fc5fa5300ff82b5df09068dcb4838685ae8556b5ee8279d74f97ab4", size = 403677, upload-time = "2026-08-01T20:41:02.561Z" },
+ { url = "https://files.pythonhosted.org/packages/35/3d/93eed770864540c5c9ea0841008208e9db686b7335f42520705b7d6dc6b2/cbor2-6.1.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:4bd29f21529e279d50fc14f1a811f7b05b4d8e66a7969163cce98983b6817245", size = 449762, upload-time = "2026-08-01T20:41:04.094Z" },
+ { url = "https://files.pythonhosted.org/packages/e3/21/69e4d37f00319b3d37322355aedc83154b4d8b75dc9e9789c06e1fbd8a92/cbor2-6.1.4-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:36ae16d64b1f7b620c1af748e7b6947e20069ef80eee56871c5fbb84cc635905", size = 460420, upload-time = "2026-08-01T20:41:05.891Z" },
+ { url = "https://files.pythonhosted.org/packages/be/26/2cfdd5ee826205a88a826bb38b7a572c676ec3efa29574be5cdbd04b4859/cbor2-6.1.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:69978901302ecbc8cda57b520487c5c5240ed217de783eb7728fceb258311d76", size = 516490, upload-time = "2026-08-01T20:41:07.52Z" },
+ { url = "https://files.pythonhosted.org/packages/82/86/d687cd1c2c9f9a986e8552ad1fdbd22411cc86389b5705dba6ec6f7e3226/cbor2-6.1.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad4efa23fee6447e56a269191044e06eb39e809458bcd674e164fe9445feafd0", size = 528810, upload-time = "2026-08-01T20:41:09.144Z" },
+ { url = "https://files.pythonhosted.org/packages/40/08/88cecf20b8825bdd991c47b317415c08ef9e7d5f05a1def9acd346edabde/cbor2-6.1.4-cp313-cp313-win32.whl", hash = "sha256:d2560c2ba6a95904ba2a0ca257af878c4344409d9b46d8e646d8ebb617b1e0dd", size = 278058, upload-time = "2026-08-01T20:41:10.48Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/67/ba140234a6415c16dcfbe0585ce12f905157b70e9cb1bb63a2b6d5721e70/cbor2-6.1.4-cp313-cp313-win_amd64.whl", hash = "sha256:c08b9c7d2ea013e24a0cb819b872b0119dde404f64a1182c0b24095b7bba781f", size = 299315, upload-time = "2026-08-01T20:41:12.067Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/7f/35d53ff4252a5a85656480d3a81d5a5af823979ccd0c5cac95196a7548a6/cbor2-6.1.4-cp313-cp313-win_arm64.whl", hash = "sha256:598710183daae69cbdeb177a870ec64aa601de8138a61491fd256826d15a860f", size = 289976, upload-time = "2026-08-01T20:41:13.63Z" },
+ { url = "https://files.pythonhosted.org/packages/05/5d/c5374c76471ab41dff4420a276569a56352e83166374fba6f40fd0bde7ad/cbor2-6.1.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:24da0a481294ac416e1e369e2d204b2b1d993cbd082d0d99fa3d6f5f27ae5e69", size = 407497, upload-time = "2026-08-01T20:41:15.189Z" },
+ { url = "https://files.pythonhosted.org/packages/46/f9/b9f12a5e24d5ae355e4c0f6d37330a2bbedad3331247a223a51c4cd39d5e/cbor2-6.1.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0859a0837e6e2d4fe5f5b849f6475797e4db545da98c19db4b1d3487bd47aa22", size = 452191, upload-time = "2026-08-01T20:41:16.705Z" },
+ { url = "https://files.pythonhosted.org/packages/67/22/8224b01f95a6fe07b1a64082aea34d9f49068392b3de93f5f3a10c73c62e/cbor2-6.1.4-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:c0f5f2d6d3b58e44146860c049f3c082207a4005588b8926d51bf937ab66773c", size = 462383, upload-time = "2026-08-01T20:41:18.17Z" },
+ { url = "https://files.pythonhosted.org/packages/92/52/437e4aa4f5df1fb41020d64b3d99a8239f0f99a3a75eb6ffa5cb66004b7f/cbor2-6.1.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:239db0f92d537fd29eaec4e40195fc3b2b48bc34a5887059658162489a9eb6ae", size = 518700, upload-time = "2026-08-01T20:41:19.592Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/45/2f5ea5bfe0fd800b3739c7df8679bdffa9f7def6b2f2fee064ada1c63e85/cbor2-6.1.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3f4a434c36bb0d33aeb48ddae8e8b673ca7e1f14545ee7cf4a4c7c39380ea9a2", size = 531243, upload-time = "2026-08-01T20:41:21.21Z" },
+ { url = "https://files.pythonhosted.org/packages/bd/c6/0beac64cb74cd3217f295f9bb0d64675e1809c683a31ea2a49ac9d4d1504/cbor2-6.1.4-cp314-cp314-win32.whl", hash = "sha256:6abcf072b8c0fdc8ad7902ee26a906cafbf3427d026b662ff21166a253f85e18", size = 285248, upload-time = "2026-08-01T20:41:22.658Z" },
+ { url = "https://files.pythonhosted.org/packages/bb/7d/4afa096ddc94049f5a514690891b02a18319e146ceb14465ce30c8340a8b/cbor2-6.1.4-cp314-cp314-win_amd64.whl", hash = "sha256:855764e02dc60ab9413acd044e997c3170000fdea6155d6c43a923a1d966dbe6", size = 313044, upload-time = "2026-08-01T20:41:24.066Z" },
+ { url = "https://files.pythonhosted.org/packages/e5/b5/e614cee861772f6b5c4d926b066d2e7dbc11e220b50ba716ba91e430fb0f/cbor2-6.1.4-cp314-cp314-win_arm64.whl", hash = "sha256:c6b28b928c5f2dbf47dffa12dce9c8e36fe6ac1c1358bc326499c0736263b66f", size = 304088, upload-time = "2026-08-01T20:41:25.431Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/41/3b28184154f6cbf7e47c1b7fb4a7a291c54f27a6f3a0a2f64b078c6a13e1/cbor2-6.1.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7336ff4cb7d161ec43b65eef43bf3e9bcab44bd152efb54dd637b7afe711254f", size = 401042, upload-time = "2026-08-01T20:41:26.819Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/1a/a8624023b84b41c43a150a89517c104aed0e467bd258866f13be4c3ac0c6/cbor2-6.1.4-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:8f1019494b0ec81a3df3ebb01b6acb446d5b946fe35845b1726379abd66a71da", size = 445301, upload-time = "2026-08-01T20:41:28.35Z" },
+ { url = "https://files.pythonhosted.org/packages/60/39/07dd0ea957c1f48673d3947f97ee36826efd4a824053dd0ec4df2f0c89d6/cbor2-6.1.4-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:179a794bf4be1d46ff190695929f65f0b42019c156919846ae539d2a7ec42e54", size = 459816, upload-time = "2026-08-01T20:41:29.839Z" },
+ { url = "https://files.pythonhosted.org/packages/23/8e/2015175132a27c1daed434f671ac6d9c1311461995df47f201307700e0da/cbor2-6.1.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9b904b8d0f4ddac9259197d21d121fae4cb8b555700d65bc12c5d46a2e6c2025", size = 511565, upload-time = "2026-08-01T20:41:31.939Z" },
+ { url = "https://files.pythonhosted.org/packages/82/66/420991095d9473614b205d4c4e40b5d3b9f1ee4410eb3c48c1e902947837/cbor2-6.1.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:71fcf4f237d68bf4445bf45070f36f82b333f2e6a62612aa2c256683b51378a9", size = 527709, upload-time = "2026-08-01T20:41:33.413Z" },
+ { url = "https://files.pythonhosted.org/packages/cc/7c/73057e7a38488a816a0d40ff9e7cd9f418800894582e2e48fb2f47ce66a2/cbor2-6.1.4-cp314-cp314t-win32.whl", hash = "sha256:7deccc50fd0b55c4c7dd265b144c5358a645121e457c0ae3722b5ad59832b257", size = 281462, upload-time = "2026-08-01T20:41:35.127Z" },
+ { url = "https://files.pythonhosted.org/packages/99/5d/d5db22837cb566de733b9d1c418cdf1912ccb1efc7b179e295430b1d81a2/cbor2-6.1.4-cp314-cp314t-win_amd64.whl", hash = "sha256:f3fc7d15cba4174373df2496070faa4a927fe3ed772130d281808120aec7b61c", size = 309165, upload-time = "2026-08-01T20:41:36.716Z" },
+ { url = "https://files.pythonhosted.org/packages/29/5f/ff2c6da83553a692219a0a62a21b57a27ded4405200e50db758a17fbaf15/cbor2-6.1.4-cp314-cp314t-win_arm64.whl", hash = "sha256:164ca22b509408435b2d8236c80c964e4fc77c085ab034569cd04c40d5cc8883", size = 298386, upload-time = "2026-08-01T20:41:38.392Z" },
+]
+
[[package]]
name = "celery"
version = "5.6.3"
@@ -14741,6 +14801,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cf/f2/bbeef86c687225b7bbc7c0acdfbd25c8bcaa3f5b1c941db053e5c3d9e859/grpcio_tools-1.81.1-cp314-cp314-win_amd64.whl", hash = "sha256:cb08172b7b629e75cb33866928d319a3196540a725eaab628ba721007140f1af", size = 1207490, upload-time = "2026-06-11T12:51:17.598Z" },
]
+[[package]]
+name = "grpclib"
+version = "0.4.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "h2" },
+ { name = "multidict" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5b/28/5a2c299ec82a876a252c5919aa895a6f1d1d35c96417c5ce4a4660dc3a80/grpclib-0.4.9.tar.gz", hash = "sha256:cc589c330fa81004c6400a52a566407574498cb5b055fa927013361e21466c46", size = 84798, upload-time = "2025-12-14T22:23:14.349Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5c/90/b0cbbd9efcc82816c58f31a34963071aa19fb792a212a5d9caf8e0fc3097/grpclib-0.4.9-py3-none-any.whl", hash = "sha256:7762ec1c8ed94dfad597475152dd35cbd11aecaaca2f243e29702435ca24cf0e", size = 77063, upload-time = "2025-12-14T22:23:13.224Z" },
+]
+
[[package]]
name = "gssapi"
version = "1.12.0"
@@ -17571,6 +17644,30 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a2/0a/033fef3d7a486aa369ec11cdf0455bbae6640df13491f6e7d35fd90aa000/mmh3-5.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:a8389010a2721e568fbe40ba94e7d8f931beabd59b95af2b2ab8238fd40a6b3e", size = 40275, upload-time = "2026-08-26T04:58:18.883Z" },
]
+[[package]]
+name = "modal"
+version = "1.5.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "aiohttp" },
+ { name = "cbor2" },
+ { name = "certifi" },
+ { name = "click" },
+ { name = "grpclib" },
+ { name = "protobuf" },
+ { name = "rich" },
+ { name = "synchronicity" },
+ { name = "toml" },
+ { name = "types-certifi" },
+ { name = "types-toml" },
+ { name = "typing-extensions" },
+ { name = "watchfiles" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b9/a5/9e322043716e511b7f6c9120804f33b92e21a818bf4709b2d98b3c76d665/modal-1.5.5.tar.gz", hash = "sha256:30df363ed1898cc3d91a09ff3f95c38ab043f6b6294011b01085312c6a0ac777", size = 870356, upload-time = "2026-08-28T19:51:34.881Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/04/19/b3dca8baec119126058b12ca620e44022e4d08a091956668bd04180c89a7/modal-1.5.5-py3-none-any.whl", hash = "sha256:8d10d3ee09818aaba1973b73ce2521ab8961b63a29b5b52e3ff0d25e7a74808e", size = 985163, upload-time = "2026-08-28T19:51:32.404Z" },
+]
+
[[package]]
name = "more-itertools"
version = "11.1.0"
@@ -24649,6 +24746,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" },
]
+[[package]]
+name = "synchronicity"
+version = "0.12.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5d/1c/f51dc54bbd302991026a53f9790735540e0e9e1184e9d5939f02446aa5bc/synchronicity-0.12.5.tar.gz", hash = "sha256:94d96b1d85698e3056b96a793b8c0949af6584e4a7d877fabdeb5385efe230aa", size = 60745, upload-time = "2026-06-18T21:06:23.545Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/06/74/ad9b99520f70c0bc3318e582e359d360cfc0f7afd7bf368a7f24013cece7/synchronicity-0.12.5-py3-none-any.whl", hash = "sha256:fdbbb10d437bc08a6b0f814fc66fddd1b58ffed314533d42f1ab555801e781af", size = 41107, upload-time = "2026-06-18T21:06:22.505Z" },
+]
+
[[package]]
name = "tableauserverclient"
version = "0.41"
@@ -25016,6 +25125,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d7/b0/dee84cb44175be1b4c35bd2f770727494e78f0bb38e571a623ade94dbebb/tokenizers-0.23.2-cp310-abi3-win_arm64.whl", hash = "sha256:e49c394456dd9985787fec76132438ba3fb8911f857b1bf3d40119f9292d41aa", size = 2729352, upload-time = "2026-09-03T08:55:44.345Z" },
]
+[[package]]
+name = "toml"
+version = "0.10.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" },
+]
+
[[package]]
name = "tomli"
version = "2.4.1"