You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2025/03/04 17:57:53 UTC
(airflow) 01/01: Start working on the docs for the Task SDK
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch prototype-task-sdk-docs
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 22513e3648bb65b41101993be3fccdb4db15df1b
Author: Ash Berlin-Taylor <as...@apache.org>
AuthorDate: Tue Mar 4 17:39:47 2025 +0000
Start working on the docs for the Task SDK
In order to have a nicer experience (i.e. rather than just being given an
alphabetical list of classes/functions) I have chosen to not use the auto
generate feature of the AutoAPI extension, but instead to precisely control
the order and grouping of the classes.
For this to be complete we will likely need some testing that compares the
items on the generated `objects.inv` with the things we re-export from
`airflow/sdk/__init__.py`.
The `airflow/sdk/__init__.pyi` type stub is sadly needed, as without it (and
with the imports only inside the `if TYPE_CHECKING` block) we weren't getting
the re-exported classes showing up where we want them, specifically we want
users to import things directly from airflow.sdk, not the submodule.
It was generated with `stubgen -o task_sdk/src -m airflow.sdk`
Test this with `sphinx-build -b html -Tv task_sdk/docs/ task_sdk/docs/_build`
---
task_sdk/docs/.gitignore | 1 +
task_sdk/docs/api.rst | 75 ++++++++++++++++++
task_sdk/docs/conf.py | 88 ++++++++++++++++++++++
task_sdk/docs/index.rst | 26 +++++++
task_sdk/src/airflow/sdk/__init__.pyi | 67 ++++++++++++++++
.../src/airflow/sdk/definitions/baseoperator.py | 4 +-
task_sdk/src/airflow/sdk/definitions/context.py | 2 +
task_sdk/src/airflow/sdk/definitions/dag.py | 4 +-
8 files changed, 262 insertions(+), 5 deletions(-)
diff --git a/task_sdk/docs/.gitignore b/task_sdk/docs/.gitignore
new file mode 100644
index 00000000000..69fa449dd96
--- /dev/null
+++ b/task_sdk/docs/.gitignore
@@ -0,0 +1 @@
+_build/
diff --git a/task_sdk/docs/api.rst b/task_sdk/docs/api.rst
new file mode 100644
index 00000000000..367cd21c29d
--- /dev/null
+++ b/task_sdk/docs/api.rst
@@ -0,0 +1,75 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+airflow.sdk API Reference
+=========================
+
+This page documents the full public API exposed in Airflow 3.0+ via the Task SDK python module.
+
+If something is not on this page it is best to assume that it is not part of the public API and use of it is entirely at your own risk
+-- we won't go out of our way break usage of them, but we make no promises either.
+
+
+Defining DAGs
+-------------
+
+.. autoapiclass:: airflow.sdk.DAG
+
+.. autoapifunction:: airflow.sdk.dag
+
+.. autoapiclass:: airflow.sdk.TaskGroup
+
+.. autoapifunction:: airflow.sdk.get_parsing_context
+
+.. autoapiclass:: airflow.sdk.definitions.context.AirflowParsingContext
+ :undoc-members:
+ :members:
+
+
+Tasks and Operators
+-------------------
+
+.. autoapiclass:: airflow.sdk.BaseOperator
+
+.. autoapiclass:: airflow.sdk.XComArg
+
+.. autoapifunction:: airflow.sdk.get_current_context
+
+Assets
+------
+
+.. autoapiclass:: airflow.sdk.Asset
+
+.. autoapiclass:: airflow.sdk.AssetAlias
+
+.. autoapiclass:: airflow.sdk.AssetAll
+
+.. autoapiclass:: airflow.sdk.AssetAny
+
+.. autoapiclass:: airflow.sdk.AssetWatcher
+
+
+.. Asset, AssetAlias, AssetAll, AssetAny, AssetWatcher
+
+Everything else
+---------------
+
+.. autoapimodule:: airflow.sdk
+ :members:
+ :exclude-members: BaseOperator, DAG, dag, asset, Asset, AssetAlias, AssetAll, AssetAny, AssetWatcher, TaskGroup, XComArg, get_current_context, get_parsing_context
+ :undoc-members:
+ :imported-members:
diff --git a/task_sdk/docs/conf.py b/task_sdk/docs/conf.py
new file mode 100644
index 00000000000..e1354829872
--- /dev/null
+++ b/task_sdk/docs/conf.py
@@ -0,0 +1,88 @@
+# Disable Flake8 because of all the sphinx imports
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from pathlib import Path
+
+CONF_DIR = Path(__file__).parent.absolute()
+
+project = "Apache Airflow Task SDK"
+
+language = "en"
+
+extensions = [
+ "sphinx.ext.autodoc",
+ "autoapi.extension",
+ "sphinx.ext.intersphinx",
+]
+
+autoapi_dirs = [CONF_DIR.joinpath("..", "src").resolve()]
+autoapi_root = "api"
+autoapi_ignore = [
+ "*/airflow/sdk/execution_time",
+ "*/airflow/sdk/api",
+ "*/_internal*",
+]
+autoapi_options = [
+ "undoc-members",
+ "members",
+]
+autoapi_add_toctree_entry = False
+autoapi_generate_api_docs = False
+
+autodoc_typehints = "description"
+
+# Prefer pyi over py files if both are found
+autoapi_file_patterns = ["*.pyi", "*.py"]
+# autoapi_generate_api_docs = False
+
+html_theme = "furo"
+
+global_substitutions = {
+ "experimental": "This is an :ref:`experimental feature <experimental>`.",
+}
+
+rst_epilog = "\n".join(f".. |{key}| replace:: {replace}" for key, replace in global_substitutions.items())
+
+
+intersphinx_resolve_self = "airflow"
+intersphinx_mapping = {
+ "airflow": (
+ "https://airflow.apache.org/docs/apache-airflow/stable/",
+ (
+ "../../docs/_inventory_cache/apache-airflow/objects.inv",
+ "../../docs/_build/apache-airflow/objects.inv",
+ None,
+ ),
+ )
+}
+
+
+def skip_util_classes(app, objtype, name, obj, skip, options):
+ if "definitions" in name:
+ if name == "DAG":
+ obj.id = "airflow.sdk.DAG"
+ return skip
+ skip = True
+ return skip
+
+
+def setup(sphinx):
+ # sphinx.connect("autoapi-skip-member", skip_util_classes)
+ ...
diff --git a/task_sdk/docs/index.rst b/task_sdk/docs/index.rst
new file mode 100644
index 00000000000..06e1e808026
--- /dev/null
+++ b/task_sdk/docs/index.rst
@@ -0,0 +1,26 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Apache Airflow Task Execution SDK
+=================================
+
+:any:`DAG` is where to start. :any:`dag`
+
+.. toctree::
+ :hidden:
+
+ api
diff --git a/task_sdk/src/airflow/sdk/__init__.pyi b/task_sdk/src/airflow/sdk/__init__.pyi
new file mode 100644
index 00000000000..c16f5e7b98b
--- /dev/null
+++ b/task_sdk/src/airflow/sdk/__init__.pyi
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.sdk.definitions.asset import (
+ Asset as Asset,
+ AssetAlias as AssetAlias,
+ AssetAll as AssetAll,
+ AssetAny as AssetAny,
+ AssetWatcher as AssetWatcher,
+)
+from airflow.sdk.definitions.assets.decorators import asset as asset
+from airflow.sdk.definitions.assets.metadata import Metadata as Metadata
+from airflow.sdk.definitions.baseoperator import BaseOperator as BaseOperator
+from airflow.sdk.definitions.connection import Connection as Connection
+from airflow.sdk.definitions.context import (
+ Context as Context,
+ get_current_context as get_current_context,
+ get_parsing_context as get_parsing_context,
+)
+from airflow.sdk.definitions.dag import DAG as DAG, dag as dag
+from airflow.sdk.definitions.edges import EdgeModifier as EdgeModifier, Label as Label
+from airflow.sdk.definitions.param import Param as Param
+from airflow.sdk.definitions.taskgroup import TaskGroup as TaskGroup
+from airflow.sdk.definitions.template import literal as literal
+from airflow.sdk.definitions.variable import Variable as Variable
+from airflow.sdk.definitions.xcom_arg import XComArg as XComArg
+
+__all__ = [
+ "__version__",
+ "Asset",
+ "AssetAlias",
+ "AssetAll",
+ "AssetAny",
+ "AssetWatcher",
+ "BaseOperator",
+ "Connection",
+ "Context",
+ "DAG",
+ "EdgeModifier",
+ "Label",
+ "Metadata",
+ "Param",
+ "TaskGroup",
+ "Variable",
+ "XComArg",
+ "asset",
+ "dag",
+ "get_current_context",
+ "get_parsing_context",
+ "literal",
+]
+
+__version__: str
diff --git a/task_sdk/src/airflow/sdk/definitions/baseoperator.py b/task_sdk/src/airflow/sdk/definitions/baseoperator.py
index 072da2a6ae5..40c3efaa5fa 100644
--- a/task_sdk/src/airflow/sdk/definitions/baseoperator.py
+++ b/task_sdk/src/airflow/sdk/definitions/baseoperator.py
@@ -74,9 +74,7 @@ if TYPE_CHECKING:
import jinja2
from airflow.models.xcom_arg import XComArg
- from airflow.sdk.definitions.context import Context
- from airflow.sdk.definitions.dag import DAG
- from airflow.sdk.definitions.taskgroup import TaskGroup
+ from airflow.sdk import DAG, Context, TaskGroup
from airflow.serialization.enums import DagAttributeTypes
from airflow.task.priority_strategy import PriorityWeightStrategy
from airflow.triggers.base import BaseTrigger
diff --git a/task_sdk/src/airflow/sdk/definitions/context.py b/task_sdk/src/airflow/sdk/definitions/context.py
index ee6a48cd904..4764f05523c 100644
--- a/task_sdk/src/airflow/sdk/definitions/context.py
+++ b/task_sdk/src/airflow/sdk/definitions/context.py
@@ -119,6 +119,8 @@ class AirflowParsingContext(NamedTuple):
If these values are not None, they will contain the specific DAG and Task ID that Airflow is requesting to
execute. You can use these for optimizing dynamically generated DAG files.
+
+ You can obtain the current values via :py:func:`.get_parsing_context`.
"""
dag_id: str | None
diff --git a/task_sdk/src/airflow/sdk/definitions/dag.py b/task_sdk/src/airflow/sdk/definitions/dag.py
index d716daa0f46..a1dd459776b 100644
--- a/task_sdk/src/airflow/sdk/definitions/dag.py
+++ b/task_sdk/src/airflow/sdk/definitions/dag.py
@@ -271,7 +271,7 @@ class DAG:
:param schedule: If provided, this defines the rules according to which DAG
runs are scheduled. Possible values include a cron expression string,
timedelta object, Timetable, or list of Asset objects.
- See also :doc:`/howto/timetable`.
+ See also :external:doc:`howto/timetable`.
:param start_date: The timestamp from which the scheduler will
attempt to backfill. If this is not provided, backfilling must be done
manually with an explicit time range.
@@ -345,7 +345,7 @@ class DAG:
:param tags: List of tags to help filtering DAGs in the UI.
:param owner_links: Dict of owners and their links, that will be clickable on the DAGs view UI.
Can be used as an HTTP link (for example the link to your Slack channel), or a mailto link.
- e.g: {"dag_owner": "https://airflow.apache.org/"}
+ e.g: ``{"dag_owner": "https://airflow.apache.org/"}``
:param auto_register: Automatically register this DAG when it is used in a ``with`` block
:param fail_fast: Fails currently running tasks when task in DAG fails.
**Warning**: A fail stop dag can only have tasks with the default trigger rule ("all_success").