You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bu...@apache.org on 2025/07/24 23:09:12 UTC
(airflow) branch main updated: AIP-81 Make list operations to return all results (#50132)
This is an automated email from the ASF dual-hosted git repository.
bugraoz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f5b12db393b AIP-81 Make list operations to return all results (#50132)
f5b12db393b is described below
commit f5b12db393bde99192fc7e43388f03d1e96f6b5a
Author: Pratiksha <12...@users.noreply.github.com>
AuthorDate: Fri Jul 25 04:39:06 2025 +0530
AIP-81 Make list operations to return all results (#50132)
* ensure total_entries return all entries for corresponding endpoints
* use API endpoint parameters limit and offset to retrieve all entries
resolve conflict
* use a common fun to get all values
* Use a common fun for return all results
* create a method in BaseOperation and use in Pools list, restore common.py and restore endpoints - pools.py, variables.py, job.py
* improve looping and fix static checks
* restore endpoints and fix static checks for job.py
* fix static checks from common.py
* update offset params
* update default limit = 50 to limit
* remove equal condition from while loop and remove unnecessary params
* use first pass response and merge it with the return list
* define missing parameters-type in return_all_entries method
* add tests for return_all_entries and update list in pool and variable tests
* fix tests
* fix static checks
* make chnages regarding params and dict
* change offset defulat value since intial pass added
* fix static checks
* add try-except at return_all_list method
* fix static check
* fix static checks for command_hashes.txt
* Remove Base Operation from output_main.svg file
* remove end-space from output_main.svg
* add a generic method for list methods
* remove unnecessary code
* generic method returns pydantic_response
* fix broken execute_list method
* generic method returns TypeVar
* implement generic method in all list methods and remove uncessary code
* Remove unnecessary code and add type of data_model
* add dag_id params to backfills list method
* remove duplicate total_entries and kwargs from generic method
* remove duplicate code , assert tests
* simplified tests
* remove extra variable ,print statement
---
airflow-ctl/docs/images/command_hashes.txt | 2 +-
airflow-ctl/docs/images/output_main.svg | 112 ++++++++---------
airflow-ctl/src/airflowctl/api/operations.py | 139 +++++++++------------
.../tests/airflow_ctl/api/test_operations.py | 52 +++++++-
4 files changed, 168 insertions(+), 137 deletions(-)
diff --git a/airflow-ctl/docs/images/command_hashes.txt b/airflow-ctl/docs/images/command_hashes.txt
index 84cab0f38ec..d689874c2a0 100644
--- a/airflow-ctl/docs/images/command_hashes.txt
+++ b/airflow-ctl/docs/images/command_hashes.txt
@@ -1,4 +1,4 @@
-main:8d768c837899829dfd21d37253d2fb44
+main:32f8c659348c161980e59398a17f1bb0
assets:b3ae2b933e54528bf486ff28e887804d
auth:f396d4bce90215599dde6ad0a8f30f29
backfills:725109470cd2613de8cc8af022fb54bc
diff --git a/airflow-ctl/docs/images/output_main.svg b/airflow-ctl/docs/images/output_main.svg
index 98bf85d2fd2..a5f8703a854 100644
--- a/airflow-ctl/docs/images/output_main.svg
+++ b/airflow-ctl/docs/images/output_main.svg
@@ -19,135 +19,135 @@
font-weight: 700;
}
- .terminal-101498644-matrix {
+ .terminal-2610352951-matrix {
font-family: Fira Code, monospace;
font-size: 20px;
line-height: 24.4px;
font-variant-east-asian: full-width;
}
- .terminal-101498644-title {
+ .terminal-2610352951-title {
font-size: 18px;
font-weight: bold;
font-family: arial;
}
- .terminal-101498644-r1 { fill: #c5c8c6 }
+ .terminal-2610352951-r1 { fill: #c5c8c6 }
</style>
<defs>
- <clipPath id="terminal-101498644-clip-terminal">
+ <clipPath id="terminal-2610352951-clip-terminal">
<rect x="0" y="0" width="975.0" height="609.0" />
</clipPath>
- <clipPath id="terminal-101498644-line-0">
+ <clipPath id="terminal-2610352951-line-0">
<rect x="0" y="1.5" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-1">
+<clipPath id="terminal-2610352951-line-1">
<rect x="0" y="25.9" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-2">
+<clipPath id="terminal-2610352951-line-2">
<rect x="0" y="50.3" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-3">
+<clipPath id="terminal-2610352951-line-3">
<rect x="0" y="74.7" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-4">
+<clipPath id="terminal-2610352951-line-4">
<rect x="0" y="99.1" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-5">
+<clipPath id="terminal-2610352951-line-5">
<rect x="0" y="123.5" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-6">
+<clipPath id="terminal-2610352951-line-6">
<rect x="0" y="147.9" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-7">
+<clipPath id="terminal-2610352951-line-7">
<rect x="0" y="172.3" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-8">
+<clipPath id="terminal-2610352951-line-8">
<rect x="0" y="196.7" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-9">
+<clipPath id="terminal-2610352951-line-9">
<rect x="0" y="221.1" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-10">
+<clipPath id="terminal-2610352951-line-10">
<rect x="0" y="245.5" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-11">
+<clipPath id="terminal-2610352951-line-11">
<rect x="0" y="269.9" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-12">
+<clipPath id="terminal-2610352951-line-12">
<rect x="0" y="294.3" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-13">
+<clipPath id="terminal-2610352951-line-13">
<rect x="0" y="318.7" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-14">
+<clipPath id="terminal-2610352951-line-14">
<rect x="0" y="343.1" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-15">
+<clipPath id="terminal-2610352951-line-15">
<rect x="0" y="367.5" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-16">
+<clipPath id="terminal-2610352951-line-16">
<rect x="0" y="391.9" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-17">
+<clipPath id="terminal-2610352951-line-17">
<rect x="0" y="416.3" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-18">
+<clipPath id="terminal-2610352951-line-18">
<rect x="0" y="440.7" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-19">
+<clipPath id="terminal-2610352951-line-19">
<rect x="0" y="465.1" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-20">
+<clipPath id="terminal-2610352951-line-20">
<rect x="0" y="489.5" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-21">
+<clipPath id="terminal-2610352951-line-21">
<rect x="0" y="513.9" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-22">
+<clipPath id="terminal-2610352951-line-22">
<rect x="0" y="538.3" width="976" height="24.65"/>
</clipPath>
-<clipPath id="terminal-101498644-line-23">
+<clipPath id="terminal-2610352951-line-23">
<rect x="0" y="562.7" width="976" height="24.65"/>
</clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="992" height="658" rx="8"/><text class="terminal-101498644-title" fill="#c5c8c6" text-anchor="middle" x="496" y="27">Command: main</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="992" height="658" rx="8"/><text class="terminal-2610352951-title" fill="#c5c8c6" text-anchor="middle" x="496" y="27">Command: main</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
<circle cx="44" cy="0" r="7" fill="#28c840"/>
</g>
- <g transform="translate(9, 41)" clip-path="url(#terminal-101498644-clip-terminal)">
+ <g transform="translate(9, 41)" clip-path="url(#terminal-2610352951-clip-terminal)">
- <g class="terminal-101498644-matrix">
- <text class="terminal-101498644-r1" x="0" y="20" textLength="524.6" clip-path="url(#terminal-101498644-line-0)">Usage: airflowctl [-h] GROUP_OR_COMMAND ...</text><text class="terminal-101498644-r1" x="976" y="20" textLength="12.2" clip-path="url(#terminal-101498644-line-0)">
-</text><text class="terminal-101498644-r1" x="976" y="44.4" textLength="12.2" clip-path="url(#terminal-101498644-line-1)">
-</text><text class="terminal-101498644-r1" x="0" y="68.8" textLength="256.2" clip-path="url(#terminal-101498644-line-2)">Positional Arguments:</text><text class="terminal-101498644-r1" x="976" y="68.8" textLength="12.2" clip-path="url(#terminal-101498644-line-2)">
-</text><text class="terminal-101498644-r1" x="0" y="93.2" textLength="219.6" clip-path="url(#terminal-101498644-line-3)">  GROUP_OR_COMMAND</text><text class="terminal-101498644-r1" x="976" y="93.2" textLength="12.2" clip-path="url(#terminal-101498644-line-3)">
-</text><text class="terminal-101498644-r1" x="976" y="117.6" textLength="12.2" clip-path="url(#terminal-101498644-line-4)">
-</text><text class="terminal-101498644-r1" x="0" y="142" textLength="122" clip-path="url(#terminal-101498644-line-5)">    Groups</text><text class="terminal-101498644-r1" x="976" y="142" textLength="12.2" clip-path="url(#terminal-101498644-line-5)">
-</text><text class="terminal-101498644-r1" x="0" y="166.4" textLength="549" clip-path="url(#terminal-101498644-line-6)">      assets        Perform Assets operations</text><text class="terminal-101498644-r1" x="976" y="166.4" textLength="12.2" clip-path="url(#terminal-101498644-line-6)">
-</text><text class="terminal-101498644-r1" x="0" y="190.8" textLength="890.6" clip-path="url(#terminal-101498644-line-7)">      auth          Manage authentication for CLI. Either pass token from</text><text class="terminal-101498644-r1" x="976" y="190.8" textLength="12.2" clip-path="url(#terminal-101498644-line-7)">
-</text><text class="terminal-101498644-r1" x="0" y="215.2" textLength="866.2" clip-path="url(#terminal-101498644-line-8)">                    environment variable/parameter or pass username and</text><text class="terminal-101498644-r1" x="976" y="215.2" textLength="12.2" clip-path="url(#terminal-101498644-line-8)">
-</text><text class="terminal-101498644-r1" x="0" y="239.6" textLength="353.8" clip-path="url(#terminal-101498644-line-9)">                    password.</text><text class="terminal-101498644-r1" x="976" y="239.6" textLength="12.2" clip-path="url(#terminal-101498644-line-9)">
-</text><text class="terminal-101498644-r1" x="0" y="264" textLength="585.6" clip-path="url(#terminal-101498644-line-10)">      backfills     Perform Backfills operations</text><text class="terminal-101498644-r1" x="976" y="264" textLength="12.2" clip-path="url(#terminal-101498644-line-10)">
-</text><text class="terminal-101498644-r1" x="0" y="288.4" textLength="549" clip-path="url(#terminal-101498644-line-11)">      config        Perform Config operations</text><text class="terminal-101498644-r1" x="976" y="288.4" textLength="12.2" clip-path="url(#terminal-101498644-line-11)">
-</text><text class="terminal-101498644-r1" x="0" y="312.8" textLength="610" clip-path="url(#terminal-101498644-line-12)">      connections   Perform Connections operations</text><text class="terminal-101498644-r1" x="976" y="312.8" textLength="12.2" clip-path="url(#terminal-101498644-line-12)">
-</text><text class="terminal-101498644-r1" x="0" y="337.2" textLength="512.4" clip-path="url(#terminal-101498644-line-13)">      dag           Perform Dag operations</text><text class="terminal-101498644-r1" x="976" y="337.2" textLength="12.2" clip-path="url(#terminal-101498644-line-13)">
-</text><text class="terminal-101498644-r1" x="0" y="361.6" textLength="549" clip-path="url(#terminal-101498644-line-14)">      dagrun        Perform DagRun operations</text><text class="terminal-101498644-r1" x="976" y="361.6" textLength="12.2" clip-path="url(#terminal-101498644-line-14)">
-</text><text class="terminal-101498644-r1" x="0" y="386" textLength="524.6" clip-path="url(#terminal-101498644-line-15)">      jobs          Perform Jobs operations</text><text class="terminal-101498644-r1" x="976" y="386" textLength="12.2" clip-path="url(#terminal-101498644-line-15)">
-</text><text class="terminal-101498644-r1" x="0" y="410.4" textLength="536.8" clip-path="url(#terminal-101498644-line-16)">      pools         Perform Pools operations</text><text class="terminal-101498644-r1" x="976" y="410.4" textLength="12.2" clip-path="url(#terminal-101498644-line-16)">
-</text><text class="terminal-101498644-r1" x="0" y="434.8" textLength="585.6" clip-path="url(#terminal-101498644-line-17)">      providers     Perform Providers operations</text><text class="terminal-101498644-r1" x="976" y="434.8" textLength="12.2" clip-path="url(#terminal-101498644-line-17)">
-</text><text class="terminal-101498644-r1" x="0" y="459.2" textLength="585.6" clip-path="url(#terminal-101498644-line-18)">      variables     Perform Variables operations</text><text class="terminal-101498644-r1" x="976" y="459.2" textLength="12.2" clip-path="url(#terminal-101498644-line-18)">
-</text><text class="terminal-101498644-r1" x="976" y="483.6" textLength="12.2" clip-path="url(#terminal-101498644-line-19)">
-</text><text class="terminal-101498644-r1" x="0" y="508" textLength="158.6" clip-path="url(#terminal-101498644-line-20)">    Commands:</text><text class="terminal-101498644-r1" x="976" y="508" textLength="12.2" clip-path="url(#terminal-101498644-line-20)">
-</text><text class="terminal-101498644-r1" x="0" y="532.4" textLength="536.8" clip-path="url(#terminal-101498644-line-21)">      version       Show version information</text><text class="terminal-101498644-r1" x="976" y="532.4" textLength="12.2" clip-path="url(#terminal-101498644-line-21)">
-</text><text class="terminal-101498644-r1" x="976" y="556.8" textLength="12.2" clip-path="url(#terminal-101498644-line-22)">
-</text><text class="terminal-101498644-r1" x="0" y="581.2" textLength="97.6" clip-path="url(#terminal-101498644-line-23)">Options:</text><text class="terminal-101498644-r1" x="976" y="581.2" textLength="12.2" clip-path="url(#terminal-101498644-line-23)">
-</text><text class="terminal-101498644-r1" x="0" y="605.6" textLength="622.2" clip-path="url(#terminal-101498644-line-24)">  -h, --help        show this help message and exit</text><text class="terminal-101498644-r1" x="976" y="605.6" textLength="12.2" clip-path="url(#terminal-101498644-line-24)">
+ <g class="terminal-2610352951-matrix">
+ <text class="terminal-2610352951-r1" x="0" y="20" textLength="524.6" clip-path="url(#terminal-2610352951-line-0)">Usage: airflowctl [-h] GROUP_OR_COMMAND ...</text><text class="terminal-2610352951-r1" x="976" y="20" textLength="12.2" clip-path="url(#terminal-2610352951-line-0)">
+</text><text class="terminal-2610352951-r1" x="976" y="44.4" textLength="12.2" clip-path="url(#terminal-2610352951-line-1)">
+</text><text class="terminal-2610352951-r1" x="0" y="68.8" textLength="256.2" clip-path="url(#terminal-2610352951-line-2)">Positional Arguments:</text><text class="terminal-2610352951-r1" x="976" y="68.8" textLength="12.2" clip-path="url(#terminal-2610352951-line-2)">
+</text><text class="terminal-2610352951-r1" x="0" y="93.2" textLength="219.6" clip-path="url(#terminal-2610352951-line-3)">  GROUP_OR_COMMAND</text><text class="terminal-2610352951-r1" x="976" y="93.2" textLength="12.2" clip-path="url(#terminal-2610352951-line-3)">
+</text><text class="terminal-2610352951-r1" x="976" y="117.6" textLength="12.2" clip-path="url(#terminal-2610352951-line-4)">
+</text><text class="terminal-2610352951-r1" x="0" y="142" textLength="122" clip-path="url(#terminal-2610352951-line-5)">    Groups</text><text class="terminal-2610352951-r1" x="976" y="142" textLength="12.2" clip-path="url(#terminal-2610352951-line-5)">
+</text><text class="terminal-2610352951-r1" x="0" y="166.4" textLength="549" clip-path="url(#terminal-2610352951-line-6)">      assets        Perform Assets operations</text><text class="terminal-2610352951-r1" x="976" y="166.4" textLength="12.2" clip-path="url(#terminal-2610352951-line-6)">
+</text><text class="terminal-2610352951-r1" x="0" y="190.8" textLength="890.6" clip-path="url(#terminal-2610352951-line-7)">      auth          Manage authentication for CLI. Either pass token from</text><text class="terminal-2610352951-r1" x="976" y="190.8" textLength="12.2" clip-path="url(#terminal-2610352951-line-7)">
+</text><text class="terminal-2610352951-r1" x="0" y="215.2" textLength="866.2" clip-path="url(#terminal-2610352951-line-8)">                    environment variable/parameter or pass username and</text><text class="terminal-2610352951-r1" x="976" y="215.2" textLength="12.2" clip-path="url(#terminal-2610352951-line-8)">
+</text><text class="terminal-2610352951-r1" x="0" y="239.6" textLength="353.8" clip-path="url(#terminal-2610352951-line-9)">                    password.</text><text class="terminal-2610352951-r1" x="976" y="239.6" textLength="12.2" clip-path="url(#terminal-2610352951-line-9)">
+</text><text class="terminal-2610352951-r1" x="0" y="264" textLength="585.6" clip-path="url(#terminal-2610352951-line-10)">      backfills     Perform Backfills operations</text><text class="terminal-2610352951-r1" x="976" y="264" textLength="12.2" clip-path="url(#terminal-2610352951-line-10)">
+</text><text class="terminal-2610352951-r1" x="0" y="288.4" textLength="549" clip-path="url(#terminal-2610352951-line-11)">      config        Perform Config operations</text><text class="terminal-2610352951-r1" x="976" y="288.4" textLength="12.2" clip-path="url(#terminal-2610352951-line-11)">
+</text><text class="terminal-2610352951-r1" x="0" y="312.8" textLength="610" clip-path="url(#terminal-2610352951-line-12)">      connections   Perform Connections operations</text><text class="terminal-2610352951-r1" x="976" y="312.8" textLength="12.2" clip-path="url(#terminal-2610352951-line-12)">
+</text><text class="terminal-2610352951-r1" x="0" y="337.2" textLength="512.4" clip-path="url(#terminal-2610352951-line-13)">      dag           Perform Dag operations</text><text class="terminal-2610352951-r1" x="976" y="337.2" textLength="12.2" clip-path="url(#terminal-2610352951-line-13)">
+</text><text class="terminal-2610352951-r1" x="0" y="361.6" textLength="549" clip-path="url(#terminal-2610352951-line-14)">      dagrun        Perform DagRun operations</text><text class="terminal-2610352951-r1" x="976" y="361.6" textLength="12.2" clip-path="url(#terminal-2610352951-line-14)">
+</text><text class="terminal-2610352951-r1" x="0" y="386" textLength="524.6" clip-path="url(#terminal-2610352951-line-15)">      jobs          Perform Jobs operations</text><text class="terminal-2610352951-r1" x="976" y="386" textLength="12.2" clip-path="url(#terminal-2610352951-line-15)">
+</text><text class="terminal-2610352951-r1" x="0" y="410.4" textLength="536.8" clip-path="url(#terminal-2610352951-line-16)">      pools         Perform Pools operations</text><text class="terminal-2610352951-r1" x="976" y="410.4" textLength="12.2" clip-path="url(#terminal-2610352951-line-16)">
+</text><text class="terminal-2610352951-r1" x="0" y="434.8" textLength="585.6" clip-path="url(#terminal-2610352951-line-17)">      providers     Perform Providers operations</text><text class="terminal-2610352951-r1" x="976" y="434.8" textLength="12.2" clip-path="url(#terminal-2610352951-line-17)">
+</text><text class="terminal-2610352951-r1" x="0" y="459.2" textLength="585.6" clip-path="url(#terminal-2610352951-line-18)">      variables     Perform Variables operations</text><text class="terminal-2610352951-r1" x="976" y="459.2" textLength="12.2" clip-path="url(#terminal-2610352951-line-18)">
+</text><text class="terminal-2610352951-r1" x="976" y="483.6" textLength="12.2" clip-path="url(#terminal-2610352951-line-19)">
+</text><text class="terminal-2610352951-r1" x="0" y="508" textLength="158.6" clip-path="url(#terminal-2610352951-line-20)">    Commands:</text><text class="terminal-2610352951-r1" x="976" y="508" textLength="12.2" clip-path="url(#terminal-2610352951-line-20)">
+</text><text class="terminal-2610352951-r1" x="0" y="532.4" textLength="536.8" clip-path="url(#terminal-2610352951-line-21)">      version       Show version information</text><text class="terminal-2610352951-r1" x="976" y="532.4" textLength="12.2" clip-path="url(#terminal-2610352951-line-21)">
+</text><text class="terminal-2610352951-r1" x="976" y="556.8" textLength="12.2" clip-path="url(#terminal-2610352951-line-22)">
+</text><text class="terminal-2610352951-r1" x="0" y="581.2" textLength="97.6" clip-path="url(#terminal-2610352951-line-23)">Options:</text><text class="terminal-2610352951-r1" x="976" y="581.2" textLength="12.2" clip-path="url(#terminal-2610352951-line-23)">
+</text><text class="terminal-2610352951-r1" x="0" y="605.6" textLength="622.2" clip-path="url(#terminal-2610352951-line-24)">  -h, --help        show this help message and exit</text><text class="terminal-2610352951-r1" x="976" y="605.6" textLength="12.2" clip-path="url(#terminal-2610352951-line-24)">
</text>
</g>
</g>
diff --git a/airflow-ctl/src/airflowctl/api/operations.py b/airflow-ctl/src/airflowctl/api/operations.py
index 0161f566ff7..b247fd787aa 100644
--- a/airflow-ctl/src/airflowctl/api/operations.py
+++ b/airflow-ctl/src/airflowctl/api/operations.py
@@ -18,10 +18,11 @@
from __future__ import annotations
import datetime
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypeVar
import httpx
import structlog
+from pydantic import BaseModel
from airflowctl.api.datamodels.auth_generated import LoginBody, LoginResponse
from airflowctl.api.datamodels.generated import (
@@ -77,6 +78,8 @@ if TYPE_CHECKING:
log = structlog.get_logger(logger_name=__name__)
+T = TypeVar("T", bound=BaseModel)
+
# Generic Server Response Error
class ServerResponseError(httpx.HTTPStatusError):
@@ -147,6 +150,37 @@ class BaseOperations:
if callable(value):
setattr(cls, attr, _check_flag_and_exit_if_server_response_error(value))
+ def execute_list(
+ self,
+ *,
+ path: str,
+ data_model: type[T],
+ offset: int = 0,
+ limit: int = 50,
+ params: dict | None = None,
+ ) -> T | ServerResponseError:
+ shared_params = {**(params or {})}
+ try:
+ self.response = self.client.get(path, params=shared_params)
+ first_pass = data_model.model_validate_json(self.response.content)
+ total_entries = first_pass.total_entries # type: ignore[attr-defined]
+ if total_entries < limit:
+ return first_pass
+ for key, value in first_pass.model_dump().items():
+ if key != "total_entries" and isinstance(value, list):
+ break
+ entry_list = getattr(first_pass, key)
+ offset = offset + limit
+ while offset < total_entries:
+ self.response = self.client.get(path, params={**shared_params, "offset": offset})
+ entry = data_model.model_validate_json(self.response.content)
+ offset = offset + limit
+ entry_list.extend(getattr(entry, key))
+ obj = data_model(**{key: entry_list, "total_entries": total_entries})
+ return data_model.model_validate(obj.model_dump())
+ except ServerResponseError as e:
+ raise e
+
# Login operations
class LoginOperations:
@@ -188,19 +222,11 @@ class AssetsOperations(BaseOperations):
def list(self) -> AssetCollectionResponse | ServerResponseError:
"""List all assets from the API server."""
- try:
- self.response = self.client.get("assets")
- return AssetCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(path="assets", data_model=AssetCollectionResponse)
def list_by_alias(self) -> AssetAliasCollectionResponse | ServerResponseError:
"""List all assets by alias from the API server."""
- try:
- self.response = self.client.get("/assets/aliases")
- return AssetAliasCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(path="/assets/aliases", data_model=AssetAliasCollectionResponse)
def create_event(
self, asset_event_body: CreateAssetEventsBody
@@ -298,13 +324,10 @@ class BackfillsOperations(BaseOperations):
except ServerResponseError as e:
raise e
- def list(self) -> BackfillCollectionResponse | ServerResponseError:
+ def list(self, dag_id: str) -> BackfillCollectionResponse | ServerResponseError:
"""List all backfills."""
- try:
- self.response = self.client.get("backfills")
- return BackfillCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ params = {"dag_id": dag_id}
+ return super().execute_list(path="backfills", data_model=BackfillCollectionResponse, params=params)
def pause(self, backfill_id: str) -> BackfillResponse | ServerResponseError:
"""Pause a backfill."""
@@ -364,11 +387,7 @@ class ConnectionsOperations(BaseOperations):
def list(self) -> ConnectionCollectionResponse | ServerResponseError:
"""List all connections from the API server."""
- try:
- self.response = self.client.get("connections")
- return ConnectionCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(path="connections", data_model=ConnectionCollectionResponse)
def create(
self,
@@ -451,19 +470,11 @@ class DagOperations(BaseOperations):
def get_tags(self) -> DAGTagCollectionResponse | ServerResponseError:
"""Get all DAG tags."""
- try:
- self.response = self.client.get("dagTags")
- return DAGTagCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(path="dagTags", data_model=DAGTagCollectionResponse)
def list(self) -> DAGCollectionResponse | ServerResponseError:
"""List DAGs."""
- try:
- self.response = self.client.get("dags")
- return DAGCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(path="dags", data_model=DAGCollectionResponse)
def patch(self, dag_id: str, dag_body: DAGPatchBody) -> DAGResponse | ServerResponseError:
try:
@@ -487,11 +498,7 @@ class DagOperations(BaseOperations):
raise e
def list_import_error(self) -> ImportErrorCollectionResponse | ServerResponseError:
- try:
- self.response = self.client.get("importErrors")
- return ImportErrorCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(path="importErrors", data_model=ImportErrorCollectionResponse)
def get_stats(self, dag_ids: list) -> DagStatsCollectionResponse | ServerResponseError: # type: ignore
try:
@@ -508,18 +515,12 @@ class DagOperations(BaseOperations):
raise e
def list_version(self, dag_id: str) -> DAGVersionCollectionResponse | ServerResponseError:
- try:
- self.response = self.client.get(f"dags/{dag_id}/dagVersions")
- return DAGVersionCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(
+ path=f"dags/{dag_id}/dagVersions", data_model=DAGVersionCollectionResponse
+ )
def list_warning(self) -> DAGWarningCollectionResponse | ServerResponseError:
- try:
- self.response = self.client.get("dagWarnings")
- return DAGWarningCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(path="dagWarnings", data_model=DAGWarningCollectionResponse)
class DagRunOperations(BaseOperations):
@@ -542,17 +543,13 @@ class DagRunOperations(BaseOperations):
limit: int,
) -> DAGRunCollectionResponse | ServerResponseError:
"""List all dag runs."""
- try:
- params = {
- "start_date": start_date,
- "end_date": end_date,
- "state": state,
- "limit": limit,
- }
- self.response = self.client.get("dag_runs", params=params) # type: ignore
- return DAGRunCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ params = {
+ "start_date": start_date,
+ "end_date": end_date,
+ "state": state,
+ "limit": limit,
+ }
+ return super().execute_list(path="dag_runs", data_model=DAGRunCollectionResponse, params=params)
def create(
self, dag_id: str, trigger_dag_run: TriggerDAGRunPostBody
@@ -573,12 +570,8 @@ class JobsOperations(BaseOperations):
self, job_type: str, hostname: str, is_alive: bool
) -> JobCollectionResponse | ServerResponseError:
"""List all jobs."""
- try:
- params = {"job_type": job_type, "hostname": hostname, "is_alive": is_alive}
- self.response = self.client.get("jobs", params=params) # type: ignore
- return JobCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ params = {"job_type": job_type, "hostname": hostname, "is_alive": is_alive}
+ return super().execute_list(path="jobs", data_model=JobCollectionResponse, params=params)
class PoolsOperations(BaseOperations):
@@ -594,11 +587,7 @@ class PoolsOperations(BaseOperations):
def list(self) -> PoolCollectionResponse | ServerResponseError:
"""List all pools."""
- try:
- self.response = self.client.get("pools")
- return PoolCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(path="pools", data_model=PoolCollectionResponse)
def create(self, pool: PoolBody) -> PoolResponse | ServerResponseError:
"""Create a pool."""
@@ -638,11 +627,7 @@ class ProvidersOperations(BaseOperations):
def list(self) -> ProviderCollectionResponse | ServerResponseError:
"""List all providers."""
- try:
- self.response = self.client.get("providers")
- return ProviderCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(path="providers", data_model=ProviderCollectionResponse)
class VariablesOperations(BaseOperations):
@@ -658,11 +643,7 @@ class VariablesOperations(BaseOperations):
def list(self) -> VariableCollectionResponse | ServerResponseError:
"""List all variables."""
- try:
- self.response = self.client.get("variables")
- return VariableCollectionResponse.model_validate_json(self.response.content)
- except ServerResponseError as e:
- raise e
+ return super().execute_list(path="variables", data_model=VariableCollectionResponse)
def create(self, variable: VariableBody) -> VariableResponse | ServerResponseError:
"""Create a variable."""
diff --git a/airflow-ctl/tests/airflow_ctl/api/test_operations.py b/airflow-ctl/tests/airflow_ctl/api/test_operations.py
index 1a894990183..c8f88f046fb 100644
--- a/airflow-ctl/tests/airflow_ctl/api/test_operations.py
+++ b/airflow-ctl/tests/airflow_ctl/api/test_operations.py
@@ -24,6 +24,7 @@ from typing import TYPE_CHECKING
import httpx
import pytest
+from pydantic import BaseModel
from airflowctl.api.client import Client, ClientKind
from airflowctl.api.datamodels.auth_generated import LoginBody, LoginResponse
@@ -106,6 +107,15 @@ def make_api_client(
return Client(base_url=base_url, transport=transport, token=token, kind=kind)
+class HelloResponse(BaseModel):
+ name: str
+
+
+class HelloCollectionResponse(BaseModel):
+ hellos: list[HelloResponse]
+ total_entries: int
+
+
class TestBaseOperations:
def test_server_connection_refused(self):
client = make_api_client(base_url="http://localhost")
@@ -114,6 +124,46 @@ class TestBaseOperations:
):
client.connections.get("1")
+ @pytest.mark.parametrize(
+ "total_entries, limit, expected_response",
+ [
+ (0, 0, (HelloCollectionResponse(hellos=[], total_entries=0))),
+ (1, 50, (HelloCollectionResponse(hellos=[HelloResponse(name="hello")], total_entries=1))),
+ (
+ 150,
+ 50,
+ (
+ HelloCollectionResponse(
+ hellos=[
+ HelloResponse(name="hello"),
+ HelloResponse(name="hello"),
+ HelloResponse(name="hello"),
+ ],
+ total_entries=150,
+ )
+ ),
+ ),
+ (
+ 90,
+ 50,
+ (
+ HelloCollectionResponse(
+ hellos=[HelloResponse(name="hello"), HelloResponse(name="hello")], total_entries=90
+ )
+ ),
+ ),
+ ],
+ )
+ def test_execute_list(self, total_entries, limit, expected_response):
+ hello_response = []
+ if total_entries != 0:
+ update = (total_entries + limit - 1) // limit
+ hello_response.extend([HelloResponse(name="hello")] * update)
+ hello_collection_response = HelloCollectionResponse(
+ hellos=hello_response, total_entries=total_entries
+ )
+ assert expected_response == hello_collection_response
+
class TestAssetsOperations:
asset_id: int = 1
@@ -367,7 +417,7 @@ class TestBackfillOperations:
return httpx.Response(200, json=json.loads(self.backfills_collection_response.model_dump_json()))
client = make_api_client(transport=httpx.MockTransport(handle_request))
- response = client.backfills.list()
+ response = client.backfills.list(dag_id="dag_id")
assert response == self.backfills_collection_response
def test_pause(self):