You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/10/20 06:38:54 UTC
[airflow] branch main updated: Add extra fields to plugins endpoint (#34913)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d58da227f6 Add extra fields to plugins endpoint (#34913)
d58da227f6 is described below
commit d58da227f6cd419185e7b7a14c6ba23eb5195c10
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Fri Oct 20 07:38:47 2023 +0100
Add extra fields to plugins endpoint (#34913)
* Add extra fields to plugins endpoint
I added three extra fields, ti_deps, timetables, and listeners which I think are worth having since they will help in visualizing if those are included in a plugin.
I also found out that the UI has admin_views and menu_links which seems to come from airflow 1 but for consistency, I merged the attributes for both UI & webserver to be the same. The REST API does not have these two attributes as I feel they will soon be removed.
* fixup! Add extra fields to plugins endpoint
* Remove nullable=true in the arrays
* update the listener serialization
---
airflow/api_connexion/openapi/v1.yaml | 15 +++++++++
airflow/api_connexion/schemas/plugin_schema.py | 3 ++
airflow/plugins_manager.py | 4 ++-
airflow/www/static/js/types/api-generated.ts | 6 ++++
airflow/www/views.py | 15 ++-------
.../endpoints/test_plugin_endpoint.py | 37 ++++++++++++++++++++++
tests/api_connexion/schemas/test_plugin_schema.py | 9 ++++++
tests/cli/commands/test_plugins_command.py | 2 ++
8 files changed, 77 insertions(+), 14 deletions(-)
diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml
index ebd10e855a..8fd0a582e6 100644
--- a/airflow/api_connexion/openapi/v1.yaml
+++ b/airflow/api_connexion/openapi/v1.yaml
@@ -3833,6 +3833,21 @@ components:
type: string
description: The plugin source
nullable: true
+ ti_deps:
+ type: array
+ items:
+ type: string
+ description: The plugin task instance dependencies
+ listeners:
+ type: array
+ items:
+ type: string
+ description: The plugin listeners
+ timetables:
+ type: array
+ items:
+ type: string
+ description: The plugin timetables
PluginCollection:
type: object
diff --git a/airflow/api_connexion/schemas/plugin_schema.py b/airflow/api_connexion/schemas/plugin_schema.py
index 4b62111482..afdef350bc 100644
--- a/airflow/api_connexion/schemas/plugin_schema.py
+++ b/airflow/api_connexion/schemas/plugin_schema.py
@@ -34,6 +34,9 @@ class PluginSchema(Schema):
global_operator_extra_links = fields.List(fields.String())
operator_extra_links = fields.List(fields.String())
source = fields.String()
+ ti_deps = fields.List(fields.String())
+ listeners = fields.List(fields.String())
+ timetables = fields.List(fields.String())
class PluginCollection(NamedTuple):
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
index 7275588d52..143e3af570 100644
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -78,14 +78,16 @@ PLUGINS_ATTRIBUTES_TO_DUMP = {
"hooks",
"executors",
"macros",
+ "admin_views",
"flask_blueprints",
+ "menu_links",
"appbuilder_views",
"appbuilder_menu_items",
"global_operator_extra_links",
"operator_extra_links",
+ "source",
"ti_deps",
"timetables",
- "source",
"listeners",
}
diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts
index 5c7a8ecb08..e3368f8a79 100644
--- a/airflow/www/static/js/types/api-generated.ts
+++ b/airflow/www/static/js/types/api-generated.ts
@@ -1597,6 +1597,12 @@ export interface components {
operator_extra_links?: (string | null)[];
/** @description The plugin source */
source?: string | null;
+ /** @description The plugin task instance dependencies */
+ ti_deps?: string[];
+ /** @description The plugin listeners */
+ listeners?: string[];
+ /** @description The plugin timetables */
+ timetables?: string[];
};
/**
* @description A collection of plugin.
diff --git a/airflow/www/views.py b/airflow/www/views.py
index f99205db98..88e46a14c2 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -106,6 +106,7 @@ from airflow.models.dataset import DagScheduleDatasetReference, DatasetDagRunQue
from airflow.models.operator import needs_expansion
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance, TaskInstanceNote
+from airflow.plugins_manager import PLUGINS_ATTRIBUTES_TO_DUMP
from airflow.providers_manager import ProvidersManager
from airflow.security import permissions
from airflow.ti_deps.dep_context import DepContext
@@ -4521,19 +4522,7 @@ class PluginView(AirflowBaseView):
permissions.ACTION_CAN_ACCESS_MENU,
]
- plugins_attributes_to_dump = [
- "hooks",
- "executors",
- "macros",
- "admin_views",
- "flask_blueprints",
- "menu_links",
- "appbuilder_views",
- "appbuilder_menu_items",
- "global_operator_extra_links",
- "operator_extra_links",
- "source",
- ]
+ plugins_attributes_to_dump = PLUGINS_ATTRIBUTES_TO_DUMP
@expose("/plugin")
@auth.has_access_website()
diff --git a/tests/api_connexion/endpoints/test_plugin_endpoint.py b/tests/api_connexion/endpoints/test_plugin_endpoint.py
index a6f67ab5a7..6a941d6997 100644
--- a/tests/api_connexion/endpoints/test_plugin_endpoint.py
+++ b/tests/api_connexion/endpoints/test_plugin_endpoint.py
@@ -16,6 +16,8 @@
# under the License.
from __future__ import annotations
+import inspect
+
import pytest
from flask import Blueprint
from flask_appbuilder import BaseView
@@ -24,6 +26,8 @@ from airflow.hooks.base import BaseHook
from airflow.models.baseoperator import BaseOperatorLink
from airflow.plugins_manager import AirflowPlugin
from airflow.security import permissions
+from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
+from airflow.timetables.base import Timetable
from airflow.utils.module_loading import qualname
from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user
from tests.test_utils.config import conf_vars
@@ -60,6 +64,30 @@ appbuilder_menu_items = {
}
+class CustomTIDep(BaseTIDep):
+ pass
+
+
+ti_dep = CustomTIDep()
+
+
+class CustomTimetable(Timetable):
+ def infer_manual_data_interval(self, *, run_after):
+ pass
+
+ def next_dagrun_info(
+ self,
+ *,
+ last_automated_data_interval,
+ restriction,
+ ):
+ pass
+
+
+class MyCustomListener:
+ pass
+
+
class MockPlugin(AirflowPlugin):
name = "mock_plugin"
flask_blueprints = [bp]
@@ -69,6 +97,9 @@ class MockPlugin(AirflowPlugin):
operator_extra_links = [MockOperatorLink()]
hooks = [PluginHook]
macros = [plugin_macro]
+ ti_deps = [ti_dep]
+ timetables = [CustomTimetable]
+ listeners = [pytest, MyCustomListener()] # using pytest here because we need a module(just for test)
@pytest.fixture(scope="module")
@@ -120,6 +151,12 @@ class TestGetPlugins(TestPluginsEndpoint):
"operator_extra_links": [f"<{qualname(MockOperatorLink().__class__)} object>"],
"source": None,
"name": "test_plugin",
+ "timetables": [qualname(CustomTimetable)],
+ "ti_deps": [str(ti_dep)],
+ "listeners": [
+ d.__name__ if inspect.ismodule(d) else qualname(d)
+ for d in [pytest, MyCustomListener()]
+ ],
}
],
"total_entries": 1,
diff --git a/tests/api_connexion/schemas/test_plugin_schema.py b/tests/api_connexion/schemas/test_plugin_schema.py
index 179a318fe5..1472fd2db7 100644
--- a/tests/api_connexion/schemas/test_plugin_schema.py
+++ b/tests/api_connexion/schemas/test_plugin_schema.py
@@ -91,6 +91,9 @@ class TestPluginSchema(TestPluginBase):
"operator_extra_links": [str(MockOperatorLink())],
"source": None,
"name": "test_plugin",
+ "ti_deps": [],
+ "listeners": [],
+ "timetables": [],
}
@@ -112,6 +115,9 @@ class TestPluginCollectionSchema(TestPluginBase):
"operator_extra_links": [str(MockOperatorLink())],
"source": None,
"name": "test_plugin",
+ "ti_deps": [],
+ "listeners": [],
+ "timetables": [],
},
{
"appbuilder_menu_items": [appbuilder_menu_items],
@@ -124,6 +130,9 @@ class TestPluginCollectionSchema(TestPluginBase):
"operator_extra_links": [str(MockOperatorLink())],
"source": None,
"name": "test_plugin_2",
+ "ti_deps": [],
+ "listeners": [],
+ "timetables": [],
},
],
"total_entries": 2,
diff --git a/tests/cli/commands/test_plugins_command.py b/tests/cli/commands/test_plugins_command.py
index 049fab4354..cbf6afeab9 100644
--- a/tests/cli/commands/test_plugins_command.py
+++ b/tests/cli/commands/test_plugins_command.py
@@ -61,7 +61,9 @@ class TestPluginsCommand:
assert info == [
{
"name": "test_plugin",
+ "admin_views": [],
"macros": ["tests.plugins.test_plugin.plugin_macro"],
+ "menu_links": [],
"executors": ["tests.plugins.test_plugin.PluginExecutor"],
"flask_blueprints": [
"<flask.blueprints.Blueprint: name='test_plugin' import_name='tests.plugins.test_plugin'>"