You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by pi...@apache.org on 2025/07/31 13:47:34 UTC
(airflow) branch v3-0-test updated: Handle unmodified sensitived fields when updating connections (#53943) (#53973)
This is an automated email from the ASF dual-hosted git repository.
pierrejeambrun pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new 75df7271858 Handle unmodified sensitived fields when updating connections (#53943) (#53973)
75df7271858 is described below
commit 75df727185815ae57c694e9023408ab24e67646c
Author: Pierre Jeambrun <pi...@gmail.com>
AuthorDate: Thu Jul 31 15:47:27 2025 +0200
Handle unmodified sensitived fields when updating connections (#53943) (#53973)
* Fix redacted values editing
* Small improvements
* Small adjustments
* Update UI and fix some errors
* Address PR comments
(cherry picked from commit 0abcfdf0e61ebbaa6ae24883642439f40852c325)
---
.../core_api/services/public/connections.py | 19 +-
.../src/components/FlexibleForm/FlexibleForm.tsx | 7 +
.../ui/src/pages/Connections/ConnectionForm.tsx | 8 +
.../src/pages/Connections/EditConnectionButton.tsx | 1 +
.../core_api/routes/public/test_connections.py | 21 ++
.../airflow/sdk/execution_time/secrets_masker.py | 118 ++++++++
.../task_sdk/definitions/test_secrets_masker.py | 304 +++++++++++++++++++++
7 files changed, 476 insertions(+), 2 deletions(-)
diff --git a/airflow-core/src/airflow/api_fastapi/core_api/services/public/connections.py b/airflow-core/src/airflow/api_fastapi/core_api/services/public/connections.py
index 4c236cf9756..05ab88ab90c 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/services/public/connections.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/services/public/connections.py
@@ -17,6 +17,8 @@
from __future__ import annotations
+import json
+
from fastapi import HTTPException, status
from pydantic import ValidationError
from sqlalchemy import select
@@ -32,6 +34,7 @@ from airflow.api_fastapi.core_api.datamodels.common import (
from airflow.api_fastapi.core_api.datamodels.connections import ConnectionBody
from airflow.api_fastapi.core_api.services.public.common import BulkService
from airflow.models.connection import Connection
+from airflow.sdk.execution_time.secrets_masker import merge
def update_orm_from_pydantic(
@@ -56,11 +59,23 @@ def update_orm_from_pydantic(
if (not update_mask and "password" in pydantic_conn.model_fields_set) or (
update_mask and "password" in update_mask
):
- orm_conn.set_password(pydantic_conn.password)
+ if pydantic_conn.password is None:
+ orm_conn.set_password(pydantic_conn.password)
+ else:
+ merged_password = merge(pydantic_conn.password, orm_conn.password, "password")
+ orm_conn.set_password(merged_password)
if (not update_mask and "extra" in pydantic_conn.model_fields_set) or (
update_mask and "extra" in update_mask
):
- orm_conn.set_extra(pydantic_conn.extra)
+ if pydantic_conn.extra is None or orm_conn.extra is None:
+ orm_conn.set_extra(pydantic_conn.extra)
+ return
+ try:
+ merged_extra = merge(json.loads(pydantic_conn.extra), json.loads(orm_conn.extra))
+ orm_conn.set_extra(json.dumps(merged_extra))
+ except json.JSONDecodeError:
+ # We can't merge fields in an unstructured `extra`
+ orm_conn.set_extra(pydantic_conn.extra)
class BulkConnectionService(BulkService[ConnectionBody]):
diff --git a/airflow-core/src/airflow/ui/src/components/FlexibleForm/FlexibleForm.tsx b/airflow-core/src/airflow/ui/src/components/FlexibleForm/FlexibleForm.tsx
index 4fea751e599..cf9f2bf4b85 100644
--- a/airflow-core/src/airflow/ui/src/components/FlexibleForm/FlexibleForm.tsx
+++ b/airflow-core/src/airflow/ui/src/components/FlexibleForm/FlexibleForm.tsx
@@ -32,12 +32,14 @@ export type FlexibleFormProps = {
initialParamsDict: { paramsDict: ParamsSpec };
key?: string;
setError: (error: boolean) => void;
+ subHeader?: string;
};
export const FlexibleForm = ({
flexibleFormDefaultSection,
initialParamsDict,
setError,
+ subHeader,
}: FlexibleFormProps) => {
const { paramsDict: params, setInitialParamDict, setParamsDict } = useParamStore();
const processedSections = new Map();
@@ -126,6 +128,11 @@ export const FlexibleForm = ({
<Accordion.ItemContent pt={0}>
<Accordion.ItemBody>
+ {Boolean(subHeader) ? (
+ <Text color="fg.muted" fontSize="xs" mb={2}>
+ {subHeader}
+ </Text>
+ ) : undefined}
<Stack separator={<StackSeparator />}>
{Object.entries(params)
.filter(
diff --git a/airflow-core/src/airflow/ui/src/pages/Connections/ConnectionForm.tsx b/airflow-core/src/airflow/ui/src/pages/Connections/ConnectionForm.tsx
index 866d555c068..5ca5f6756a6 100644
--- a/airflow-core/src/airflow/ui/src/pages/Connections/ConnectionForm.tsx
+++ b/airflow-core/src/airflow/ui/src/pages/Connections/ConnectionForm.tsx
@@ -36,13 +36,17 @@ import type { ConnectionBody } from "./Connections";
type AddConnectionFormProps = {
readonly error: unknown;
readonly initialConnection: ConnectionBody;
+ readonly isEditMode?: boolean;
readonly isPending: boolean;
readonly mutateConnection: (requestBody: ConnectionBody) => void;
};
+const redactedFieldDisclaimer = "Redacted fields ('***') will remain unchanged if not modified.";
+
const ConnectionForm = ({
error,
initialConnection,
+ isEditMode = false,
isPending,
mutateConnection,
}: AddConnectionFormProps) => {
@@ -202,6 +206,7 @@ const ConnectionForm = ({
initialParamsDict={paramsDic}
key={selectedConnType}
setError={setFormErrors}
+ subHeader={isEditMode ? redactedFieldDisclaimer : undefined}
/>
<Accordion.Item key="extraJson" value="extraJson">
<Accordion.ItemTrigger cursor="button">Extra Fields JSON</Accordion.ItemTrigger>
@@ -218,6 +223,9 @@ const ConnectionForm = ({
}}
/>
{Boolean(errors.conf) ? <Field.ErrorText>{errors.conf}</Field.ErrorText> : undefined}
+ {isEditMode ? (
+ <Field.HelperText>{redactedFieldDisclaimer}</Field.HelperText>
+ ) : undefined}
</Field.Root>
)}
/>
diff --git a/airflow-core/src/airflow/ui/src/pages/Connections/EditConnectionButton.tsx b/airflow-core/src/airflow/ui/src/pages/Connections/EditConnectionButton.tsx
index 33ce9832b9a..3781b4c198e 100644
--- a/airflow-core/src/airflow/ui/src/pages/Connections/EditConnectionButton.tsx
+++ b/airflow-core/src/airflow/ui/src/pages/Connections/EditConnectionButton.tsx
@@ -79,6 +79,7 @@ const EditConnectionButton = ({ connection, disabled }: Props) => {
<ConnectionForm
error={error}
initialConnection={initialConnectionValue}
+ isEditMode
isPending={isPending}
mutateConnection={editConnection}
/>
diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_connections.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_connections.py
index ddff799431c..5f210278be8 100644
--- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_connections.py
+++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_connections.py
@@ -467,6 +467,27 @@ class TestPatchConnection(TestConnectionEndpoint):
"schema": None,
},
),
+ (
+ # Sensitive "***" should be ignored.
+ {
+ "connection_id": TEST_CONN_ID,
+ "conn_type": TEST_CONN_TYPE,
+ "port": 80,
+ "login": "test_login_patch",
+ "password": "***",
+ },
+ {
+ "conn_type": TEST_CONN_TYPE,
+ "connection_id": TEST_CONN_ID,
+ "description": TEST_CONN_DESCRIPTION,
+ "extra": None,
+ "host": TEST_CONN_HOST,
+ "login": "test_login_patch",
+ "password": None,
+ "port": 80,
+ "schema": None,
+ },
+ ),
(
{
"connection_id": TEST_CONN_ID,
diff --git a/task-sdk/src/airflow/sdk/execution_time/secrets_masker.py b/task-sdk/src/airflow/sdk/execution_time/secrets_masker.py
index cfc6214325d..4abdcd7cf93 100644
--- a/task-sdk/src/airflow/sdk/execution_time/secrets_masker.py
+++ b/task-sdk/src/airflow/sdk/execution_time/secrets_masker.py
@@ -34,6 +34,7 @@ from typing import (
TextIO,
TypeVar,
Union,
+ overload,
)
from airflow import settings
@@ -127,6 +128,27 @@ def redact(value: Redactable, name: str | None = None, max_depth: int | None = N
return _secrets_masker().redact(value, name, max_depth)
+@overload
+def merge(new_value: str, old_value: str, name: str | None = None, max_depth: int | None = None) -> str: ...
+
+
+@overload
+def merge(new_value: dict, old_value: dict, name: str | None = None, max_depth: int | None = None) -> str: ...
+
+
+def merge(
+ new_value: Redacted, old_value: Redactable, name: str | None = None, max_depth: int | None = None
+) -> Redacted:
+ """
+ Merge a redacted value with its original unredacted counterpart.
+
+ Takes a user-modified redacted value and merges it with the original unredacted value.
+ For sensitive fields that still contain "***" (unchanged), the original value is restored.
+ For fields that have been updated by the user, the new value is preserved.
+ """
+ return _secrets_masker().merge(new_value, old_value, name, max_depth)
+
+
@cache
def _secrets_masker() -> SecretsMasker:
for flt in logging.getLogger("airflow.task").filters:
@@ -302,6 +324,83 @@ class SecretsMasker(logging.Filter):
)
return item
+ def _merge(
+ self,
+ new_item: Redacted,
+ old_item: Redactable,
+ name: str | None,
+ depth: int,
+ max_depth: int,
+ force_sensitive: bool = False,
+ ) -> Redacted:
+ """Merge a redacted item with its original unredacted counterpart."""
+ if depth > max_depth:
+ if isinstance(new_item, str) and new_item == "***":
+ return old_item
+ return new_item
+
+ try:
+ # Determine if we should treat this as sensitive
+ is_sensitive = force_sensitive or (name is not None and should_hide_value_for_key(name))
+
+ if isinstance(new_item, dict) and isinstance(old_item, dict):
+ merged = {}
+ for key in new_item.keys():
+ if key in old_item:
+ # For dicts, pass the key as name unless we're in sensitive mode
+ child_name = None if is_sensitive else key
+ merged[key] = self._merge(
+ new_item[key],
+ old_item[key],
+ name=child_name,
+ depth=depth + 1,
+ max_depth=max_depth,
+ force_sensitive=is_sensitive,
+ )
+ else:
+ merged[key] = new_item[key]
+ return merged
+
+ if isinstance(new_item, (list, tuple)) and type(old_item) is type(new_item):
+ merged_list = []
+ for i in range(len(new_item)):
+ if i < len(old_item):
+ # In sensitive mode, check if individual item is redacted
+ if is_sensitive and isinstance(new_item[i], str) and new_item[i] == "***":
+ merged_list.append(old_item[i])
+ else:
+ merged_list.append(
+ self._merge(
+ new_item[i],
+ old_item[i],
+ name=None,
+ depth=depth + 1,
+ max_depth=max_depth,
+ force_sensitive=is_sensitive,
+ )
+ )
+ else:
+ merged_list.append(new_item[i])
+
+ if isinstance(new_item, list):
+ return list(merged_list)
+ return tuple(merged_list)
+
+ if isinstance(new_item, set) and isinstance(old_item, set):
+ # Sets are unordered, we cannot restore original items.
+ return new_item
+
+ if _is_v1_env_var(new_item) and _is_v1_env_var(old_item):
+ # TODO: Handle Kubernetes V1EnvVar objects if needed
+ return new_item
+
+ if is_sensitive and isinstance(new_item, str) and new_item == "***":
+ return old_item
+ return new_item
+
+ except (TypeError, AttributeError, ValueError):
+ return new_item
+
def redact(self, item: Redactable, name: str | None = None, max_depth: int | None = None) -> Redacted:
"""
Redact an any secrets found in ``item``, if it is a string.
@@ -312,6 +411,25 @@ class SecretsMasker(logging.Filter):
"""
return self._redact(item, name, depth=0, max_depth=max_depth or self.MAX_RECURSION_DEPTH)
+ def merge(
+ self, new_item: Redacted, old_item: Redactable, name: str | None = None, max_depth: int | None = None
+ ) -> Redacted:
+ """
+ Merge a redacted item with its original unredacted counterpart.
+
+ Takes a user-modified redacted item and merges it with the original unredacted item.
+ For sensitive fields that still contain "***" (unchanged), the original value is restored.
+ For fields that have been updated, the new value is preserved.
+ """
+ return self._merge(
+ new_item,
+ old_item,
+ name=name,
+ depth=0,
+ max_depth=max_depth or self.MAX_RECURSION_DEPTH,
+ force_sensitive=False,
+ )
+
@cached_property
def _mask_adapter(self) -> None | Callable:
"""
diff --git a/task-sdk/tests/task_sdk/definitions/test_secrets_masker.py b/task-sdk/tests/task_sdk/definitions/test_secrets_masker.py
index 2a111ceefba..100cffeac7c 100644
--- a/task-sdk/tests/task_sdk/definitions/test_secrets_masker.py
+++ b/task-sdk/tests/task_sdk/definitions/test_secrets_masker.py
@@ -34,6 +34,7 @@ from airflow.sdk.execution_time.secrets_masker import (
RedactedIO,
SecretsMasker,
mask_secret,
+ merge,
redact,
reset_secrets_masker,
should_hide_value_for_key,
@@ -53,6 +54,7 @@ def lineno():
class MyEnum(str, Enum):
testname = "testvalue"
+ testname2 = "testvalue2"
@pytest.fixture
@@ -729,3 +731,305 @@ class TestMixedDataScenarios:
assert "***" in redacted_data["description"]
assert redacted_data["nested"]["token"] == "***"
assert redacted_data["nested"]["info"] == "No secrets here"
+
+
+class TestSecretsMaskerMerge:
+ """Test the merge functionality for restoring original values from redacted data."""
+
+ @pytest.mark.parametrize(
+ ("new_value", "old_value", "name", "expected"),
+ [
+ ("***", "original_secret", "password", "original_secret"),
+ ("new_secret", "original_secret", "password", "new_secret"),
+ ("***", "original_value", "normal_field", "***"),
+ ("new_value", "original_value", "normal_field", "new_value"),
+ ("***", "original_value", None, "***"),
+ ("new_value", "original_value", None, "new_value"),
+ ],
+ )
+ @pytest.mark.usefixtures("patched_secrets_masker")
+ def test_merge_simple_strings(self, new_value, old_value, name, expected):
+ result = merge(new_value, old_value, name)
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ ("old_data", "new_data", "expected"),
+ [
+ (
+ {
+ "password": "original_password",
+ "api_key": "original_api_key",
+ "normal_field": "original_normal",
+ },
+ {
+ "password": "***",
+ "api_key": "new_api_key",
+ "normal_field": "new_normal",
+ },
+ {
+ "password": "original_password",
+ "api_key": "new_api_key",
+ "normal_field": "new_normal",
+ },
+ ),
+ (
+ {
+ "config": {"password": "original_password", "host": "original_host"},
+ "credentials": {"api_key": "original_api_key", "username": "original_user"},
+ },
+ {
+ "config": {
+ "password": "***",
+ "host": "new_host",
+ },
+ "credentials": {
+ "api_key": "new_api_key",
+ "username": "new_user",
+ },
+ },
+ {
+ "config": {
+ "password": "original_password",
+ "host": "new_host",
+ },
+ "credentials": {
+ "api_key": "new_api_key",
+ "username": "new_user",
+ },
+ },
+ ),
+ ],
+ )
+ @pytest.mark.usefixtures("patched_secrets_masker")
+ def test_merge_dictionaries(self, old_data, new_data, expected):
+ result = merge(new_data, old_data)
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ ("old_data", "new_data", "name", "expected"),
+ [
+ # Lists
+ (
+ ["original_item1", "original_item2", "original_item3"],
+ ["new_item1", "new_item2"],
+ None,
+ ["new_item1", "new_item2"],
+ ),
+ (
+ ["original_item1", "original_item2"],
+ ["new_item1", "new_item2", "new_item3", "new_item4"],
+ None,
+ ["new_item1", "new_item2", "new_item3", "new_item4"],
+ ),
+ (
+ ["secret1", "secret2", "secret3"],
+ ["***", "new_secret2", "***"],
+ "password",
+ ["secret1", "new_secret2", "secret3"],
+ ),
+ (
+ ["value1", "value2", "value3"],
+ ["***", "new_value2", "***"],
+ "normal_list",
+ ["***", "new_value2", "***"],
+ ),
+ # Tuples
+ (
+ ("original_item1", "original_item2", "original_item3"),
+ ("new_item1", "new_item2"),
+ None,
+ ("new_item1", "new_item2"),
+ ),
+ (
+ ("original_item1", "original_item2"),
+ ("new_item1", "new_item2", "new_item3", "new_item4"),
+ None,
+ ("new_item1", "new_item2", "new_item3", "new_item4"),
+ ),
+ (
+ ("secret1", "secret2", "secret3"),
+ ("***", "new_secret2", "***"),
+ "password",
+ ("secret1", "new_secret2", "secret3"),
+ ),
+ (
+ ("value1", "value2", "value3"),
+ ("***", "new_value2", "***"),
+ "normal_tuple",
+ ("***", "new_value2", "***"),
+ ),
+ # Sets
+ (
+ {"original_item1", "original_item2", "original_item3"},
+ {"new_item1", "new_item2"},
+ None,
+ {"new_item1", "new_item2"},
+ ),
+ (
+ {"original_item1", "original_item2"},
+ {"new_item1", "new_item2", "new_item3", "new_item4"},
+ None,
+ {"new_item1", "new_item2", "new_item3", "new_item4"},
+ ),
+ (
+ {"secret1", "secret2", "secret3"},
+ {"***", "new_secret2", "***"},
+ "password",
+ {"***", "new_secret2", "***"},
+ ),
+ (
+ {"value1", "value2", "value3"},
+ {"***", "new_value2", "***"},
+ "normal_tuple",
+ {"***", "new_value2", "***"},
+ ),
+ ],
+ )
+ @pytest.mark.usefixtures("patched_secrets_masker")
+ def test_merge_collections(self, old_data, new_data, name, expected):
+ result = merge(new_data, old_data, name)
+ assert result == expected
+
+ @pytest.mark.usefixtures("patched_secrets_masker")
+ def test_merge_mismatched_types(self):
+ old_data = {"key": "value"}
+ new_data = "some_string" # Different type
+
+ # When types don't match, prefer the new item
+ expected = "some_string"
+
+ result = merge(new_data, old_data)
+ assert result == expected
+
+ @pytest.mark.usefixtures("patched_secrets_masker")
+ def test_merge_with_missing_keys(self):
+ old_data = {"password": "original_password", "old_only_key": "old_value", "common_key": "old_common"}
+
+ new_data = {
+ "password": "***",
+ "new_only_key": "new_value",
+ "common_key": "new_common",
+ }
+
+ expected = {
+ "password": "original_password",
+ "new_only_key": "new_value",
+ "common_key": "new_common",
+ }
+
+ result = merge(new_data, old_data)
+ assert result == expected
+
+ @pytest.mark.usefixtures("patched_secrets_masker")
+ def test_merge_complex_redacted_structures(self):
+ old_data = {
+ "some_config": {
+ "nested_password": "original_nested_password",
+ "passwords": ["item1", "item2"],
+ },
+ "normal_field": "normal_value",
+ }
+
+ new_data = {
+ "some_config": {"nested_password": "***", "passwords": ["***", "new_item2"]},
+ "normal_field": "new_normal_value",
+ }
+
+ result = merge(new_data, old_data)
+ expected = {
+ "some_config": {
+ "nested_password": "original_nested_password",
+ "passwords": ["item1", "new_item2"],
+ },
+ "normal_field": "new_normal_value",
+ }
+ assert result == expected
+
+ @pytest.mark.usefixtures("patched_secrets_masker")
+ def test_merge_partially_redacted_structures(self):
+ old_data = {
+ "config": {
+ "password": "original_password",
+ "host": "original_host",
+ "nested": {"api_key": "original_api_key", "timeout": 30},
+ }
+ }
+
+ new_data = {
+ "config": {
+ "password": "***",
+ "host": "new_host",
+ "nested": {
+ "api_key": "***",
+ "timeout": 60,
+ },
+ }
+ }
+
+ expected = {
+ "config": {
+ "password": "original_password",
+ "host": "new_host",
+ "nested": {
+ "api_key": "original_api_key",
+ "timeout": 60,
+ },
+ }
+ }
+
+ result = merge(new_data, old_data)
+ assert result == expected
+
+ @pytest.mark.usefixtures("patched_secrets_masker")
+ def test_merge_max_depth(self):
+ old_data = {"level1": {"level2": {"level3": {"password": "original_password"}}}}
+ new_data = {"level1": {"level2": {"level3": {"password": "***"}}}}
+
+ result = merge(new_data, old_data, max_depth=1)
+ assert result == new_data
+
+ result = merge(new_data, old_data, max_depth=10)
+ assert result["level1"]["level2"]["level3"]["password"] == "original_password"
+
+ @pytest.mark.usefixtures("patched_secrets_masker")
+ def test_merge_enum_values(self):
+ old_enum = MyEnum.testname
+ new_enum = MyEnum.testname2
+
+ result = merge(new_enum, old_enum)
+ assert result == new_enum
+ assert isinstance(result, MyEnum)
+
+ @pytest.mark.usefixtures("patched_secrets_masker")
+ def test_merge_round_trip(self):
+ # Original data with sensitive information
+ original_config = {
+ "database": {"host": "db.example.com", "password": "super_secret_password", "username": "admin"},
+ "api": {"api_key": "secret_api_key_12345", "endpoint": "https://api.example.com", "timeout": 30},
+ "app_name": "my_application",
+ }
+
+ # Step 1: Redact the original data
+ redacted_dict = redact(original_config)
+
+ # Verify sensitive fields are redacted
+ assert redacted_dict["database"]["password"] == "***"
+ assert redacted_dict["api"]["api_key"] == "***"
+ assert redacted_dict["database"]["host"] == "db.example.com"
+
+ # Step 2: User modifies some fields
+ updated_dict = redacted_dict.copy()
+ updated_dict["database"]["host"] = "new-db.example.com"
+ updated_dict["api"]["timeout"] = 60
+ updated_dict["api"]["api_key"] = "new_api_key_67890"
+ # User left password as "***" (unchanged)
+
+ # Step 3: Merge to restore unchanged sensitive values
+ final_dict = merge(updated_dict, original_config)
+
+ # Verify the results
+ assert final_dict["database"]["password"] == "super_secret_password" # Restored
+ assert final_dict["database"]["host"] == "new-db.example.com" # User modification kept
+ assert final_dict["api"]["api_key"] == "new_api_key_67890" # User modification kept
+ assert final_dict["api"]["timeout"] == 60 # User modification kept
+ assert final_dict["app_name"] == "my_application" # Unchanged