You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/11/01 08:37:41 UTC
(airflow) branch main updated: feat(provider/azure): add managed identity support to synapse hook (#35329)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0652038c09 feat(provider/azure): add managed identity support to synapse hook (#35329)
0652038c09 is described below
commit 0652038c09a79420f46570e9ca5713eb4f2f6552
Author: Wei Lee <we...@gmail.com>
AuthorDate: Wed Nov 1 16:37:35 2023 +0800
feat(provider/azure): add managed identity support to synapse hook (#35329)
---
airflow/providers/microsoft/azure/hooks/synapse.py | 20 +++++++++++++++++---
.../connections/azure_synapse.rst | 13 ++++++++++---
2 files changed, 27 insertions(+), 6 deletions(-)
diff --git a/airflow/providers/microsoft/azure/hooks/synapse.py b/airflow/providers/microsoft/azure/hooks/synapse.py
index 47e6eea1f9..eb275781bb 100644
--- a/airflow/providers/microsoft/azure/hooks/synapse.py
+++ b/airflow/providers/microsoft/azure/hooks/synapse.py
@@ -24,7 +24,7 @@ from azure.synapse.spark import SparkClient
from airflow.exceptions import AirflowTaskTimeout
from airflow.hooks.base import BaseHook
-from airflow.providers.microsoft.azure.utils import get_field
+from airflow.providers.microsoft.azure.utils import get_default_azure_credential, get_field
if TYPE_CHECKING:
from azure.synapse.spark.models import SparkBatchJobOptions
@@ -72,6 +72,12 @@ class AzureSynapseHook(BaseHook):
return {
"tenantId": StringField(lazy_gettext("Tenant ID"), widget=BS3TextFieldWidget()),
"subscriptionId": StringField(lazy_gettext("Subscription ID"), widget=BS3TextFieldWidget()),
+ "managed_identity_client_id": StringField(
+ lazy_gettext("Managed Identity Client ID"), widget=BS3TextFieldWidget()
+ ),
+ "workload_identity_tenant_id": StringField(
+ lazy_gettext("Workload Identity Tenant ID"), widget=BS3TextFieldWidget()
+ ),
}
@staticmethod
@@ -79,7 +85,13 @@ class AzureSynapseHook(BaseHook):
"""Returns custom field behaviour."""
return {
"hidden_fields": ["schema", "port", "extra"],
- "relabeling": {"login": "Client ID", "password": "Secret", "host": "Synapse Workspace URL"},
+ "relabeling": {
+ "login": "Client ID",
+ "password": "Secret",
+ "host": "Synapse Workspace URL",
+ "managed_identity_client_id": "Managed Identity Client ID",
+ "workload_identity_tenant_id": "Workload Identity Tenant ID",
+ },
}
def __init__(self, azure_synapse_conn_id: str = default_conn_name, spark_pool: str = ""):
@@ -120,7 +132,9 @@ class AzureSynapseHook(BaseHook):
client_id=conn.login, client_secret=conn.password, tenant_id=tenant
)
else:
- credential = DefaultAzureCredential()
+ managed_identity_client_id = self._get_field(extras, "managed_identity_client_id")
+ workload_identity_tenant_id = self._get_field(extras, "workload_identity_tenant_id")
+ credential = get_default_azure_credential(managed_identity_client_id, workload_identity_tenant_id)
self._conn = self._create_client(credential, conn.host, spark_pool, livy_api_version, subscription_id)
diff --git a/docs/apache-airflow-providers-microsoft-azure/connections/azure_synapse.rst b/docs/apache-airflow-providers-microsoft-azure/connections/azure_synapse.rst
index 2327b1474c..ee0da07521 100644
--- a/docs/apache-airflow-providers-microsoft-azure/connections/azure_synapse.rst
+++ b/docs/apache-airflow-providers-microsoft-azure/connections/azure_synapse.rst
@@ -27,13 +27,13 @@ The Microsoft Azure Synapse connection type enables the Azure Synapse Integratio
Authenticating to Azure Synapse
-------------------------------
-There are multiple ways to connect to Azure Synapse using Airflow.
+There are three ways to connect to Azure Synapse using Airflow.
1. Use `token credentials
<https://docs.microsoft.com/en-us/azure/developer/python/azure-sdk-authenticate?tabs=cmd#authenticate-with-token-credentials>`_
i.e. add specific credentials (client_id, secret, tenant) and subscription id to the Airflow connection.
-2. Fallback on `DefaultAzureCredential
- <https://docs.microsoft.com/en-us/python/api/overview/azure/identity-readme?view=azure-python#defaultazurecredential>`_.
+2. Use managed identity by setting ``managed_identity_client_id``, ``workload_identity_tenant_id`` (under the hook, it uses DefaultAzureCredential_ with these arguments)
+3. Fallback on DefaultAzureCredential_.
This includes a mechanism to try different options to authenticate: Managed System Identity, environment variables, authentication through Azure CLI...
Default Connection IDs
@@ -65,3 +65,10 @@ Subscription ID
Synapse Workspace URL
Specify the Azure Synapse endpoint to interface with.
+
+
+.. _DefaultAzureCredential: https://docs.microsoft.com/en-us/python/api/overview/azure/identity-readme?view=azure-python#defaultazurecredential
+
+.. spelling:word-list::
+
+ Entra