You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by GitBox <gi...@apache.org> on 2020/05/23 02:37:01 UTC
[GitHub] [airflow] kaxil commented on a change in pull request #8974: Vault has now VaultHook not only SecretBackend
kaxil commented on a change in pull request #8974:
URL: https://github.com/apache/airflow/pull/8974#discussion_r429505660
##########
File path: airflow/providers/hashicorp/secrets/vault.py
##########
@@ -72,98 +68,90 @@ class VaultBackend(BaseSecretsBackend, LoggingMixin):
:type username: str
:param password: Password for Authentication (for ``ldap`` and ``userpass`` auth_type)
:type password: str
- :param role_id: Role ID for Authentication (for ``approle`` auth_type)
+ :param key_id: Key ID for Authentication (for ``aws_iam`` and ''azure`` auth_type)
+ :type key_id: str
Review comment:
```suggestion
:type key_id: str
```
##########
File path: airflow/providers/hashicorp/secrets/vault.py
##########
@@ -72,98 +68,90 @@ class VaultBackend(BaseSecretsBackend, LoggingMixin):
:type username: str
:param password: Password for Authentication (for ``ldap`` and ``userpass`` auth_type)
:type password: str
- :param role_id: Role ID for Authentication (for ``approle`` auth_type)
+ :param key_id: Key ID for Authentication (for ``aws_iam`` and ''azure`` auth_type)
+ :type key_id: str
+ :param secret_id: Secret ID for Authentication (for ``approle``, ``aws_iam`` and ``azure`` auth_types)
+ :type secret_id: str
:type role_id: str
+ :param secret_id: Secret ID for Authentication (for ``approle`` auth_type)
+ :type secret_id: str
:param kubernetes_role: Role for Authentication (for ``kubernetes`` auth_type)
:type kubernetes_role: str
- :param kubernetes_jwt_path: Path for kubernetes jwt token (for ``kubernetes`` auth_type, deafult:
+ :param kubernetes_jwt_path: Path for kubernetes jwt token (for ``kubernetes`` auth_type, default:
``/var/run/secrets/kubernetes.io/serviceaccount/token``)
:type kubernetes_jwt_path: str
- :param secret_id: Secret ID for Authentication (for ``approle`` auth_type)
- :type secret_id: str
:param gcp_key_path: Path to GCP Credential JSON file (for ``gcp`` auth_type)
+ Mutually exclusive with gcp_keyfile_dict
:type gcp_key_path: str
+ :param gcp_keyfile_dict: Dictionary of keyfile parameters. (for ``gcp`` auth_type).
+ Mutually exclusive with gcp_key_path
+ :type gcp_keyfile_dict: dict
:param gcp_scopes: Comma-separated string containing GCP scopes (for ``gcp`` auth_type)
:type gcp_scopes: str
+ :param azure_tenant_id: Tenant of azure (for ``azure`` auth_type)
+ :type azure_tenant_id: str
+ :param azure_resource: Resource if of azure (for ``azure`` auth_type)
+ :type azure_resource: str
+ :param radius_host: Host for radius (for ``radius`` auth_type)
+ :type radius_host: str
+ :param radius_secret: Secret for radius (for ``radius`` auth_type)
+ :type radius_secret: str
+ :param radius_port: Port for radius (for ``radius`` auth_type)
+ :type radius_port: str
"""
def __init__( # pylint: disable=too-many-arguments
self,
connections_path: str = 'connections',
variables_path: str = 'variables',
url: Optional[str] = None,
auth_type: str = 'token',
- mount_point: str = 'secret',
+ mount_point: Optional[str] = None,
kv_engine_version: int = 2,
token: Optional[str] = None,
username: Optional[str] = None,
password: Optional[str] = None,
+ key_id: Optional[str] = None,
+ secret_id: Optional[str] = None,
role_id: Optional[str] = None,
kubernetes_role: Optional[str] = None,
kubernetes_jwt_path: str = '/var/run/secrets/kubernetes.io/serviceaccount/token',
- secret_id: Optional[str] = None,
gcp_key_path: Optional[str] = None,
+ gcp_keyfile_dict=None,
Review comment:
Typehint
##########
File path: airflow/providers/hashicorp/hooks/vault.py
##########
@@ -0,0 +1,281 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Hook for HashiCorp Vault"""
+import json
+from typing import Optional
+
+import hvac
+from hvac.exceptions import VaultError
+from requests import Response
+
+from airflow.hooks.base_hook import BaseHook
+from airflow.providers.hashicorp.common.vault_client import VaultClient
+
+
+class VaultHook(BaseHook):
+ """
+ HashiCorp Vault wrapper to Interact with HashiCorp Vault KeyValue Secret engine.
+
+ HashiCorp HVac documentation:
+ * https://hvac.readthedocs.io/en/stable/
+
+ You connect to the host specified as host in the connection. The login/password from the connection
+ are used as credentials usually and you can specify different authentication parameters
+ via init params or via corresponding extras in the connection.
+
+ The extras in the connection are named the same as the parameters (`mount_point`,'kv_engine_version' ...).
+
+ Login/Password are used as credentials:
+ * approle: password -> secret_id
+ * aws_iam: login -> key_id, password -> secret_id
+ * azure: login -> client_id, password -> client_secret
+ * ldap: login -> username, password -> password
+ * userpass: login -> username, password -> password
+ * radius: password -> radius_secret
+
+ :param vault_conn_id: id of the connection to use
+ :type vault_conn_id: str
+ :param auth_type: type of authentication. One of
+ :type auth_type: str
+ :param mount_point: The "path" the secret engine was mounted on. (Default: ``secret``)
+ :type mount_point: str
+ :param kv_engine_version: Select the version of the engine to run (``1`` or ``2``, default: ``2``)
+ :type kv_engine_version: int
+ :param token: Authentication token to include in requests sent to Vault.
+ (for ``token`` and ``github`` auth_type)
+ :type token: str
+ :param role_id: Role (for ``aws_iam``, 'approle', auth_types)
+ :type role_id: str
+ :param kubernetes_role: Role for Authentication (for ``kubernetes`` auth_type)
+ :type kubernetes_role: str
+ :param kubernetes_jwt_path: Path for kubernetes jwt token (for ``kubernetes`` auth_type, default:
+ ``/var/run/secrets/kubernetes.io/serviceaccount/token``)
+ :type kubernetes_jwt_path: str
+ :param gcp_key_path: Path to GCP Credential JSON file (for ``gcp`` auth_type)
+ :type gcp_key_path: str
+ :param gcp_keyfile_dict: Dictionary of keyfile parameters. (for ``gcp`` auth_type).
+ Mutually exclusive with gcp_key_path
+ :type gcp_keyfile_dict: dict
+ :param gcp_scopes: Comma-separated string containing GCP scopes (for ``gcp`` auth_type)
+ :type gcp_scopes: str
+ :param azure_tenant_id: Tenant of azure (for ``azure`` auth_type)
+ :type azure_tenant_id: str
+ :param azure_resource: Resource if of azure (for ``azure`` auth_type)
+ :type azure_resource: str
+ :param radius_host: Host for radius (for ``radius`` auth_type)
+ :type radius_host: str
+ :param radius_port: Port for radius (for ``radius`` auth_type)
+ :type radius_port: int
+
+ """
+ def __init__(self, # pylint: disable=too-many-arguments, too-many-branches
+ vault_conn_id=None,
+ auth_type=None,
+ mount_point=None,
+ kv_engine_version: Optional[int] = None,
+ token=None,
+ role_id=None,
+ kubernetes_role: Optional[str] = None,
+ kubernetes_jwt_path=None,
+ gcp_key_path=None,
+ gcp_keyfile_dict=None,
+ gcp_scopes=None,
+ azure_tenant_id: Optional[str] = None,
+ azure_resource: Optional[str] = None,
+ radius_host: Optional[str] = None,
+ radius_port: Optional[int] = None):
+ super().__init__()
+ connection = self.get_connection(vault_conn_id)
Review comment:
```suggestion
self.connection = self.get_connection(vault_conn_id)
```
##########
File path: airflow/providers/hashicorp/hooks/vault.py
##########
@@ -0,0 +1,281 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Hook for HashiCorp Vault"""
+import json
+from typing import Optional
+
+import hvac
+from hvac.exceptions import VaultError
+from requests import Response
+
+from airflow.hooks.base_hook import BaseHook
+from airflow.providers.hashicorp.common.vault_client import VaultClient
+
+
+class VaultHook(BaseHook):
+ """
+ HashiCorp Vault wrapper to Interact with HashiCorp Vault KeyValue Secret engine.
+
+ HashiCorp HVac documentation:
+ * https://hvac.readthedocs.io/en/stable/
+
+ You connect to the host specified as host in the connection. The login/password from the connection
+ are used as credentials usually and you can specify different authentication parameters
+ via init params or via corresponding extras in the connection.
+
+ The extras in the connection are named the same as the parameters (`mount_point`,'kv_engine_version' ...).
+
+ Login/Password are used as credentials:
+ * approle: password -> secret_id
+ * aws_iam: login -> key_id, password -> secret_id
+ * azure: login -> client_id, password -> client_secret
+ * ldap: login -> username, password -> password
+ * userpass: login -> username, password -> password
+ * radius: password -> radius_secret
+
+ :param vault_conn_id: id of the connection to use
+ :type vault_conn_id: str
+ :param auth_type: type of authentication. One of
+ :type auth_type: str
+ :param mount_point: The "path" the secret engine was mounted on. (Default: ``secret``)
+ :type mount_point: str
+ :param kv_engine_version: Select the version of the engine to run (``1`` or ``2``, default: ``2``)
+ :type kv_engine_version: int
+ :param token: Authentication token to include in requests sent to Vault.
+ (for ``token`` and ``github`` auth_type)
+ :type token: str
+ :param role_id: Role (for ``aws_iam``, 'approle', auth_types)
+ :type role_id: str
+ :param kubernetes_role: Role for Authentication (for ``kubernetes`` auth_type)
+ :type kubernetes_role: str
+ :param kubernetes_jwt_path: Path for kubernetes jwt token (for ``kubernetes`` auth_type, default:
+ ``/var/run/secrets/kubernetes.io/serviceaccount/token``)
+ :type kubernetes_jwt_path: str
+ :param gcp_key_path: Path to GCP Credential JSON file (for ``gcp`` auth_type)
+ :type gcp_key_path: str
+ :param gcp_keyfile_dict: Dictionary of keyfile parameters. (for ``gcp`` auth_type).
+ Mutually exclusive with gcp_key_path
+ :type gcp_keyfile_dict: dict
+ :param gcp_scopes: Comma-separated string containing GCP scopes (for ``gcp`` auth_type)
+ :type gcp_scopes: str
+ :param azure_tenant_id: Tenant of azure (for ``azure`` auth_type)
+ :type azure_tenant_id: str
+ :param azure_resource: Resource if of azure (for ``azure`` auth_type)
+ :type azure_resource: str
+ :param radius_host: Host for radius (for ``radius`` auth_type)
+ :type radius_host: str
+ :param radius_port: Port for radius (for ``radius`` auth_type)
+ :type radius_port: int
+
+ """
+ def __init__(self, # pylint: disable=too-many-arguments, too-many-branches
+ vault_conn_id=None,
+ auth_type=None,
+ mount_point=None,
+ kv_engine_version: Optional[int] = None,
+ token=None,
+ role_id=None,
+ kubernetes_role: Optional[str] = None,
+ kubernetes_jwt_path=None,
+ gcp_key_path=None,
+ gcp_keyfile_dict=None,
+ gcp_scopes=None,
+ azure_tenant_id: Optional[str] = None,
+ azure_resource: Optional[str] = None,
+ radius_host: Optional[str] = None,
+ radius_port: Optional[int] = None):
+ super().__init__()
+ connection = self.get_connection(vault_conn_id)
+
+ if not auth_type:
+ auth_type = connection.extra_dejson.get('auth_type')
+
+ if not mount_point:
+ mount_point = connection.extra_dejson.get('mount_point')
+
+ if not kv_engine_version:
+ conn_version = connection.extra_dejson.get("kv_engine_version")
+ try:
+ kv_engine_version = int(conn_version) if conn_version else 2
+ except ValueError:
+ raise VaultError(f"The version is not an int: {conn_version}. ")
+
+ if auth_type in ["aws_iam", "approle"] and not role_id:
+ role_id = connection.extra_dejson.get('role_id')
+
+ if auth_type in ["token", "github"] and not token:
+ token = connection.extra_dejson.get('token')
+
+ if auth_type == "azure":
+ if not azure_tenant_id:
+ azure_tenant_id = connection.extra_dejson.get("azure_tenant_id")
+
+ if not azure_resource:
+ azure_resource = connection.extra_dejson.get("azure_resource")
+
+ elif auth_type == "gcp":
+ if not gcp_scopes:
+ gcp_scopes = connection.extra_dejson.get("gcp_scopes")
+
+ if not gcp_key_path:
+ gcp_key_path = connection.extra_dejson.get("gcp_key_path")
+
+ if not gcp_keyfile_dict:
+ string_keyfile_dict = connection.extra_dejson.get("gcp_keyfile_dict")
+ if string_keyfile_dict:
+ gcp_keyfile_dict = json.loads(string_keyfile_dict)
+
+ elif auth_type == "kubernetes":
+ if not kubernetes_jwt_path:
+ kubernetes_jwt_path = connection.extra_dejson.get("kubernetes_jwt_path")
+ if not kubernetes_role:
+ kubernetes_role = connection.extra_dejson.get("kubernetes_role")
+
+ elif auth_type == "radius": # pylint: disable=too-many-nested-blocks
+ if not radius_port:
+ radius_port_str = connection.extra_dejson.get("radius_port")
+ if radius_port_str:
+ try:
+ radius_port = int(radius_port_str)
+ except ValueError:
+ raise VaultError(f"Radius port was wrong: {radius_port_str}")
+ if not radius_host:
+ radius_host = connection.extra_dejson.get("radius_host")
+
+ url = f"{connection.schema}://{connection.host}"
+ if connection.port:
+ url += f":{connection.port}"
+
+ self.vault_client = VaultClient(
+ url=url,
+ auth_type=auth_type,
+ mount_point=mount_point,
+ kv_engine_version=kv_engine_version,
+ token=token,
+ username=connection.login,
+ password=connection.password,
+ key_id=connection.login,
+ secret_id=connection.password,
+ role_id=role_id,
+ kubernetes_role=kubernetes_role,
+ kubernetes_jwt_path=kubernetes_jwt_path,
+ gcp_key_path=gcp_key_path,
+ gcp_keyfile_dict=gcp_keyfile_dict,
+ gcp_scopes=gcp_scopes,
+ azure_tenant_id=azure_tenant_id,
+ azure_resource=azure_resource,
+ radius_host=radius_host,
+ radius_secret=connection.password,
+ radius_port=radius_port
+ )
+
+ def get_conn(self) -> hvac.Client:
+ """
+ Retrieves connection to Vault.
+
+ :rtype: hvac.Client
+ :return: connection used.
+ """
+ return self.vault_client.client
+
+ def get_secret(self, secret_path: str, secret_version: Optional[int] = None) -> Optional[dict]:
+ """
+ Get secret value from the engine.
+
+ :param secret_path: path of the secret
+ :type secret_path: str
+ :param secret_version: optional version of key to read - can only be used in case of version 2 of KV
+ :type secret_version: int
+
+ See https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v1.html
+ and https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v2.html for details.
+
+ :param secret_path: path of the secret
+ :type secret_path: str
+ :rtype: dict
+ :return: secret stored in the vault as a dictionary
+ """
+ return self.vault_client.get_secret(secret_path=secret_path, secret_version=secret_version)
+
+ def get_secret_metadata(self, secret_path: str) -> Optional[dict]:
+ """
+ Reads secret metadata (including versions) from the engine. It is only valid for KV version 2.
+
+ :param secret_path: path to read from
+ :type secret_path: str
+ :rtype: dict
+ :return: secret metadata. This is a Dict containing metadata for the secret.
+
+ See https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v2.html for details.
+
+ """
+ return self.vault_client.get_secret_metadata(secret_path=secret_path)
+
+ def get_secret_including_metadata(self,
+ secret_path: str,
+ secret_version: Optional[int] = None) -> Optional[dict]:
+ """
+ Reads secret including metadata. It is only valid for KV version 2.
+
+ See https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v2.html for details.
+
+ :param secret_path: path of the secret
+ :type secret_path: str
+ :param secret_version: optional version of key to read - can only be used in case of version 2 of KV
+ :type secret_version: int
+ :rtype: dict
+ :return: key info. This is a Dict with "data" mapping keeping secret
+ and "metadata" mapping keeping metadata of the secret.
+
+ """
+ return self.vault_client.get_secret_including_metadata(
+ secret_path=secret_path, secret_version=secret_version)
+
+ def create_or_update_secret(self,
+ secret_path: str,
+ secret: dict,
+ method: Optional[str] = None,
+ cas: Optional[int] = None) -> Response:
+ """
+ Creates or updates secret.
+
+ :param secret_path: path to read from
Review comment:
```suggestion
:param secret_path: Path to read from
```
##########
File path: airflow/providers/hashicorp/hooks/vault.py
##########
@@ -0,0 +1,281 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Hook for HashiCorp Vault"""
+import json
+from typing import Optional
+
+import hvac
+from hvac.exceptions import VaultError
+from requests import Response
+
+from airflow.hooks.base_hook import BaseHook
+from airflow.providers.hashicorp.common.vault_client import VaultClient
+
+
+class VaultHook(BaseHook):
+ """
+ HashiCorp Vault wrapper to Interact with HashiCorp Vault KeyValue Secret engine.
+
+ HashiCorp HVac documentation:
+ * https://hvac.readthedocs.io/en/stable/
+
+ You connect to the host specified as host in the connection. The login/password from the connection
+ are used as credentials usually and you can specify different authentication parameters
+ via init params or via corresponding extras in the connection.
+
+ The extras in the connection are named the same as the parameters (`mount_point`,'kv_engine_version' ...).
+
+ Login/Password are used as credentials:
+ * approle: password -> secret_id
+ * aws_iam: login -> key_id, password -> secret_id
+ * azure: login -> client_id, password -> client_secret
+ * ldap: login -> username, password -> password
+ * userpass: login -> username, password -> password
+ * radius: password -> radius_secret
+
+ :param vault_conn_id: id of the connection to use
+ :type vault_conn_id: str
+ :param auth_type: type of authentication. One of
+ :type auth_type: str
+ :param mount_point: The "path" the secret engine was mounted on. (Default: ``secret``)
+ :type mount_point: str
+ :param kv_engine_version: Select the version of the engine to run (``1`` or ``2``, default: ``2``)
+ :type kv_engine_version: int
+ :param token: Authentication token to include in requests sent to Vault.
+ (for ``token`` and ``github`` auth_type)
+ :type token: str
+ :param role_id: Role (for ``aws_iam``, 'approle', auth_types)
+ :type role_id: str
+ :param kubernetes_role: Role for Authentication (for ``kubernetes`` auth_type)
+ :type kubernetes_role: str
+ :param kubernetes_jwt_path: Path for kubernetes jwt token (for ``kubernetes`` auth_type, default:
+ ``/var/run/secrets/kubernetes.io/serviceaccount/token``)
+ :type kubernetes_jwt_path: str
+ :param gcp_key_path: Path to GCP Credential JSON file (for ``gcp`` auth_type)
+ :type gcp_key_path: str
+ :param gcp_keyfile_dict: Dictionary of keyfile parameters. (for ``gcp`` auth_type).
+ Mutually exclusive with gcp_key_path
+ :type gcp_keyfile_dict: dict
+ :param gcp_scopes: Comma-separated string containing GCP scopes (for ``gcp`` auth_type)
+ :type gcp_scopes: str
+ :param azure_tenant_id: Tenant of azure (for ``azure`` auth_type)
+ :type azure_tenant_id: str
+ :param azure_resource: Resource if of azure (for ``azure`` auth_type)
+ :type azure_resource: str
+ :param radius_host: Host for radius (for ``radius`` auth_type)
+ :type radius_host: str
+ :param radius_port: Port for radius (for ``radius`` auth_type)
+ :type radius_port: int
+
+ """
+ def __init__(self, # pylint: disable=too-many-arguments, too-many-branches
+ vault_conn_id=None,
+ auth_type=None,
+ mount_point=None,
+ kv_engine_version: Optional[int] = None,
+ token=None,
+ role_id=None,
+ kubernetes_role: Optional[str] = None,
+ kubernetes_jwt_path=None,
+ gcp_key_path=None,
+ gcp_keyfile_dict=None,
+ gcp_scopes=None,
+ azure_tenant_id: Optional[str] = None,
+ azure_resource: Optional[str] = None,
+ radius_host: Optional[str] = None,
+ radius_port: Optional[int] = None):
+ super().__init__()
+ connection = self.get_connection(vault_conn_id)
+
+ if not auth_type:
+ auth_type = connection.extra_dejson.get('auth_type')
+
+ if not mount_point:
+ mount_point = connection.extra_dejson.get('mount_point')
+
+ if not kv_engine_version:
+ conn_version = connection.extra_dejson.get("kv_engine_version")
+ try:
+ kv_engine_version = int(conn_version) if conn_version else 2
+ except ValueError:
+ raise VaultError(f"The version is not an int: {conn_version}. ")
+
+ if auth_type in ["aws_iam", "approle"] and not role_id:
+ role_id = connection.extra_dejson.get('role_id')
+
+ if auth_type in ["token", "github"] and not token:
+ token = connection.extra_dejson.get('token')
+
+ if auth_type == "azure":
+ if not azure_tenant_id:
+ azure_tenant_id = connection.extra_dejson.get("azure_tenant_id")
+
+ if not azure_resource:
+ azure_resource = connection.extra_dejson.get("azure_resource")
+
+ elif auth_type == "gcp":
+ if not gcp_scopes:
+ gcp_scopes = connection.extra_dejson.get("gcp_scopes")
+
+ if not gcp_key_path:
+ gcp_key_path = connection.extra_dejson.get("gcp_key_path")
+
+ if not gcp_keyfile_dict:
+ string_keyfile_dict = connection.extra_dejson.get("gcp_keyfile_dict")
+ if string_keyfile_dict:
+ gcp_keyfile_dict = json.loads(string_keyfile_dict)
+
+ elif auth_type == "kubernetes":
+ if not kubernetes_jwt_path:
+ kubernetes_jwt_path = connection.extra_dejson.get("kubernetes_jwt_path")
+ if not kubernetes_role:
+ kubernetes_role = connection.extra_dejson.get("kubernetes_role")
+
+ elif auth_type == "radius": # pylint: disable=too-many-nested-blocks
+ if not radius_port:
+ radius_port_str = connection.extra_dejson.get("radius_port")
+ if radius_port_str:
+ try:
+ radius_port = int(radius_port_str)
+ except ValueError:
+ raise VaultError(f"Radius port was wrong: {radius_port_str}")
+ if not radius_host:
+ radius_host = connection.extra_dejson.get("radius_host")
+
+ url = f"{connection.schema}://{connection.host}"
+ if connection.port:
+ url += f":{connection.port}"
+
+ self.vault_client = VaultClient(
+ url=url,
+ auth_type=auth_type,
+ mount_point=mount_point,
+ kv_engine_version=kv_engine_version,
+ token=token,
+ username=connection.login,
+ password=connection.password,
+ key_id=connection.login,
+ secret_id=connection.password,
+ role_id=role_id,
+ kubernetes_role=kubernetes_role,
+ kubernetes_jwt_path=kubernetes_jwt_path,
+ gcp_key_path=gcp_key_path,
+ gcp_keyfile_dict=gcp_keyfile_dict,
+ gcp_scopes=gcp_scopes,
+ azure_tenant_id=azure_tenant_id,
+ azure_resource=azure_resource,
+ radius_host=radius_host,
+ radius_secret=connection.password,
+ radius_port=radius_port
+ )
+
+ def get_conn(self) -> hvac.Client:
+ """
+ Retrieves connection to Vault.
+
+ :rtype: hvac.Client
+ :return: connection used.
+ """
+ return self.vault_client.client
+
+ def get_secret(self, secret_path: str, secret_version: Optional[int] = None) -> Optional[dict]:
+ """
+ Get secret value from the engine.
+
+ :param secret_path: path of the secret
+ :type secret_path: str
+ :param secret_version: optional version of key to read - can only be used in case of version 2 of KV
+ :type secret_version: int
+
+ See https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v1.html
+ and https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v2.html for details.
+
+ :param secret_path: path of the secret
+ :type secret_path: str
+ :rtype: dict
+ :return: secret stored in the vault as a dictionary
+ """
+ return self.vault_client.get_secret(secret_path=secret_path, secret_version=secret_version)
+
+ def get_secret_metadata(self, secret_path: str) -> Optional[dict]:
+ """
+ Reads secret metadata (including versions) from the engine. It is only valid for KV version 2.
+
+ :param secret_path: path to read from
+ :type secret_path: str
+ :rtype: dict
+ :return: secret metadata. This is a Dict containing metadata for the secret.
+
+ See https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v2.html for details.
+
+ """
+ return self.vault_client.get_secret_metadata(secret_path=secret_path)
+
+ def get_secret_including_metadata(self,
+ secret_path: str,
+ secret_version: Optional[int] = None) -> Optional[dict]:
+ """
+ Reads secret including metadata. It is only valid for KV version 2.
Review comment:
```suggestion
Reads secret including metadata. It is only valid for KV version 2.
```
##########
File path: airflow/providers/hashicorp/secrets/vault.py
##########
@@ -72,98 +68,90 @@ class VaultBackend(BaseSecretsBackend, LoggingMixin):
:type username: str
:param password: Password for Authentication (for ``ldap`` and ``userpass`` auth_type)
:type password: str
- :param role_id: Role ID for Authentication (for ``approle`` auth_type)
+ :param key_id: Key ID for Authentication (for ``aws_iam`` and ''azure`` auth_type)
+ :type key_id: str
+ :param secret_id: Secret ID for Authentication (for ``approle``, ``aws_iam`` and ``azure`` auth_types)
+ :type secret_id: str
:type role_id: str
+ :param secret_id: Secret ID for Authentication (for ``approle`` auth_type)
+ :type secret_id: str
Review comment:
Duplicate of above
##########
File path: airflow/providers/hashicorp/secrets/vault.py
##########
@@ -72,98 +68,90 @@ class VaultBackend(BaseSecretsBackend, LoggingMixin):
:type username: str
:param password: Password for Authentication (for ``ldap`` and ``userpass`` auth_type)
:type password: str
- :param role_id: Role ID for Authentication (for ``approle`` auth_type)
+ :param key_id: Key ID for Authentication (for ``aws_iam`` and ''azure`` auth_type)
+ :type key_id: str
+ :param secret_id: Secret ID for Authentication (for ``approle``, ``aws_iam`` and ``azure`` auth_types)
+ :type secret_id: str
:type role_id: str
+ :param secret_id: Secret ID for Authentication (for ``approle`` auth_type)
+ :type secret_id: str
:param kubernetes_role: Role for Authentication (for ``kubernetes`` auth_type)
:type kubernetes_role: str
- :param kubernetes_jwt_path: Path for kubernetes jwt token (for ``kubernetes`` auth_type, deafult:
+ :param kubernetes_jwt_path: Path for kubernetes jwt token (for ``kubernetes`` auth_type, default:
``/var/run/secrets/kubernetes.io/serviceaccount/token``)
:type kubernetes_jwt_path: str
- :param secret_id: Secret ID for Authentication (for ``approle`` auth_type)
- :type secret_id: str
:param gcp_key_path: Path to GCP Credential JSON file (for ``gcp`` auth_type)
+ Mutually exclusive with gcp_keyfile_dict
:type gcp_key_path: str
+ :param gcp_keyfile_dict: Dictionary of keyfile parameters. (for ``gcp`` auth_type).
+ Mutually exclusive with gcp_key_path
+ :type gcp_keyfile_dict: dict
:param gcp_scopes: Comma-separated string containing GCP scopes (for ``gcp`` auth_type)
:type gcp_scopes: str
+ :param azure_tenant_id: Tenant of azure (for ``azure`` auth_type)
+ :type azure_tenant_id: str
+ :param azure_resource: Resource if of azure (for ``azure`` auth_type)
+ :type azure_resource: str
+ :param radius_host: Host for radius (for ``radius`` auth_type)
+ :type radius_host: str
+ :param radius_secret: Secret for radius (for ``radius`` auth_type)
+ :type radius_secret: str
+ :param radius_port: Port for radius (for ``radius`` auth_type)
+ :type radius_port: str
"""
def __init__( # pylint: disable=too-many-arguments
self,
connections_path: str = 'connections',
variables_path: str = 'variables',
url: Optional[str] = None,
auth_type: str = 'token',
- mount_point: str = 'secret',
+ mount_point: Optional[str] = None,
kv_engine_version: int = 2,
token: Optional[str] = None,
username: Optional[str] = None,
password: Optional[str] = None,
+ key_id: Optional[str] = None,
+ secret_id: Optional[str] = None,
role_id: Optional[str] = None,
kubernetes_role: Optional[str] = None,
kubernetes_jwt_path: str = '/var/run/secrets/kubernetes.io/serviceaccount/token',
- secret_id: Optional[str] = None,
gcp_key_path: Optional[str] = None,
+ gcp_keyfile_dict=None,
gcp_scopes: Optional[str] = None,
+ azure_tenant_id: Optional[str] = None,
+ azure_resource: Optional[str] = None,
+ radius_host: Optional[str] = None,
+ radius_secret: Optional[str] = None,
+ radius_port: Optional[int] = None,
**kwargs
):
super().__init__(**kwargs)
self.connections_path = connections_path.rstrip('/')
self.variables_path = variables_path.rstrip('/')
- self.url = url
- self.auth_type = auth_type
- self.kwargs = kwargs
- self.token = token
- self.username = username
- self.password = password
- self.role_id = role_id
- self.kubernetes_role = kubernetes_role
- self.kubernetes_jwt_path = kubernetes_jwt_path
- self.secret_id = secret_id
self.mount_point = mount_point
self.kv_engine_version = kv_engine_version
- self.gcp_key_path = gcp_key_path
- self.gcp_scopes = gcp_scopes
-
- @cached_property
- def client(self) -> hvac.Client:
- """
- Return an authenticated Hashicorp Vault client
- """
-
- _client = hvac.Client(url=self.url, **self.kwargs)
Review comment:
`**self.kwargs` allowed passing any kwargs (via airflow.cfg) to the hvac Vault client which we should preserve
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org