[GitHub] [airflow] dstandish commented on a change in pull request #7741: [AIRFLOW-7076] Add support for HashiCorp Vault as Secrets Backend

[GitBox <gi...@apache.org>]

dstandish commented on a change in pull request #7741: [AIRFLOW-7076] Add support for HashiCorp Vault as Secrets Backend
URL: https://github.com/apache/airflow/pull/7741#discussion_r393470882
 
 

 ##########
 File path: airflow/providers/hashicorp/secrets/vault.py
 ##########
 @@ -0,0 +1,197 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Objects relating to sourcing connections from Hashicorp Vault
+"""
+from typing import List, Optional
+
+import hvac
+from hvac.exceptions import VaultError
+
+from airflow import AirflowException
+from airflow.models import Connection
+from airflow.secrets import CONN_ENV_PREFIX, BaseSecretsBackend
+from airflow.utils.log.logging_mixin import LoggingMixin
+
+
+class VaultSecrets(BaseSecretsBackend, LoggingMixin):
+    """
+    Retrieves Connection object from Hashicorp Vault Secrets
+
+    Configurable via ``airflow.cfg`` as follows:
+
+    .. code-block:: ini
+
+        [secrets]
+        backend = airflow.providers.hashicorp.secrets.vault.VaultSecrets
+        backend_kwargs = {"path": "airflow", "url": "http://127.0.0.1:8200"}
+
+    For example, if your keys are under ``airflow/AIRFLOW_CONN_SMTP_DEFAULT``, this would be accessible if you
+    provide ``{"path": "airflow"}`` and request conn_id ``smtp_default``.
+
+    :param path: Specifies the path of the secret to read.
+    :type path: str
+    :param url: Base URL for the Vault instance being addressed.
+    :type url: str
+    :param auth_type: Authentication Type for Vault (one of 'token', 'ldap', 'userpass', 'approle',
+        'github', 'gcp). Default is ``token``.
+    :type auth_type: str
+    :param mount_point: The "path" the secret engine was mounted on. (Default: ``secret``)
+    :type mount_point: str
+    :param token: Authentication token to include in requests sent to Vault.
+        (for ``token`` and ``github`` auth_type)
+    :type token: str
+    :param username: Username for Authentication (for ``ldap`` and ``userpass`` auth_type)
+    :type username: str
+    :param password: Password for Authentication (for ``ldap`` and ``userpass`` auth_type)
+    :type password: str
+    :param role_id: Role ID for Authentication (for ``approle`` auth_type)
+    :type role_id: str
+    :param secret_id: Secret ID for Authentication (for ``approle`` auth_type)
+    :type secret_id: str
+    :param gcp_key_path: Path to GCP Credential JSON file (for ``gcp`` auth_type)
+    :type gcp_key_path: str
+    :param gcp_scopes: Comma-separated string containing GCP scopes (for ``gcp`` auth_type)
+    :type gcp_scopes: str
+    """
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        path: str,
+        url: Optional[str] = None,
+        auth_type: str = 'token',
+        mount_point: str = 'secret',
+        token: Optional[str] = None,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        role_id: Optional[str] = None,
+        secret_id: Optional[str] = None,
+        gcp_key_path: Optional[str] = None,
+        gcp_scopes: Optional[str] = None,
+        **kwargs
+    ):
+        super().__init__()
+        self.path = path.rstrip('/')
+        self.url = url
+        self.auth_type = auth_type
+        self.kwargs = kwargs
+        self.token = token
+        self.username = username
+        self.password = password
+        self.role_id = role_id
+        self.secret_id = secret_id
+        self.mount_point = mount_point
+        self.gcp_key_path = gcp_key_path
+        self.gcp_scopes = gcp_scopes
+        self._client: Optional[hvac.Client] = None
+
+    def get_client(self) -> hvac.Client:
+        """
+        Return an authenticated Hashicorp Vault client
+        """
+        if not self._client:
+            self._client = hvac.Client(url=self.url, **self.kwargs)
+            if self.auth_type == "token":
+                self._client.token = self.token
+            elif self.auth_type == "ldap":
+                self._client.auth.ldap.login(
+                    username=self.username, password=self.password)
+            elif self.auth_type == "userpass":
+                self._client.auth_userpass(username=self.username, password=self.password)
+            elif self.auth_type == "approle":
+                self._client.auth_approle(role_id=self.role_id, secret_id=self.secret_id)
+            elif self.auth_type == "github":
+                self._client.auth.github.login(token=self.token)
+            elif self.auth_type == "gcp":
+                credentials = self._get_gcp_credentials()
+                self._client.auth.gcp.configure(credentials=credentials)
+            else:
+                raise AirflowException(f"Authentication type '{self.auth_type}' not supported")
+
+            if self._client.is_authenticated():
+                return self._client
+            else:
+                raise VaultError("Vault Authentication Error!")
+        else:
+            return self._client
+
+    def get_conn_uri(self, conn_id: str) -> str:
+        """
+        Get secret value from Vault. Store the secret in the form of URI
+
+        :param conn_id: connection id
+        :type conn_id: str
+        """
+        connection_id = f"{CONN_ENV_PREFIX}{conn_id}".upper()
+        client = self.get_client()
+
+        self.log.debug("Path: %s", self.path)
+        self.log.debug("Mount Point: %s", self.mount_point)
+        self.log.debug("Retrieving the secret for Connection ID: %s", connection_id)
+
+        response = client.secrets.kv.v2.read_secret_version(
+            path=self.path, mount_point=self.mount_point)
+        return_data = response["data"]["data"]
+
+        try:
+            value = return_data[connection_id]
+            self.log.debug("Value of the secret: %s", return_data)
+        except KeyError:
 
 Review comment:
   @kaxil if you raise here, then i think airflow will not continue to search in env vars then in metastore; it will just stop here and raise.
   
   see https://github.com/apache/airflow/blob/master/airflow/secrets/__init__.py#L62
   

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services