You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/03/09 10:33:48 UTC
[airflow] 01/04: Move help message to the google auth code (#29888)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d6b92bfb25ec0628569a171c9c906bbe04d69a19
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Mar 3 11:11:23 2023 +0100
Move help message to the google auth code (#29888)
The "google-auth" 2.16.2 just released, removed the _HELP_MESSAGE
that was imported by google provider auth util thus failing the
imports in canary builds of ours attempting to upgrade to
newver versions of released libraries.
This PR inlines the original help message into our code to make
it independent from google-auth package version used.
(cherry picked from commit 47ab0ca6295f301d393dbb29e134fa90be0ad7df)
---
...et_candidate_file_descriptor_ranges-to-us.patch | 76 ++++++++++++++++++++++
airflow/_vendor/daemon/module.mk | 27 ++++++++
.../google/common/utils/id_token_credentials.py | 26 +++++++-
.../common/utils/test_id_token_credentials.py | 7 +-
4 files changed, 128 insertions(+), 8 deletions(-)
diff --git a/airflow/_vendor/daemon/0001-Refactor-_get_candidate_file_descriptor_ranges-to-us.patch b/airflow/_vendor/daemon/0001-Refactor-_get_candidate_file_descriptor_ranges-to-us.patch
new file mode 100644
index 0000000000..75e9cb187d
--- /dev/null
+++ b/airflow/_vendor/daemon/0001-Refactor-_get_candidate_file_descriptor_ranges-to-us.patch
@@ -0,0 +1,76 @@
+From dd2f3d76c32ed12d56ae18b8ecb7086a8d9b1180 Mon Sep 17 00:00:00 2001
+From: Igor Kholopov <kh...@gmail.com>
+Date: Thu, 1 Dec 2022 18:34:30 +1100
+Subject: [PATCH 01/15] =?UTF-8?q?Refactor=20=E2=80=98=5Fget=5Fcandidate=5F?=
+ =?UTF-8?q?file=5Fdescriptor=5Franges=E2=80=99=20to=20use=20=E2=80=98range?=
+ =?UTF-8?q?=E2=80=99=20objects.?=
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+This avoids the requirement for a concrete set of all the actual file
+descriptors. When the range is very large, this can greatly improve the memory
+requirements and execution time of this function.
+---
+ daemon/daemon.py | 43 ++++++++++++++++++++++---------------------
+ 1 file changed, 22 insertions(+), 21 deletions(-)
+
+diff --git a/daemon/daemon.py b/daemon/daemon.py
+index 08245a4..d1673be 100644
+--- a/daemon/daemon.py
++++ b/daemon/daemon.py
+@@ -921,29 +921,30 @@ def _get_candidate_file_descriptor_ranges(exclude):
+ in this process, excluding those integers in the `exclude`
+ collection.
+ """
+- candidates_list = sorted(_get_candidate_file_descriptors(exclude))
++ _validate_fd_values(exclude)
+ ranges = []
+
+- def append_range_if_needed(candidate_range):
+- (low, high) = candidate_range
+- if (low < high):
+- # The range is not empty.
+- ranges.append(candidate_range)
+-
+- this_range = (
+- (min(candidates_list), (min(candidates_list) + 1))
+- if candidates_list else (0, 0))
+- for fd in candidates_list[1:]:
+- high = fd + 1
+- if this_range[1] == fd:
+- # This file descriptor extends the current range.
+- this_range = (this_range[0], high)
+- else:
+- # The previous range has ended at a gap.
+- append_range_if_needed(this_range)
+- # This file descriptor begins a new range.
+- this_range = (fd, high)
+- append_range_if_needed(this_range)
++ remaining_range = _total_file_descriptor_range
++ for exclude_fd in sorted(exclude):
++ if (exclude_fd > remaining_range.stop):
++ # This and all later exclusions are higher than the remaining
++ # range.
++ break
++ if (exclude_fd < remaining_range.start):
++ # The remaining range does not include the current exclusion.
++ continue
++ if (exclude_fd != remaining_range.start):
++ # There is a candidate range below the current exclusion.
++ ranges.append((remaining_range.start, exclude_fd))
++ # Narrow the remaining range to those above the current exclusion.
++ remaining_range = range(exclude_fd + 1, remaining_range.stop)
++
++ if (remaining_range.start < remaining_range.stop):
++ # After excluding all the specified file descriptors, there is a
++ # remaining range; append it as a candidate for closing file
++ # descriptors.
++ ranges.append((remaining_range.start, remaining_range.stop))
++
+ return ranges
+
+
+--
+2.34.1
+
diff --git a/airflow/_vendor/daemon/module.mk b/airflow/_vendor/daemon/module.mk
new file mode 100644
index 0000000000..063a7ca5a1
--- /dev/null
+++ b/airflow/_vendor/daemon/module.mk
@@ -0,0 +1,27 @@
+# daemon/module.mk
+# Part of ‘python-daemon’, an implementation of PEP 3143.
+#
+# This is free software, and you are welcome to redistribute it under
+# certain conditions; see the end of this file for copyright
+# information, grant of license, and disclaimer of warranty.
+
+# Makefile module for ‘daemon’ Python package.
+
+MODULE_DIR := $(CURDIR)/daemon
+
+CODE_MODULES += $(shell find ${MODULE_DIR}/ -name '*.py')
+
+
+# Copyright © 2006–2023 Ben Finney <be...@benfinney.id.au>
+#
+# This is free software: you may copy, modify, and/or distribute this work
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; version 3 of that license or any later version.
+# No warranty expressed or implied. See the file ‘LICENSE.GPL-3’ for details.
+
+
+# Local Variables:
+# mode: makefile
+# coding: utf-8
+# End:
+# vim: fileencoding=utf-8 filetype=make :
diff --git a/airflow/providers/google/common/utils/id_token_credentials.py b/airflow/providers/google/common/utils/id_token_credentials.py
index 6cac058992..73a3d588c3 100644
--- a/airflow/providers/google/common/utils/id_token_credentials.py
+++ b/airflow/providers/google/common/utils/id_token_credentials.py
@@ -36,9 +36,26 @@ import os
import google.auth.transport
import google.oauth2
from google.auth import credentials as google_auth_credentials, environment_vars, exceptions
-from google.auth._default import _AUTHORIZED_USER_TYPE, _HELP_MESSAGE, _SERVICE_ACCOUNT_TYPE, _VALID_TYPES
from google.oauth2 import credentials as oauth2_credentials, service_account
+# Valid types accepted for file-based credentials.
+# They are taken from "google.auth._default" and since they are all "protected" and the imports might
+# change any time and fail the whole Google provider functionality - we should inline them
+_AUTHORIZED_USER_TYPE = "authorized_user"
+_SERVICE_ACCOUNT_TYPE = "service_account"
+_EXTERNAL_ACCOUNT_TYPE = "external_account"
+_EXTERNAL_ACCOUNT_AUTHORIZED_USER_TYPE = "external_account_authorized_user"
+_IMPERSONATED_SERVICE_ACCOUNT_TYPE = "impersonated_service_account"
+_GDCH_SERVICE_ACCOUNT_TYPE = "gdch_service_account"
+_VALID_TYPES = (
+ _AUTHORIZED_USER_TYPE,
+ _SERVICE_ACCOUNT_TYPE,
+ _EXTERNAL_ACCOUNT_TYPE,
+ _EXTERNAL_ACCOUNT_AUTHORIZED_USER_TYPE,
+ _IMPERSONATED_SERVICE_ACCOUNT_TYPE,
+ _GDCH_SERVICE_ACCOUNT_TYPE,
+)
+
class IDTokenCredentialsAdapter(google_auth_credentials.Credentials):
"""Convert Credentials with ``openid`` scope to IDTokenCredentials."""
@@ -198,7 +215,12 @@ def get_default_id_token_credentials(
if current_credentials is not None:
return current_credentials
- raise exceptions.DefaultCredentialsError(_HELP_MESSAGE)
+ raise exceptions.DefaultCredentialsError(
+ f"""Could not automatically determine credentials. Please set {environment_vars.CREDENTIALS} or
+ explicitly create credentials and re-run the application. For more information, please see
+ https://cloud.google.com/docs/authentication/getting-started
+""".strip()
+ )
if __name__ == "__main__":
diff --git a/tests/providers/google/common/utils/test_id_token_credentials.py b/tests/providers/google/common/utils/test_id_token_credentials.py
index cf0d4c1f71..a3a4d88dcb 100644
--- a/tests/providers/google/common/utils/test_id_token_credentials.py
+++ b/tests/providers/google/common/utils/test_id_token_credentials.py
@@ -18,7 +18,6 @@ from __future__ import annotations
import json
import os
-import re
import unittest
from unittest import mock
@@ -61,11 +60,7 @@ class TestGetDefaultIdTokenCredentials(unittest.TestCase):
del os.environ[CREDENTIALS]
with pytest.raises(
exceptions.DefaultCredentialsError,
- match=re.escape(
- "Could not automatically determine credentials. Please set GOOGLE_APPLICATION_CREDENTIALS "
- "or explicitly create credentials and re-run the application. For more information, please "
- "see https://cloud.google.com/docs/authentication/getting-started"
- ),
+ match="Please set GOOGLE_APPLICATION_CREDENTIALS",
):
get_default_id_token_credentials(target_audience="example.org")