You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by el...@apache.org on 2023/02/11 02:00:40 UTC
[airflow] branch main updated: Make known deprecated warnings more resilient to changes (#29471)
This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c44b24ec29 Make known deprecated warnings more resilient to changes (#29471)
c44b24ec29 is described below
commit c44b24ec29673886caa15325c55d3753ae66cc5f
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Feb 11 03:00:27 2023 +0100
Make known deprecated warnings more resilient to changes (#29471)
Some of the deprecated warning messages we "know" and deliberately
ignore as "known" in provider import verification, had variable
parts that kept on changing when new version of dependencies
(for exmaple snowflake library) has been released. The known
warning already supports substring match for a while, this PR just
turns the known warnings with variable parts into substrings.
Shortening the messages and turning them into one-liners makes it
also cleaner to read and understand how many warnings we ignore,
as well as allows to remove some duplication.
---
scripts/in_container/verify_providers.py | 151 ++++++-------------------------
1 file changed, 28 insertions(+), 123 deletions(-)
diff --git a/scripts/in_container/verify_providers.py b/scripts/in_container/verify_providers.py
index c952673807..4e9e2d596a 100755
--- a/scripts/in_container/verify_providers.py
+++ b/scripts/in_container/verify_providers.py
@@ -116,118 +116,32 @@ EXPECTED_SUFFIXES: dict[EntityType, str] = {
# It contains tuples of "message" and the module that generates the warning - so when the
# Same warning is generated by different module, it is not treated as "known" warning.
KNOWN_DEPRECATED_MESSAGES: set[tuple[str, str]] = {
- (
- "This version of Apache Beam has not been sufficiently tested on Python 3.9. "
- "You may encounter bugs or missing features.",
- "apache_beam",
- ),
- (
- "This version of Apache Beam has not been sufficiently tested on Python 3.10. "
- "You may encounter bugs or missing features.",
- "apache_beam",
- ),
- (
- "Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since"
- " Python 3.3,and in 3.9 it will stop working",
- "apache_beam",
- ),
- (
- "pyarrow.HadoopFileSystem is deprecated as of 2.0.0, please use pyarrow.fs.HadoopFileSystem instead.",
- "papermill",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (4.0.1), please install a version that "
- "adheres to: 'pyarrow<3.1.0,>=3.0.0; extra == \"pandas\"'",
- "apache_beam",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (4.0.1), please install a version that "
- "adheres to: 'pyarrow<5.1.0,>=5.0.0; extra == \"pandas\"'",
- "snowflake",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (6.0.1), please install a version that "
- "adheres to: 'pyarrow<8.1.0,>=8.0.0; extra == \"pandas\"'",
- "snowflake",
- ),
- ("dns.hash module will be removed in future versions. Please use hashlib instead.", "dns"),
- ("PKCS#7 support in pyOpenSSL is deprecated. You should use the APIs in cryptography.", "eventlet"),
- ("PKCS#12 support in pyOpenSSL is deprecated. You should use the APIs in cryptography.", "eventlet"),
- (
- "the imp module is deprecated in favour of importlib; see the module's documentation"
- " for alternative uses",
- "hdfs",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (4.0.1), please install a version that"
- " adheres to: 'pyarrow<3.1.0,>=3.0.0; extra == \"pandas\"'",
- "snowflake",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (5.0.0), please install a version that"
- " adheres to: 'pyarrow<6.1.0,>=6.0.0; extra == \"pandas\"'",
- "snowflake",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (6.0.1), please install a version that"
- " adheres to: 'pyarrow<5.1.0,>=5.0.0; extra == \"pandas\"'",
- "snowflake",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (6.0.1), please install a version that"
- " adheres to: 'pyarrow<3.1.0,>=3.0.0; extra == \"pandas\"'",
- "snowflake",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (9.0.0), please install a version that"
- " adheres to: 'pyarrow<8.1.0,>=8.0.0; extra == \"pandas\"'",
- "snowflake",
- ),
- ("SelectableGroups dict interface is deprecated. Use select.", "kombu"),
- ("The module cloudant is now deprecated. The replacement is ibmcloudant.", "cloudant"),
- (
- "'nteract-scrapbook' package has been renamed to `scrapbook`. No new releases are "
- "going out for this old package name.",
- "scrapbook",
- ),
+ ("This version of Apache Beam has not been sufficiently tested on Python", "apache_beam"),
+ ("Using or importing the ABCs from 'collections' instead of from 'collections.abc'", "apache_beam"),
+ ("pyarrow.HadoopFileSystem is deprecated as of", "papermill"),
+ ("You have an incompatible version of 'pyarrow' installed", "snowflake"),
+ ("dns.hash module will be removed in future versions.", "dns"),
+ ("PKCS#7 support in pyOpenSSL is deprecated.", "eventlet"),
+ ("PKCS#12 support in pyOpenSSL is deprecated.", "eventlet"),
+ ("the imp module is deprecated in favour of importlib", "hdfs"),
+ ("SelectableGroups dict interface is deprecated.", "kombu"),
+ ("The module cloudant is now deprecated.", "cloudant"),
+ ("'nteract-scrapbook' package has been renamed to `scrapbook`", "scrapbook"),
("SelectableGroups dict interface is deprecated. Use select.", "markdown"),
- ("'_app_ctx_stack' is deprecated and will be removed in Flask 2.3.", "flask_sqlalchemy"),
- ("'_app_ctx_stack' is deprecated and will be removed in Flask 2.3.", "flask_appbuilder"),
- # Currently (2.2) Flask app builder has the `remoevd` typo in the messages,
- # and they might want to fix it, so adding both
- ("'_request_ctx_stack' is deprecated and will be remoevd in Flask 2.3.", "flask_appbuilder"),
- ("'_request_ctx_stack' is deprecated and will be removed in Flask 2.3.", "flask_appbuilder"),
- ("'_request_ctx_stack' is deprecated and will be removed in Flask 2.3.", "flask_jwt_extended"),
- (
- "'urllib3.contrib.pyopenssl' module is deprecated and will be removed in a future release of "
- "urllib3 2.x. Read more in this issue: https://github.com/urllib3/urllib3/issues/2680",
- "azure/datalake/store",
- ),
- (
- "'urllib3.contrib.pyopenssl' module is deprecated and will be removed in a future release of "
- "urllib3 2.x. Read more in this issue: https://github.com/urllib3/urllib3/issues/2680",
- "botocore",
- ),
- (
- "'urllib3.contrib.pyopenssl' module is deprecated and will be removed in a future release of "
- "urllib3 2.x. Read more in this issue: https://github.com/urllib3/urllib3/issues/2680",
- "requests_toolbelt",
- ),
- (
- "zmq.eventloop.ioloop is deprecated in pyzmq 17. pyzmq now works with default tornado and asyncio "
- "eventloops.",
- "jupyter_client",
- ),
+ ("'_app_ctx_stack' is deprecated and will be", "flask_sqlalchemy"),
+ ("'_request_ctx_stack' is deprecated and will be", "flask_appbuilder"),
+ ("'_request_ctx_stack' is deprecated and will be", "flask_jwt_extended"),
+ ("'urllib3.contrib.pyopenssl' module is deprecated and will be", "azure/datalake/store"),
+ ("'urllib3.contrib.pyopenssl' module is deprecated and will be", "botocore"),
+ ("'urllib3.contrib.pyopenssl' module is deprecated and will be", "requests_toolbelt"),
+ ("zmq.eventloop.ioloop is deprecated in pyzmq 17.", "jupyter_client"),
}
KNOWN_COMMON_DEPRECATED_MESSAGES: set[str] = {
"distutils Version classes are deprecated. Use packaging.version instead.",
- "the imp module is deprecated in favour of importlib; "
- "see the module's documentation for alternative uses",
- "Param `schedule_interval` is deprecated and will be removed in a future release. "
- "Please use `schedule` instead. ",
- "'urllib3.contrib.pyopenssl' module is deprecated and will be removed in a future "
- "release of urllib3 2.x. Read more in this issue: https://github.com/urllib3/urllib3/issues/2680",
+ "the imp module is deprecated in favour of importlib",
+ "Param `schedule_interval` is deprecated and will be removed in a future release.",
+ "'urllib3.contrib.pyopenssl' module is deprecated",
"Deprecated API features detected! These feature(s) are not compatible with SQLAlchemy 2.0",
}
@@ -237,21 +151,11 @@ KNOWN_COMMON_DEPRECATED_MESSAGES: set[str] = {
KNOWN_DEPRECATED_DIRECT_IMPORTS: set[str] = {
"This module is deprecated. Please use `kubernetes.client.models.V1Volume`.",
"This module is deprecated. Please use `kubernetes.client.models.V1VolumeMount`.",
- (
- "This module is deprecated. Please use `kubernetes.client.models.V1ResourceRequirements`"
- " and `kubernetes.client.models.V1ContainerPort`."
- ),
+ "This module is deprecated. Please use `kubernetes.client.models.V1ResourceRequirements`",
"This module is deprecated. Please use `kubernetes.client.models.V1EnvVar`.",
- "numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header,"
- " got 216 from PyObject",
+ "numpy.ufunc size changed, may indicate binary incompatibility.",
"This module is deprecated. Please use `airflow.providers.amazon.aws.operators.lambda_function`.",
- (
- """
- S3ToSnowflakeOperator is deprecated.
- Please use
- `airflow.providers.snowflake.transfers.copy_into_snowflake.CopyFromExternalStageToSnowflakeOperator`.
- """
- ),
+ "S3ToSnowflakeOperator is deprecated.",
}
@@ -789,15 +693,16 @@ def summarise_total_vs_bad_and_warnings(total: int, bad: int, warns: list[warnin
)
console.print()
console.print(
- "[yellow]In case 1), add the deprecation message to "
+ "[yellow]In case 1), add distinctive subset of the deprecation message to "
"the KNOWN_DEPRECATED_DIRECT_IMPORTS in ./scripts/ci/in_container/verify_providers.py[/]"
)
console.print(
- "[yellow]In case 2), add the deprecation message together with module it generates to "
+ "[yellow]In case 2), add distinctive subset of the deprecation message together with "
+ "module it is generates from to "
"the KNOWN_DEPRECATED_MESSAGES in ./scripts/ci/in_container/verify_providers.py[/]"
)
console.print(
- "[yellow]In case 3), add the deprecation message to "
+ "[yellow]In case 3), add distinctive subset of the deprecation message to "
"the KNOWN_COMMON_DEPRECATED_MESSAGES in ./scripts/ci/in_container/verify_providers.py[/]"
)
console.print()