You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2021/11/03 22:38:40 UTC
[libcloud] 04/04: Merge branch 'trunk' into
understand-ai-intelligent-retry
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch understand-ai-intelligent-retry
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit 5c590956d7223ff914579a714dc3d1312c1fae67
Merge: bfda9fc 583cbca
Author: Tomaz Muraus <to...@tomaz.me>
AuthorDate: Wed Nov 3 22:21:09 2021 +0100
Merge branch 'trunk' into understand-ai-intelligent-retry
.github/actions/skip-duplicate-actions | 2 +-
.github/workflows/codeql-analysis.yml | 25 +
.github/workflows/install_test.yml | 42 +
.github/workflows/integration-tests.yml | 3 +-
.github/workflows/main.yml | 23 +-
.github/workflows/publish_pricing_to_s3.yml | 3 +-
CHANGES.rst | 103 +
MANIFEST.in | 1 +
contrib/scrape-ec2-prices.py | 15 +-
docs/compute/_supported_methods_block_storage.rst | 2 +-
.../_supported_methods_key_pair_management.rst | 2 +-
docs/compute/_supported_methods_main.rst | 2 +-
docs/compute/drivers/openstack.rst | 23 +-
.../compute/create_ibm_sce_windows_node.py | 2 +-
docs/examples/compute/gandi/create_node.py | 2 +-
docs/examples/compute/gridscale/create_node.py | 2 +-
docs/examples/compute/gridscale/deploy_node.py | 2 +-
docs/examples/compute/openstack/auth_cache.py | 18 +
docs/examples/compute/ovh/attach_volume.py | 2 +-
docs/examples/compute/ovh/create_node.py | 2 +-
docs/storage/_supported_methods_cdn.rst | 2 +-
docs/storage/_supported_methods_main.rst | 2 +-
docs/storage/_supported_providers.rst | 2 +-
docs/upgrade_notes.rst | 10 +
integration/storage/base.py | 67 +-
integration/storage/test_azure_blobs.py | 37 +-
integration/storage/test_minio.py | 2 +-
libcloud/__init__.py | 10 +-
libcloud/backup/drivers/gce.py | 2 +-
libcloud/common/base.py | 14 +-
libcloud/common/cloudsigma.py | 258 +-
libcloud/common/gandi.py | 4 +-
libcloud/common/google.py | 23 +-
libcloud/common/openstack.py | 41 +-
libcloud/common/openstack_identity.py | 518 +-
libcloud/compute/drivers/azure.py | 2 +-
libcloud/compute/drivers/azure_arm.py | 3 +-
libcloud/compute/drivers/cloudsigma.py | 315 +-
libcloud/compute/drivers/ec2.py | 21 +-
libcloud/compute/drivers/gandi.py | 2 +-
libcloud/compute/drivers/gce.py | 8 +-
libcloud/compute/drivers/openstack.py | 88 +-
libcloud/compute/drivers/softlayer.py | 2 +-
libcloud/compute/drivers/vcloud.py | 8 +-
libcloud/container/drivers/gke.py | 2 +-
libcloud/data/pricing.json | 98244 ++++++++++---------
libcloud/dns/drivers/nsone.py | 21 +-
libcloud/http.py | 11 +-
libcloud/storage/drivers/azure_blobs.py | 1 +
libcloud/storage/drivers/local.py | 11 +-
libcloud/storage/drivers/minio.py | 2 +-
libcloud/test/common/test_google.py | 10 +
libcloud/test/common/test_openstack_identity.py | 188 +-
.../fixtures/cloudsigma_2_0/drives_resize.json | 2 +-
.../fixtures/cloudsigma_2_0/keypairs_get.json | 12 +
.../fixtures/cloudsigma_2_0/keypairs_import.json | 16 +
.../fixtures/cloudsigma_2_0/keypairs_list.json | 33 +
.../compute/fixtures/cloudsigma_2_0/libdrives.json | 2 +-
.../fixtures/cloudsigma_2_0/servers_clone.json | 4 +-
.../cloudsigma_2_0/servers_create_with_vlan.json | 4 +-
.../cloudsigma_2_0/servers_detail_mixed_state.json | 4 +-
.../fixtures/cloudsigma_2_0/servers_get.json | 108 +
.../fixtures/openstack/_v3__auth_unauthorized.json | 1 +
.../fixtures/openstack_v1.1/_flavors_detail.json | 2 +-
.../openstack_v1.1/_v3_0__volume_quota.json | 55 +
libcloud/test/compute/test_cloudsigma_v1_0.py | 2 +-
libcloud/test/compute/test_cloudsigma_v2_0.py | 91 +-
libcloud/test/compute/test_cloudwatt.py | 11 +
libcloud/test/compute/test_ec2.py | 6 +-
libcloud/test/compute/test_openstack.py | 105 +-
libcloud/test/compute/test_rackspace.py | 2 +-
.../nsone/create_record_already_exists.json | 3 +
.../dns/fixtures/nsone/create_record_success.json | 24 +
.../nsone/create_record_zone_not_found.json | 3 +
libcloud/test/dns/test_base.py | 3 +-
libcloud/test/dns/test_nsone.py | 66 +-
libcloud/test/storage/test_local.py | 5 +
libcloud/test/test_connection.py | 22 +
libcloud/test/test_http.py | 36 +-
libcloud/utils/connection.py | 7 +-
libcloud/utils/misc.py | 4 +-
requirements-rtd.txt | 2 +
requirements-tests.txt | 29 +-
scripts/dist_install_check.sh | 40 +
scripts/dist_wheel_install_check.sh | 41 +
setup.py | 16 +-
tox.ini | 175 +-
87 files changed, 52533 insertions(+), 48610 deletions(-)
diff --cc CHANGES.rst
index 1940e47,df16244..c2c8d6f
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@@ -7,19 -7,50 +7,63 @@@ Changes in Apache Libcloud 3.3.2 (in de
Common
~~~~~~
+ - Fix how we set HTTP request timeout on the underlying requests session
+ object. requests library has changed how timeout is set so our old
+ code had no affect.
+
+ (GITHUB-1575, GITHUB-1576)
+ [Dimitris Galanis - @dimgal1]
+
+ - Update setup.py metadata and indicate we also support Python 3.10.
+
+ - [Google] Update Google authentication code so so we don't try to contact
+ GCE metadata server when determining auth credentials type when oAuth 2.0 /
+ installed app type of credentials are used.
+
+ (GITHUB-1591, GITHUB-1621)
+
+ Reported by Veith Röthlingshöfer - @RunOrVeith.
+
+ - [Google] Update Google authentication code so we don't try to retry failed
+ request when trying to determine if GCE metadata server is available when
+ retrying is enabled globally (either via module level constant or via
+ environment variable value).
+
+ This will speed up scenarios when trying is enabled globally, but GCE
+ metadata server is not available and different type of credentials are used
+ (e.g. oAuth 2).
+
+ (GITHUB-1591, GITHUB-1621)
+
+ Reported by Veith Röthlingshöfer - @RunOrVeith.
+
+ - Update minimum ``requests`` version we require as part for install_requires
+ in setup.py to ``2.26.0`` when using Python >= 3.6.
+
+ This was done to avoid licensing issue with transitive dependency
+ (``chardet``).
+
+ NOTE: requests ``>=2.25.1`` will be used when using Python 3.5 since 2.26.0
+ doesn't support Python 3.5 anymore.
+
+ For more context, see https://github.com/psf/requests/pull/5797.
+ (GITHUB-1594)
+
+ Reported by Jarek Potiuk - @potiuk.
+
+- Update HTTP connection and request retry code to be more flexible so user
+ can specify and utilize custom retry logic which can be configured via
+ connection retryCls attribute
+ (``driver.connection.retryCls = MyRetryClass``).
+
+ (GITHUB-1558)
+ [Veith Röthlingshöfer - @RunOrVeith]
+
+- HTTP connection and request retry logic has been updated so we still respect
+ ``timeout`` argument when retrying requests due to rate limit being reached
+ errors. Previously, we would try to retry indefinitely on
+ ``RateLimitReachedError`` exceptions.
+
Storage
~~~~~~~
diff --cc libcloud/utils/misc.py
index 3932015,8088c9c..17433d0
--- a/libcloud/utils/misc.py
+++ b/libcloud/utils/misc.py
@@@ -44,9 -42,29 +44,9 @@@ __all__ =
'ReprMixin'
]
-# Error message which indicates a transient SSL error upon which request
-# can be retried
-TRANSIENT_SSL_ERROR = 'The read operation timed out'
-
-
-class TransientSSLError(ssl.SSLError):
- """Represent transient SSL errors, e.g. timeouts"""
- pass
-
-
-# Constants used by the ``retry`` decorator
-DEFAULT_TIMEOUT = 30 # default retry timeout
-DEFAULT_DELAY = 1 # default sleep delay used in each iterator
-DEFAULT_BACKOFF = 1 # retry backup multiplier
-RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror,
- httplib.NotConnected, httplib.ImproperConnectionState,
- TransientSSLError)
-
- def find(l, predicate):
- results = [x for x in l if predicate(x)]
+ def find(value, predicate):
+ results = [x for x in value if predicate(x)]
return results[0] if len(results) > 0 else None