You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by qu...@apache.org on 2017/11/08 03:36:42 UTC
[1/4] libcloud git commit: Limit number of retries in destroy_node.
Repository: libcloud
Updated Branches:
refs/heads/trunk 42709622c -> 0c5bee8b1
Limit number of retries in destroy_node.
Signed-off-by: Quentin Pradet <qu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/a71e9557
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/a71e9557
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/a71e9557
Branch: refs/heads/trunk
Commit: a71e95579dd461dd180c798091bce1216924165c
Parents: 4270962
Author: Peter Amstutz <pe...@curoverse.com>
Authored: Mon Oct 16 16:12:45 2017 -0400
Committer: Quentin Pradet <qu...@apache.org>
Committed: Wed Nov 8 07:32:04 2017 +0400
----------------------------------------------------------------------
libcloud/compute/drivers/azure_arm.py | 38 +++++++++++++++++++++---------
1 file changed, 27 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a71e9557/libcloud/compute/drivers/azure_arm.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/azure_arm.py b/libcloud/compute/drivers/azure_arm.py
index fcc9ef3..db7ef65 100644
--- a/libcloud/compute/drivers/azure_arm.py
+++ b/libcloud/compute/drivers/azure_arm.py
@@ -691,7 +691,10 @@ class AzureNodeDriver(NodeDriver):
else:
return False
- def destroy_node(self, node, ex_destroy_nic=True, ex_destroy_vhd=True):
+ def destroy_node(self, node,
+ ex_destroy_nic=True,
+ ex_destroy_vhd=True,
+ ex_retries=10):
"""
Destroy a node.
@@ -706,6 +709,10 @@ class AzureNodeDriver(NodeDriver):
this node (default True).
:type node: ``bool``
+ :param ex_retries: Number of times to retry checking if the node is gone,
+ destroying the NIC or destroying the VHD.
+ :type node: ``int``
+
:return: True if the destroy was successful, raises exception
otherwise.
:rtype: ``bool``
@@ -733,12 +740,14 @@ class AzureNodeDriver(NodeDriver):
# Poll until the node actually goes away (otherwise attempt to delete
# NIC and VHD will fail with "resource in use" errors).
- while do_node_polling:
+ retries = ex_retries
+ while do_node_polling and retries > 0:
try:
time.sleep(10)
self.connection.request(
node.id,
params={"api-version": RESOURCE_API_VERSION})
+ retries -= 1
except BaseHTTPError as h:
if h.code in (204, 404):
# Node is gone
@@ -752,13 +761,16 @@ class AzureNodeDriver(NodeDriver):
node.extra["properties"]["networkProfile"]["networkInterfaces"]
if ex_destroy_nic:
for nic in interfaces:
- while True:
+ retries = ex_retries
+ while retries > 0:
try:
self.ex_destroy_nic(self._to_nic(nic))
break
except BaseHTTPError as h:
+ retries -= 1
if (h.code == 400 and
- h.message.startswith("[NicInUse]")):
+ h.message.startswith("[NicInUse]") and
+ retries > 0):
time.sleep(10)
else:
raise
@@ -767,19 +779,23 @@ class AzureNodeDriver(NodeDriver):
vhd = node.extra["properties"]["storageProfile"]["osDisk"].get("vhd")
if ex_destroy_vhd and vhd is not None:
resourceGroup = node.id.split("/")[4]
- while True:
+ retries = ex_retries
+ while retries > 0:
try:
- if self._ex_delete_old_vhd(
- resourceGroup,
- vhd["uri"]):
+ if self._ex_delete_old_vhd(resourceGroup, vhd["uri"]):
break
# Unfortunately lease errors usually result in it returning
# "False" with no more information. Need to wait and try
# again.
except LibcloudError as e:
- if "LeaseIdMissing" in str(e):
- # If we get an lease error, need to wait and try again.
- pass
+ retries -= 1
+ if "LeaseIdMissing" in str(e) and retries > 0:
+ # Unfortunately lease errors
+ # (which occur if the vhd blob
+ # hasn't yet been released by the VM being destroyed)
+ # get raised as plain
+ # LibcloudError. Wait a bit and try again.
+ time.sleep(10)
else:
raise
time.sleep(10)
[3/4] libcloud git commit: Test retries in destroy_node
Posted by qu...@apache.org.
Test retries in destroy_node
Signed-off-by: Quentin Pradet <qu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/55b16c85
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/55b16c85
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/55b16c85
Branch: refs/heads/trunk
Commit: 55b16c8582478b88f5d8b4cf7cfbcd9562a78ecf
Parents: aaaa101
Author: Lucas Di Pentima <ld...@veritasgenetics.com>
Authored: Mon Nov 6 19:22:09 2017 -0300
Committer: Quentin Pradet <qu...@apache.org>
Committed: Wed Nov 8 07:32:06 2017 +0400
----------------------------------------------------------------------
libcloud/test/compute/test_azure_arm.py | 38 ++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/55b16c85/libcloud/test/compute/test_azure_arm.py
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/test_azure_arm.py b/libcloud/test/compute/test_azure_arm.py
index a32917d..bd8cfa0 100644
--- a/libcloud/test/compute/test_azure_arm.py
+++ b/libcloud/test/compute/test_azure_arm.py
@@ -166,6 +166,44 @@ class AzureNodeDriverTests(LibcloudTestCase):
self.assertTrue(ret)
@mock.patch('time.sleep', return_value=None)
+ def test_destroy_node__retry(self, time_sleep_mock):
+ def error(e, **kwargs):
+ raise e(**kwargs)
+ node = self.driver.list_nodes()[0]
+ AzureMockHttp.responses = [
+ # 202 - The delete will happen asynchronously
+ lambda f: error(BaseHTTPError, code=202, message='Deleting'),
+ # 200 means the node is still here - Try 1
+ lambda f: (httplib.OK, None, {}, 'OK'),
+ # 200 means the node is still here - Try 2
+ lambda f: (httplib.OK, None, {}, 'OK'),
+ # 200 means the node is still here - Try 3
+ lambda f: (httplib.OK, None, {}, 'OK'),
+ # 404 means node is gone - 4th retry: success!
+ lambda f: error(BaseHTTPError, code=404, message='Not found'),
+ ]
+ ret = self.driver.destroy_node(node)
+ self.assertTrue(ret)
+ self.assertEqual(4, time_sleep_mock.call_count) # Retries
+
+ @mock.patch('time.sleep', return_value=None)
+ def test_destroy_node__destroy_nic_retries(self, time_sleep_mock):
+ def error(e, **kwargs):
+ raise e(**kwargs)
+ node = self.driver.list_nodes()[0]
+ err = BaseHTTPError(code=400, message='[NicInUse] Cannot destroy')
+ with mock.patch.object(self.driver, 'ex_destroy_nic') as m:
+ m.side_effect = [err] * 5 + [True] # 5 errors before a success
+ ret = self.driver.destroy_node(node)
+ self.assertTrue(ret)
+ self.assertEqual(6, m.call_count) # 6th call was a success
+
+ m.side_effect = [err] * 10 + [True] # 10 errors before a success
+ with self.assertRaises(BaseHTTPError):
+ self.driver.destroy_node(node)
+ self.assertEqual(10, m.call_count) # try 10 times & fail
+
+ @mock.patch('time.sleep', return_value=None)
def test_destroy_node__async(self, time_sleep_mock):
def error(e, **kwargs):
raise e(**kwargs)
[4/4] libcloud git commit: Add changes for #1134
Posted by qu...@apache.org.
Add changes for #1134
Closes #1134
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/0c5bee8b
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/0c5bee8b
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/0c5bee8b
Branch: refs/heads/trunk
Commit: 0c5bee8b15d1f0a196fe8b332e11b44a4f85ae11
Parents: 55b16c8
Author: Quentin Pradet <qu...@apache.org>
Authored: Wed Nov 8 07:35:53 2017 +0400
Committer: Quentin Pradet <qu...@apache.org>
Committed: Wed Nov 8 07:36:26 2017 +0400
----------------------------------------------------------------------
CHANGES.rst | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/0c5bee8b/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index add7274..44bbd7b 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -86,6 +86,9 @@ Compute
- [ARM] Fix delete_old_vhd (GITHUB-1137)
[Peter Amstutz, Lucas Di Pentima]
+- [ARM] Limit number of retries in destroy_node (GITHUB-1134)
+ [Peter Amstutz, Lucas Di Pentima]
+
Storage
~~~~~~~
[2/4] libcloud git commit: Renamed ex_retries param on destroy_node()
to ex_poll_qty. Also added ex_poll_wait.
Posted by qu...@apache.org.
Renamed ex_retries param on destroy_node() to ex_poll_qty. Also
added ex_poll_wait.
Signed-off-by: Quentin Pradet <qu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/aaaa1015
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/aaaa1015
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/aaaa1015
Branch: refs/heads/trunk
Commit: aaaa10158ff91039546446309b80bf45042acf79
Parents: a71e955
Author: Lucas Di Pentima <ld...@veritasgenetics.com>
Authored: Tue Oct 31 16:49:56 2017 -0300
Committer: Quentin Pradet <qu...@apache.org>
Committed: Wed Nov 8 07:32:06 2017 +0400
----------------------------------------------------------------------
libcloud/compute/drivers/azure_arm.py | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/aaaa1015/libcloud/compute/drivers/azure_arm.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/azure_arm.py b/libcloud/compute/drivers/azure_arm.py
index db7ef65..1d9da01 100644
--- a/libcloud/compute/drivers/azure_arm.py
+++ b/libcloud/compute/drivers/azure_arm.py
@@ -694,7 +694,8 @@ class AzureNodeDriver(NodeDriver):
def destroy_node(self, node,
ex_destroy_nic=True,
ex_destroy_vhd=True,
- ex_retries=10):
+ ex_poll_qty=10,
+ ex_poll_wait=10):
"""
Destroy a node.
@@ -709,8 +710,11 @@ class AzureNodeDriver(NodeDriver):
this node (default True).
:type node: ``bool``
- :param ex_retries: Number of times to retry checking if the node is gone,
- destroying the NIC or destroying the VHD.
+ :param ex_poll_qty: Number of retries checking if the node
+ is gone, destroying the NIC or destroying the VHD (default 10).
+ :type node: ``int``
+
+ :param ex_poll_wait: Delay in seconds between retries (default 10).
:type node: ``int``
:return: True if the destroy was successful, raises exception
@@ -740,10 +744,10 @@ class AzureNodeDriver(NodeDriver):
# Poll until the node actually goes away (otherwise attempt to delete
# NIC and VHD will fail with "resource in use" errors).
- retries = ex_retries
+ retries = ex_poll_qty
while do_node_polling and retries > 0:
try:
- time.sleep(10)
+ time.sleep(ex_poll_wait)
self.connection.request(
node.id,
params={"api-version": RESOURCE_API_VERSION})
@@ -761,7 +765,7 @@ class AzureNodeDriver(NodeDriver):
node.extra["properties"]["networkProfile"]["networkInterfaces"]
if ex_destroy_nic:
for nic in interfaces:
- retries = ex_retries
+ retries = ex_poll_qty
while retries > 0:
try:
self.ex_destroy_nic(self._to_nic(nic))
@@ -771,15 +775,15 @@ class AzureNodeDriver(NodeDriver):
if (h.code == 400 and
h.message.startswith("[NicInUse]") and
retries > 0):
- time.sleep(10)
+ time.sleep(ex_poll_wait)
else:
raise
# Optionally clean up OS disk VHD.
vhd = node.extra["properties"]["storageProfile"]["osDisk"].get("vhd")
if ex_destroy_vhd and vhd is not None:
+ retries = ex_poll_qty
resourceGroup = node.id.split("/")[4]
- retries = ex_retries
while retries > 0:
try:
if self._ex_delete_old_vhd(resourceGroup, vhd["uri"]):
@@ -795,7 +799,7 @@ class AzureNodeDriver(NodeDriver):
# hasn't yet been released by the VM being destroyed)
# get raised as plain
# LibcloudError. Wait a bit and try again.
- time.sleep(10)
+ time.sleep(ex_poll_wait)
else:
raise
time.sleep(10)