You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by da...@apache.org on 2026/05/21 14:57:41 UTC
(cloudstack) branch main updated: fix(linstor): verify resource deletion completes; warn if stuck in DELETING (#13076)
This is an automated email from the ASF dual-hosted git repository.
DaanHoogland pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/main by this push:
new 850b44317a4 fix(linstor): verify resource deletion completes; warn if stuck in DELETING (#13076)
850b44317a4 is described below
commit 850b44317a4c7e4f57b9d03f6890fd143bbd63a8
Author: James Peru Mmbono <jm...@gmail.com>
AuthorDate: Thu May 21 17:57:33 2026 +0300
fix(linstor): verify resource deletion completes; warn if stuck in DELETING (#13076)
Co-authored-by: jmsperu <jm...@users.noreply.github.com>
---
.../kvm/storage/LinstorStorageAdaptor.java | 11 +++++
.../driver/LinstorPrimaryDataStoreDriverImpl.java | 14 ++++++
.../storage/datastore/util/LinstorUtil.java | 52 ++++++++++++++++++++++
3 files changed, 77 insertions(+)
diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
index 77953a32e63..31a41cd9407 100644
--- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
+++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
@@ -514,6 +514,17 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
ApiCallRcList answers = api.resourceDefinitionDelete(rd.getName());
checkLinstorAnswersThrow(answers);
deleted = true;
+
+ // LINSTOR can return success here while the resource lingers in DELETING state
+ // on the controller (down peer, lost quorum, etc.). Confirm it's actually gone
+ // — if not, log a WARN so operators can clear it manually. Don't throw: the
+ // CloudStack-side accounting has already moved on.
+ if (!LinstorUtil.waitForResourceDefinitionDeleted(api, rd.getName(),
+ LinstorUtil.DEFAULT_RD_DELETE_VERIFY_TIMEOUT_MILLIS)) {
+ logger.warn("Linstor: resource {} still present {}ms after delete returned success — " +
+ "may be stuck in DELETING. Check the LINSTOR controller (linstor resource list).",
+ rd.getName(), LinstorUtil.DEFAULT_RD_DELETE_VERIFY_TIMEOUT_MILLIS);
+ }
}
}
return deleted;
diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java
index 83dacf74e8d..d7451fab18f 100644
--- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java
+++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java
@@ -232,6 +232,20 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
throw new CloudRuntimeException("Linstor: Unable to delete resource definition: " + rscDefName);
}
logger.info("Linstor: Deleted resource {}", rscDefName);
+
+ // LINSTOR can return success on the delete API call while the resource lingers in
+ // DELETING state (peer issues, lost quorum, satellite down). Verify the resource is
+ // actually gone — if not, log a WARN so operators see it. We deliberately do NOT
+ // throw here: the volume is already considered gone on the CloudStack side, and
+ // throwing would leave the CS DB and LINSTOR in different states.
+ if (!LinstorUtil.waitForResourceDefinitionDeleted(linstorApi, rscDefName,
+ LinstorUtil.DEFAULT_RD_DELETE_VERIFY_TIMEOUT_MILLIS))
+ {
+ logger.warn("Linstor: resource {} still present {}ms after delete returned success — " +
+ "may be stuck in DELETING. Check the LINSTOR controller (linstor resource list) " +
+ "and clear manually if the resource has no live peers.",
+ rscDefName, LinstorUtil.DEFAULT_RD_DELETE_VERIFY_TIMEOUT_MILLIS);
+ }
} catch (ApiException apiEx)
{
logger.error("Linstor: ApiEx - " + apiEx.getMessage());
diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java
index 7c45493dddc..239331077e1 100644
--- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java
+++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java
@@ -401,6 +401,58 @@ public class LinstorUtil {
.collect(Collectors.toList());
}
+ /**
+ * Default per-call timeout for {@link #waitForResourceDefinitionDeleted}. Long enough for a
+ * healthy LINSTOR controller to finish a normal delete; short enough not to block the calling
+ * thread for too long if the delete is genuinely stuck. Used both from the management server
+ * (e.g. {@code LinstorPrimaryDataStoreDriverImpl}) and from KVM agent paths.
+ */
+ public static final long DEFAULT_RD_DELETE_VERIFY_TIMEOUT_MILLIS = 30_000L;
+
+ /**
+ * Returns {@code true} if the named resource definition is no longer present on the LINSTOR
+ * controller. Used after a {@code resourceDefinitionDelete} to verify the delete actually
+ * completed (LINSTOR can return success on the API call while the resource lingers in
+ * DELETING state due to peer issues, lost quorum, or down satellites). Uses the
+ * controller-side name filter rather than scanning every RD on the cluster (cheap even
+ * when polled once per second from {@link #waitForResourceDefinitionDeleted}).
+ */
+ public static boolean isResourceDefinitionGone(DevelopersApi api, String rscName) throws ApiException {
+ List<ResourceDefinition> matching =
+ api.resourceDefinitionList(Collections.singletonList(rscName), false, null, null, null);
+ return matching == null || matching.isEmpty();
+ }
+
+ /**
+ * Polls the controller until the named resource definition is gone or the timeout elapses.
+ * Returns {@code true} if the resource was confirmed gone, {@code false} if it was still
+ * present (or the controller kept erroring) at the deadline. Callers should NOT throw on a
+ * {@code false} return — the upstream API call already reported success and the operator
+ * may need to investigate manually. Log a WARN with the resource name instead.
+ */
+ public static boolean waitForResourceDefinitionDeleted(DevelopersApi api, String rscName, long timeoutMillis) {
+ final long deadline = System.currentTimeMillis() + timeoutMillis;
+ while (true) {
+ try {
+ if (isResourceDefinitionGone(api, rscName)) {
+ return true;
+ }
+ } catch (ApiException e) {
+ LOGGER.debug("LINSTOR delete-verify poll failed for {}: {}", rscName, e.getMessage());
+ // Keep polling — controller may be transiently unavailable.
+ }
+ if (System.currentTimeMillis() >= deadline) {
+ return false;
+ }
+ try {
+ Thread.sleep(1_000L);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ }
+ }
+
/**
* Returns a pair list of resource-definitions with ther 1:1 mapped resource-group objects that start with the
* resource name `startWith`