You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by da...@apache.org on 2026/06/29 13:33:57 UTC
(cloudstack) 13/24: secondary storage resource limit for upload
This is an automated email from the ASF dual-hosted git repository.
DaanHoogland pushed a commit to branch 4.22
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 021a18dd98338de9772b3b7c89cc43802d1a862e
Author: Abhisar Sinha <63...@users.noreply.github.com>
AuthorDate: Mon Mar 9 10:19:56 2026 +0530
secondary storage resource limit for upload
---
.../storage/command/UploadStatusCommand.java | 10 ++
.../cloud/storage/ImageStoreUploadMonitorImpl.java | 169 ++++++++++++++++-----
.../storage/resource/HttpUploadServerHandler.java | 2 +
.../resource/NfsSecondaryStorageResource.java | 33 +++-
4 files changed, 173 insertions(+), 41 deletions(-)
diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java
index 9e6b76e467f..f78744046f7 100644
--- a/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java
@@ -28,6 +28,7 @@ public class UploadStatusCommand extends Command {
}
private String entityUuid;
private EntityType entityType;
+ private Boolean abort;
protected UploadStatusCommand() {
}
@@ -37,6 +38,11 @@ public class UploadStatusCommand extends Command {
this.entityType = entityType;
}
+ public UploadStatusCommand(String entityUuid, EntityType entityType, Boolean abort) {
+ this(entityUuid, entityType);
+ this.abort = abort;
+ }
+
public String getEntityUuid() {
return entityUuid;
}
@@ -45,6 +51,10 @@ public class UploadStatusCommand extends Command {
return entityType;
}
+ public Boolean getAbort() {
+ return abort;
+ }
+
@Override
public boolean executeInSequence() {
return false;
diff --git a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java
index 408eb69917a..b56e5b56213 100755
--- a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java
+++ b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java
@@ -26,10 +26,10 @@ import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.agent.api.to.OVFInformationTO;
-import com.cloud.exception.ResourceAllocationException;
import com.cloud.resourcelimit.CheckedReservation;
import com.cloud.user.Account;
import com.cloud.user.dao.AccountDao;
+import com.cloud.user.AccountManager;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -126,6 +126,8 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
private ReservationDao reservationDao;
@Inject
private AccountDao accountDao;
+ @Inject
+ private AccountManager _accountMgr;
private long _nodeId;
private ScheduledExecutorService _executor = null;
@@ -214,6 +216,36 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
public UploadStatusCheck() {
}
+ private Answer sendUploadStatusCommandForVolume(EndPoint ep, UploadStatusCommand cmd, VolumeVO volume) {
+ Answer answer = null;
+ try {
+ answer = ep.sendMessage(cmd);
+ } catch (CloudRuntimeException e) {
+ logger.warn("Unable to get upload status for volume {}. Error details: {}", volume, e.getMessage());
+ answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
+ }
+ if (answer == null || !(answer instanceof UploadStatusAnswer)) {
+ logger.warn("No or invalid answer corresponding to UploadStatusCommand for volume {}", volume);
+ return null;
+ }
+ return answer;
+ }
+
+ private Answer sendUploadStatusCommandForTemplate(EndPoint ep, UploadStatusCommand cmd, VMTemplateVO template) {
+ Answer answer = null;
+ try {
+ answer = ep.sendMessage(cmd);
+ } catch (CloudRuntimeException e) {
+ logger.warn("Unable to get upload status for template {}. Error details: {}", template, e.getMessage());
+ answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
+ }
+ if (answer == null || !(answer instanceof UploadStatusAnswer)) {
+ logger.warn("No or invalid answer corresponding to UploadStatusCommand for template {}", template);
+ return null;
+ }
+ return answer;
+ }
+
@Override
protected void runInContext() {
// 1. Select all entries with download_state = Not_Downloaded or Download_In_Progress
@@ -240,18 +272,17 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
UploadStatusCommand cmd = new UploadStatusCommand(volume.getUuid(), EntityType.Volume);
if (host != null && host.getManagementServerId() != null) {
if (_nodeId == host.getManagementServerId().longValue()) {
- Answer answer = null;
- try {
- answer = ep.sendMessage(cmd);
- } catch (CloudRuntimeException e) {
- logger.warn("Unable to get upload status for volume {}. Error details: {}", volume, e.getMessage());
- answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
- }
- if (answer == null || !(answer instanceof UploadStatusAnswer)) {
- logger.warn("No or invalid answer corresponding to UploadStatusCommand for volume {}", volume);
+ Answer answer = sendUploadStatusCommandForVolume(ep, cmd, volume);
+ if (answer == null) {
continue;
}
- handleVolumeStatusResponse((UploadStatusAnswer)answer, volume, volumeDataStore);
+ if (!handleVolumeStatusResponse((UploadStatusAnswer)answer, volume, volumeDataStore)) {
+ cmd = new UploadStatusCommand(volume.getUuid(), EntityType.Volume, true);
+ answer = sendUploadStatusCommandForVolume(ep, cmd, volume);
+ if (answer == null) {
+ logger.warn("Unable to abort upload for volume {}", volume);
+ }
+ }
}
} else {
String error = "Volume " + volume.getUuid() + " failed to upload as SSVM is either destroyed or SSVM agent not in 'Up' state";
@@ -284,18 +315,17 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
UploadStatusCommand cmd = new UploadStatusCommand(template.getUuid(), EntityType.Template);
if (host != null && host.getManagementServerId() != null) {
if (_nodeId == host.getManagementServerId().longValue()) {
- Answer answer = null;
- try {
- answer = ep.sendMessage(cmd);
- } catch (CloudRuntimeException e) {
- logger.warn("Unable to get upload status for template {}. Error details: {}", template, e.getMessage());
- answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
- }
- if (answer == null || !(answer instanceof UploadStatusAnswer)) {
- logger.warn("No or invalid answer corresponding to UploadStatusCommand for template {}", template);
+ Answer answer = sendUploadStatusCommandForTemplate(ep, cmd, template);
+ if (answer == null) {
continue;
}
- handleTemplateStatusResponse((UploadStatusAnswer)answer, template, templateDataStore);
+ if (!handleTemplateStatusResponse((UploadStatusAnswer) answer, template, templateDataStore)) {
+ cmd = new UploadStatusCommand(template.getUuid(), EntityType.Template, true);
+ answer = sendUploadStatusCommandForTemplate(ep, cmd, template);
+ if (answer == null) {
+ logger.warn("Unable to abort upload for template {}", template);
+ }
+ }
}
} else {
String error = String.format(
@@ -312,7 +342,41 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
}
}
- private void handleVolumeStatusResponse(final UploadStatusAnswer answer, final VolumeVO volume, final VolumeDataStoreVO volumeDataStore) {
+ private Boolean checkAndUpdateSecondaryStorageResourceLimit(Long accountId, Long lastSize, Long currentSize) {
+ if (lastSize >= currentSize) {
+ return true;
+ }
+ Long usage = currentSize - lastSize;
+ try (CheckedReservation secStorageReservation = new CheckedReservation(_accountMgr.getAccount(accountId), Resource.ResourceType.secondary_storage, null, null, usage, reservationDao, _resourceLimitMgr)) {
+ _resourceLimitMgr.incrementResourceCount(accountId, Resource.ResourceType.secondary_storage, usage);
+ return true;
+ } catch (Exception e) {
+ _resourceLimitMgr.decrementResourceCount(accountId, Resource.ResourceType.secondary_storage, lastSize);
+ return false;
+ }
+ }
+
+ private Boolean checkAndUpdateVolumeResourceLimit(VolumeVO volume, VolumeDataStoreVO volumeDataStore, UploadStatusAnswer answer) {
+ boolean success = true;
+ Long currentSize = answer.getVirtualSize() != 0 ? answer.getVirtualSize() : answer.getPhysicalSize();
+ Long lastSize = volume.getSize() != null ? volume.getSize() : 0L;
+ if (!checkAndUpdateSecondaryStorageResourceLimit(volume.getAccountId(), volume.getSize(), currentSize)) {
+ volumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
+ volumeDataStore.setState(State.Failed);
+ volumeDataStore.setErrorString("Storage Limit Reached");
+ Account owner = accountDao.findById(volume.getAccountId());
+ String msg = String.format("Upload of volume [%s] failed because its owner [%s] does not have enough secondary storage space available.", volume.getUuid(), owner.getUuid());
+ logger.error(msg);
+ success = false;
+ }
+ VolumeVO volumeUpdate = _volumeDao.findById(volume.getId());
+ volumeUpdate.setSize(currentSize);
+ _volumeDao.update(volumeUpdate.getId(), volumeUpdate);
+ return success;
+ }
+
+ private boolean handleVolumeStatusResponse(final UploadStatusAnswer answer, final VolumeVO volume, final VolumeDataStoreVO volumeDataStore) {
+ final boolean[] needAbort = new boolean[]{false};
final StateMachine2<Volume.State, Event, Volume> stateMachine = Volume.State.getStateMachine();
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
@@ -324,6 +388,11 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
try {
switch (answer.getStatus()) {
case COMPLETED:
+ if (!checkAndUpdateVolumeResourceLimit(tmpVolume, tmpVolumeDataStore, answer)) {
+ stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao);
+ sendAlert = true;
+ break;
+ }
tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
tmpVolumeDataStore.setState(State.Ready);
tmpVolumeDataStore.setInstallPath(answer.getInstallPath());
@@ -335,7 +404,6 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
volumeUpdate.setSize(answer.getVirtualSize());
_volumeDao.update(tmpVolume.getId(), volumeUpdate);
stateMachine.transitTo(tmpVolume, Event.OperationSucceeded, null, _volumeDao);
- _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), Resource.ResourceType.secondary_storage, answer.getVirtualSize());
// publish usage events
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_UPLOAD, tmpVolume.getAccountId(),
@@ -348,6 +416,12 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
}
break;
case IN_PROGRESS:
+ if (!checkAndUpdateVolumeResourceLimit(tmpVolume, tmpVolumeDataStore, answer)) {
+ stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao);
+ sendAlert = true;
+ needAbort[0] = true;
+ break;
+ }
if (tmpVolume.getState() == Volume.State.NotUploaded) {
tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS);
tmpVolumeDataStore.setDownloadPercent(answer.getDownloadPercent());
@@ -396,10 +470,29 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
}
}
});
+ return !needAbort[0];
+ }
+
+ private Boolean checkAndUpdateTemplateResourceLimit(VMTemplateVO template, TemplateDataStoreVO templateDataStore, UploadStatusAnswer answer) {
+ boolean success = true;
+ Long currentSize = answer.getVirtualSize() != 0 ? answer.getVirtualSize() : answer.getPhysicalSize();
+ Long lastSize = template.getSize() != null ? template.getSize() : 0L;
+ if (!checkAndUpdateSecondaryStorageResourceLimit(template.getAccountId(), lastSize, currentSize)) {
+ templateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
+ templateDataStore.setErrorString("Storage Limit Reached");
+ templateDataStore.setState(State.Failed);
+ Account owner = accountDao.findById(template.getAccountId());
+ String msg = String.format("Upload of template [%s] failed because its owner [%s] does not have enough secondary storage space available.", template.getUuid(), owner.getUuid());
+ logger.error(msg);
+ success = false;
+ }
+ templateDataStore.setSize(currentSize);
+ return success;
}
- private void handleTemplateStatusResponse(final UploadStatusAnswer answer, final VMTemplateVO template, final TemplateDataStoreVO templateDataStore) {
+ private boolean handleTemplateStatusResponse(final UploadStatusAnswer answer, final VMTemplateVO template, final TemplateDataStoreVO templateDataStore) {
final StateMachine2<VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate> stateMachine = VirtualMachineTemplate.State.getStateMachine();
+ final boolean[] needAbort = new boolean[]{false};
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
@@ -410,6 +503,11 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
try {
switch (answer.getStatus()) {
case COMPLETED:
+ if (!checkAndUpdateTemplateResourceLimit(tmpTemplate, tmpTemplateDataStore, answer)) {
+ stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
+ sendAlert = true;
+ break;
+ }
tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
tmpTemplateDataStore.setState(State.Ready);
tmpTemplateDataStore.setInstallPath(answer.getInstallPath());
@@ -445,22 +543,6 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
break;
}
}
-
- Account owner = accountDao.findById(template.getAccountId());
- long templateSize = answer.getVirtualSize();
-
- try (CheckedReservation secondaryStorageReservation = new CheckedReservation(owner, Resource.ResourceType.secondary_storage, null, null, templateSize, reservationDao, _resourceLimitMgr)) {
- _resourceLimitMgr.incrementResourceCount(owner.getId(), Resource.ResourceType.secondary_storage, templateSize);
- } catch (ResourceAllocationException e) {
- tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOAD_ERROR);
- tmpTemplateDataStore.setState(State.Failed);
- stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
- msg = String.format("Upload of template [%s] failed because its owner [%s] does not have enough secondary storage space available.", template.getUuid(), owner.getUuid());
- logger.warn(msg);
- sendAlert = true;
- break;
- }
-
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationSucceeded, null, _templateDao);
//publish usage event
String etype = EventTypes.EVENT_TEMPLATE_CREATE;
@@ -477,6 +559,12 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
}
break;
case IN_PROGRESS:
+ if (!checkAndUpdateTemplateResourceLimit(tmpTemplate, tmpTemplateDataStore, answer)) {
+ stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
+ sendAlert = true;
+ needAbort[0] = true;
+ break;
+ }
if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) {
tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS);
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.UploadRequested, null, _templateDao);
@@ -526,6 +614,7 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
}
}
});
+ return !needAbort[0];
}
}
diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
index a580105d52a..605749649bb 100644
--- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
+++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
@@ -130,6 +130,7 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
if (decoder != null) {
decoder.cleanFiles();
}
+ storageResource.deregisterUploadChannel(uuid);
requestProcessed = false;
}
@@ -182,6 +183,7 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
requestProcessed = true;
return;
}
+ storageResource.registerUploadChannel(uuid, ctx.channel());
//set the base directory to download the file
DiskFileUpload.baseDirectory = uploadEntity.getInstallPathPrefix();
this.processTimeout = uploadEntity.getProcessTimeout();
diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index 9b50666258e..dc27f74bf3b 100644
--- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@ -49,6 +49,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -254,7 +255,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
protected String _parent = "/mnt/SecStorage";
final private String _tmpltpp = "template.properties";
protected String createTemplateFromSnapshotXenScript;
- private HashMap<String, UploadEntity> uploadEntityStateMap = new HashMap<>();
+ private final Map<String, UploadEntity> uploadEntityStateMap = new ConcurrentHashMap<>();
+ private final Map<String, Channel> uploadChannelMap = new ConcurrentHashMap<>();
private String _ssvmPSK = null;
private long processTimeout;
@@ -2374,6 +2376,20 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
String entityUuid = cmd.getEntityUuid();
if (uploadEntityStateMap.containsKey(entityUuid)) {
UploadEntity uploadEntity = uploadEntityStateMap.get(entityUuid);
+ if (Boolean.TRUE.equals(cmd.getAbort())) {
+ updateStateMapWithError(entityUuid, "Upload Entity aborted");
+ String errorMsg = uploadEntity.getErrorMessage();
+ if (errorMsg == null) {
+ errorMsg = "Upload aborted by management server";
+ }
+ Channel channel = uploadChannelMap.remove(entityUuid);
+ if (channel != null && channel.isActive()) {
+ logger.info("Closing upload channel for entity {}", entityUuid);
+ channel.close();
+ }
+ uploadEntityStateMap.remove(entityUuid);
+ return new UploadStatusAnswer(cmd, UploadStatus.ERROR, errorMsg);
+ }
if (uploadEntity.getUploadState() == UploadEntity.Status.ERROR) {
uploadEntityStateMap.remove(entityUuid);
return new UploadStatusAnswer(cmd, UploadStatus.ERROR, uploadEntity.getErrorMessage());
@@ -2392,6 +2408,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
UploadStatusAnswer answer = new UploadStatusAnswer(cmd, UploadStatus.IN_PROGRESS);
long downloadedSize = FileUtils.sizeOfDirectory(new File(uploadEntity.getInstallPathPrefix()));
int downloadPercent = (int)(100 * downloadedSize / uploadEntity.getContentLength());
+ answer.setPhysicalSize(downloadedSize);
answer.setDownloadPercent(Math.min(downloadPercent, 100));
return answer;
}
@@ -3421,6 +3438,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
public String postUpload(String uuid, String filename, long processTimeout) {
UploadEntity uploadEntity = uploadEntityStateMap.get(uuid);
+ if (uploadEntity == null) {
+ logger.warn("Upload entity not found for uuid: {}. Upload may have been aborted.", uuid);
+ return "Upload entity not found. Upload may have been aborted.";
+ }
int installTimeoutPerGig = 180 * 60 * 1000;
String resourcePath = uploadEntity.getInstallPathPrefix();
@@ -3571,6 +3592,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return _ssvmPSK;
}
+ public void registerUploadChannel(String uuid, Channel channel) {
+ uploadChannelMap.put(uuid, channel);
+ }
+
+ public void deregisterUploadChannel(String uuid) {
+ if (uuid != null) {
+ uploadChannelMap.remove(uuid);
+ }
+ }
+
public void updateStateMapWithError(String uuid, String errorMessage) {
UploadEntity uploadEntity = null;
if (uploadEntityStateMap.get(uuid) != null) {