You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by da...@apache.org on 2026/06/29 13:33:55 UTC
(cloudstack) 11/24: secondary storage resource limit for download
This is an automated email from the ASF dual-hosted git repository.
DaanHoogland pushed a commit to branch 4.22
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit ca7b08accebfaefce323c3c3810a144336256a99
Author: abh1sar <ab...@gmail.com>
AuthorDate: Mon Mar 2 11:06:13 2026 +0530
secondary storage resource limit for download
---
.../storage/VMTemplateStorageResourceAssoc.java | 3 +-
.../cloud/agent/api/storage/DownloadAnswer.java | 2 +-
.../storage/image/BaseImageStoreDriverImpl.java | 9 ++-
.../resourcelimit/ResourceLimitManagerImpl.java | 4 +-
.../storage/download/DownloadActiveState.java | 5 ++
.../cloud/storage/download/DownloadErrorState.java | 5 ++
.../storage/download/DownloadInactiveState.java | 6 ++
.../download/DownloadLimitReachedState.java | 54 +++++++++++++
.../cloud/storage/download/DownloadListener.java | 89 ++++++++++++++++++++--
.../com/cloud/storage/download/DownloadState.java | 6 +-
.../cloud/template/HypervisorTemplateAdapter.java | 9 ++-
.../storage/template/DownloadManagerImpl.java | 2 +-
12 files changed, 173 insertions(+), 21 deletions(-)
diff --git a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java
index db702a61f2b..7d5b2d7c57d 100644
--- a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java
+++ b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java
@@ -23,9 +23,10 @@ import org.apache.cloudstack.api.InternalIdentity;
public interface VMTemplateStorageResourceAssoc extends InternalIdentity {
public static enum Status {
- UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED
+ UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, LIMIT_REACHED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED
}
+ List<Status> ERROR_DOWNLOAD_STATES = List.of(Status.DOWNLOAD_ERROR, Status.ABANDONED, Status.LIMIT_REACHED, Status.UNKNOWN);
List<Status> PENDING_DOWNLOAD_STATES = List.of(Status.NOT_DOWNLOADED, Status.DOWNLOAD_IN_PROGRESS);
String getInstallPath();
diff --git a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java
index 0c6373134b1..1c5eb7b9a9a 100644
--- a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java
+++ b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java
@@ -140,7 +140,7 @@ public class DownloadAnswer extends Answer {
}
public Long getTemplateSize() {
- return templateSize;
+ return templateSize == 0 ? templatePhySicalSize : templateSize;
}
public void setTemplatePhySicalSize(long templatePhySicalSize) {
diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
index a2e9eff2a08..61b1a84cdc6 100644
--- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
+++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
@@ -230,8 +230,10 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
updateBuilder.setJobId(answer.getJobId());
updateBuilder.setLocalDownloadPath(answer.getDownloadPath());
updateBuilder.setInstallPath(answer.getInstallPath());
- updateBuilder.setSize(answer.getTemplateSize());
- updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize());
+ if (!VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) {
+ updateBuilder.setSize(answer.getTemplateSize());
+ updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize());
+ }
_templateStoreDao.update(tmpltStoreVO.getId(), updateBuilder);
// update size in vm_template table
VMTemplateVO tmlptUpdater = _templateDao.createForUpdate();
@@ -241,8 +243,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
AsyncCompletionCallback<CreateCmdResult> caller = context.getParentCallback();
- if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR ||
- answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) {
+ if (VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) {
CreateCmdResult result = new CreateCmdResult(null, null);
result.setSuccess(false);
result.setResult(answer.getErrorString());
diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java
index 09a0dda3aaa..43c3b383258 100644
--- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java
+++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java
@@ -258,7 +258,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
templateSizeSearch = _vmTemplateStoreDao.createSearchBuilder(SumCount.class);
templateSizeSearch.select("sum", Func.SUM, templateSizeSearch.entity().getSize());
- templateSizeSearch.and("downloadState", templateSizeSearch.entity().getDownloadState(), Op.EQ);
+ templateSizeSearch.and("downloadState", templateSizeSearch.entity().getDownloadState(), Op.IN);
templateSizeSearch.and("destroyed", templateSizeSearch.entity().getDestroyed(), Op.EQ);
SearchBuilder<VMTemplateVO> join1 = _vmTemplateDao.createSearchBuilder();
join1.and("accountId", join1.entity().getAccountId(), Op.EQ);
@@ -1410,7 +1410,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
long totalTemplatesSize = 0;
SearchCriteria<SumCount> sc = templateSizeSearch.create();
- sc.setParameters("downloadState", Status.DOWNLOADED);
+ sc.setParameters("downloadState", Status.DOWNLOADED, Status.DOWNLOAD_IN_PROGRESS);
sc.setParameters("destroyed", false);
sc.setJoinParameters("templates", "accountId", accountId);
List<SumCount> templates = _vmTemplateStoreDao.customSearch(sc, null);
diff --git a/server/src/main/java/com/cloud/storage/download/DownloadActiveState.java b/server/src/main/java/com/cloud/storage/download/DownloadActiveState.java
index 889ffbc0b1c..35b23985dd2 100644
--- a/server/src/main/java/com/cloud/storage/download/DownloadActiveState.java
+++ b/server/src/main/java/com/cloud/storage/download/DownloadActiveState.java
@@ -95,6 +95,11 @@ public abstract class DownloadActiveState extends DownloadState {
return Status.ABANDONED.toString();
}
+ @Override
+ public String handleLimitReached() {
+ return Status.LIMIT_REACHED.toString();
+ }
+
@Override
public String handleDisconnect() {
diff --git a/server/src/main/java/com/cloud/storage/download/DownloadErrorState.java b/server/src/main/java/com/cloud/storage/download/DownloadErrorState.java
index a0834456a2d..5dddefb0192 100644
--- a/server/src/main/java/com/cloud/storage/download/DownloadErrorState.java
+++ b/server/src/main/java/com/cloud/storage/download/DownloadErrorState.java
@@ -60,6 +60,11 @@ public class DownloadErrorState extends DownloadInactiveState {
return Status.ABANDONED.toString();
}
+ @Override
+ public String handleLimitReached() {
+ return Status.LIMIT_REACHED.toString();
+ }
+
@Override
public String getName() {
return Status.DOWNLOAD_ERROR.toString();
diff --git a/server/src/main/java/com/cloud/storage/download/DownloadInactiveState.java b/server/src/main/java/com/cloud/storage/download/DownloadInactiveState.java
index 8fee3d0437c..69d46879ebe 100644
--- a/server/src/main/java/com/cloud/storage/download/DownloadInactiveState.java
+++ b/server/src/main/java/com/cloud/storage/download/DownloadInactiveState.java
@@ -36,6 +36,12 @@ public abstract class DownloadInactiveState extends DownloadState {
return getName();
}
+ @Override
+ public String handleLimitReached() {
+ // ignore and stay put
+ return getName();
+ }
+
@Override
public String handleDisconnect() {
//ignore and stay put
diff --git a/server/src/main/java/com/cloud/storage/download/DownloadLimitReachedState.java b/server/src/main/java/com/cloud/storage/download/DownloadLimitReachedState.java
new file mode 100644
index 00000000000..8ce5668299e
--- /dev/null
+++ b/server/src/main/java/com/cloud/storage/download/DownloadLimitReachedState.java
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.storage.download;
+
+import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType;
+import org.apache.logging.log4j.Level;
+
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+
+public class DownloadLimitReachedState extends DownloadInactiveState {
+
+ public DownloadLimitReachedState(DownloadListener dl) {
+ super(dl);
+ }
+
+ @Override
+ public String getName() {
+ return Status.LIMIT_REACHED.toString();
+ }
+
+ @Override
+ public String handleEvent(DownloadEvent event, Object eventObj) {
+ if (logger.isTraceEnabled()) {
+ getDownloadListener().log("handleEvent, event type=" + event + ", curr state=" + getName(), Level.TRACE);
+ }
+ return Status.LIMIT_REACHED.toString();
+ }
+
+ @Override
+ public void onEntry(String prevState, DownloadEvent event, Object evtObj) {
+ if (!prevState.equalsIgnoreCase(getName())) {
+ DownloadAnswer answer = new DownloadAnswer("Storage Limit Reached", Status.LIMIT_REACHED);
+ getDownloadListener().callback(answer);
+ getDownloadListener().cancelStatusTask();
+ getDownloadListener().cancelTimeoutTask();
+ getDownloadListener().scheduleImmediateStatusCheck(RequestType.PURGE);
+ }
+ }
+}
diff --git a/server/src/main/java/com/cloud/storage/download/DownloadListener.java b/server/src/main/java/com/cloud/storage/download/DownloadListener.java
index 42b0e394db4..058881fdb54 100644
--- a/server/src/main/java/com/cloud/storage/download/DownloadListener.java
+++ b/server/src/main/java/com/cloud/storage/download/DownloadListener.java
@@ -25,6 +25,15 @@ import java.util.Timer;
import javax.inject.Inject;
+import com.cloud.configuration.Resource;
+import com.cloud.resourcelimit.CheckedReservation;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.user.Account;
+import com.cloud.user.AccountManager;
+import com.cloud.user.ResourceLimitService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -34,10 +43,13 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
import org.apache.cloudstack.managed.context.ManagedContextTimerTask;
+import org.apache.cloudstack.reservation.dao.ReservationDao;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -107,6 +119,7 @@ public class DownloadListener implements Listener {
public static final String DOWNLOAD_ERROR = Status.DOWNLOAD_ERROR.toString();
public static final String DOWNLOAD_IN_PROGRESS = Status.DOWNLOAD_IN_PROGRESS.toString();
public static final String DOWNLOAD_ABANDONED = Status.ABANDONED.toString();
+ public static final String DOWNLOAD_LIMIT_REACHED = Status.LIMIT_REACHED.toString();
private EndPoint _ssAgent;
@@ -137,6 +150,18 @@ public class DownloadListener implements Listener {
private DataStoreManager _storeMgr;
@Inject
private VolumeService _volumeSrv;
+ @Inject
+ private VMTemplateDao _templateDao;
+ @Inject
+ private TemplateDataStoreDao _templateDataStoreDao;
+ @Inject
+ private VolumeDao _volumeDao;
+ @Inject
+ private ResourceLimitService _resourceLimitMgr;
+ @Inject
+ private AccountManager _accountMgr;
+ @Inject
+ ReservationDao _reservationDao;
private LazyCache<Long, List<Hypervisor.HypervisorType>> zoneHypervisorsCache;
@@ -160,7 +185,7 @@ public class DownloadListener implements Listener {
_downloadMonitor = downloadMonitor;
_cmd = cmd;
initStateMachine();
- _currState = getState(Status.NOT_DOWNLOADED.toString());
+ _currState = getState(NOT_DOWNLOADED);
this._timer = timer;
_timeoutTask = new TimeoutTask(this);
this._timer.schedule(_timeoutTask, 3 * STATUS_POLL_INTERVAL);
@@ -184,11 +209,12 @@ public class DownloadListener implements Listener {
}
private void initStateMachine() {
- _stateMap.put(Status.NOT_DOWNLOADED.toString(), new NotDownloadedState(this));
- _stateMap.put(Status.DOWNLOADED.toString(), new DownloadCompleteState(this));
- _stateMap.put(Status.DOWNLOAD_ERROR.toString(), new DownloadErrorState(this));
- _stateMap.put(Status.DOWNLOAD_IN_PROGRESS.toString(), new DownloadInProgressState(this));
- _stateMap.put(Status.ABANDONED.toString(), new DownloadAbandonedState(this));
+ _stateMap.put(NOT_DOWNLOADED, new NotDownloadedState(this));
+ _stateMap.put(DOWNLOADED, new DownloadCompleteState(this));
+ _stateMap.put(DOWNLOAD_ERROR, new DownloadErrorState(this));
+ _stateMap.put(DOWNLOAD_IN_PROGRESS, new DownloadInProgressState(this));
+ _stateMap.put(DOWNLOAD_ABANDONED, new DownloadAbandonedState(this));
+ _stateMap.put(DOWNLOAD_LIMIT_REACHED, new DownloadLimitReachedState(this));
}
private DownloadState getState(String stateName) {
@@ -239,10 +265,53 @@ public class DownloadListener implements Listener {
return false;
}
+ private Long getAccountIdForDataObject() {
+ if (object == null) {
+ return null;
+ }
+ if (DataObjectType.TEMPLATE.equals(object.getType())) {
+ VMTemplateVO t = _templateDao.findById(object.getId());
+ return t != null ? t.getAccountId() : null;
+ } else if (DataObjectType.VOLUME.equals(object.getType())) {
+ VolumeVO v = _volumeDao.findById(object.getId());
+ return v != null ? v.getAccountId() : null;
+ }
+ return null;
+ }
+
+ private Long getSizeFromDB() {
+ Long lastSize = 0L;
+ if (DataObjectType.TEMPLATE.equals(object.getType())) {
+ TemplateDataStoreVO t = _templateDataStoreDao.findByStoreTemplate(object.getDataStore().getId(), object.getId());
+ lastSize = t.getSize();
+ } else if (DataObjectType.VOLUME.equals(object.getType())) {
+ VolumeVO v = _volumeDao.findById(object.getId());
+ lastSize = v.getSize();
+ }
+ return lastSize;
+ }
+
+ private Boolean checkAndUpdateResourceLimits(DownloadAnswer answer) {
+ Long lastSize = getSizeFromDB();
+ Long currentSize = answer.getTemplateSize();
+
+ if (currentSize > lastSize) {
+ Long accountId = getAccountIdForDataObject();
+ Account account = _accountMgr.getAccount(accountId);
+ Long usage = currentSize - lastSize;
+ try (CheckedReservation secStorageReservation = new CheckedReservation(account, Resource.ResourceType.secondary_storage, usage, _reservationDao, _resourceLimitMgr)) {
+ _resourceLimitMgr.incrementResourceCount(accountId, Resource.ResourceType.secondary_storage, usage);
+ } catch (Exception e) {
+ return false;
+ }
+ }
+ return true;
+ }
+
@Override
public boolean processAnswers(long agentId, long seq, Answer[] answers) {
boolean processed = false;
- if (answers != null & answers.length > 0) {
+ if (answers != null && answers.length > 0) {
if (answers[0] instanceof DownloadAnswer) {
final DownloadAnswer answer = (DownloadAnswer)answers[0];
if (getJobId() == null) {
@@ -250,7 +319,11 @@ public class DownloadListener implements Listener {
} else if (!getJobId().equalsIgnoreCase(answer.getJobId())) {
return false;//TODO
}
- transition(DownloadEvent.DOWNLOAD_ANSWER, answer);
+ if (!checkAndUpdateResourceLimits(answer)) {
+ transition(DownloadEvent.LIMIT_REACHED, answer);
+ } else {
+ transition(DownloadEvent.DOWNLOAD_ANSWER, answer);
+ }
processed = true;
}
}
diff --git a/server/src/main/java/com/cloud/storage/download/DownloadState.java b/server/src/main/java/com/cloud/storage/download/DownloadState.java
index 68723b53e35..e58d1f3f39f 100644
--- a/server/src/main/java/com/cloud/storage/download/DownloadState.java
+++ b/server/src/main/java/com/cloud/storage/download/DownloadState.java
@@ -26,7 +26,7 @@ import com.cloud.agent.api.storage.DownloadAnswer;
public abstract class DownloadState {
public static enum DownloadEvent {
- DOWNLOAD_ANSWER, ABANDON_DOWNLOAD, TIMEOUT_CHECK, DISCONNECT
+ DOWNLOAD_ANSWER, ABANDON_DOWNLOAD, LIMIT_REACHED, TIMEOUT_CHECK, DISCONNECT
};
protected Logger logger = LogManager.getLogger(getClass());
@@ -51,6 +51,8 @@ public abstract class DownloadState {
return handleAnswer(answer);
case ABANDON_DOWNLOAD:
return handleAbort();
+ case LIMIT_REACHED:
+ return handleLimitReached();
case TIMEOUT_CHECK:
Date now = new Date();
long update = now.getTime() - dl.getLastUpdated().getTime();
@@ -78,6 +80,8 @@ public abstract class DownloadState {
public abstract String handleAbort();
+ public abstract String handleLimitReached();
+
public abstract String handleDisconnect();
public abstract String handleAnswer(DownloadAnswer answer);
diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
index c096ef0eb1d..7e5811a2ece 100644
--- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
+++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
@@ -481,13 +481,12 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
CreateTemplateContext<TemplateApiResult> context) {
TemplateApiResult result = callback.getResult();
TemplateInfo template = context.template;
+ VMTemplateVO tmplt = _tmpltDao.findById(template.getId());
if (result.isSuccess()) {
- VMTemplateVO tmplt = _tmpltDao.findById(template.getId());
// need to grant permission for public templates
if (tmplt.isPublicTemplate()) {
_messageBus.publish(_name, TemplateManager.MESSAGE_REGISTER_PUBLIC_TEMPLATE_EVENT, PublishScope.LOCAL, tmplt.getId());
}
- long accountId = tmplt.getAccountId();
if (template.getSize() != null) {
// publish usage event
String etype = EventTypes.EVENT_TEMPLATE_CREATE;
@@ -517,7 +516,11 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
template.getSize(), VirtualMachineTemplate.class.getName(), template.getUuid());
}
}
- _resourceLimitMgr.recalculateResourceCount(accountId, tmplt.getDomainId(), ResourceType.secondary_storage.getOrdinal());
+ }
+ if (tmplt != null) {
+ long accountId = tmplt.getAccountId();
+ Account account = _accountDao.findById(accountId);
+ _resourceLimitMgr.recalculateResourceCount(accountId, account.getDomainId(), ResourceType.secondary_storage.getOrdinal());
}
return null;
diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
index 43a9422e8ad..c695b3d4b44 100644
--- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -996,7 +996,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
break; // TODO
}
return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId),
- getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId));
+ getDownloadTemplateSize(jobId), td.getDownloadedBytes(), getDownloadCheckSum(jobId));
}
private String getInstallPath(String jobId) {