You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ro...@apache.org on 2021/02/04 09:14:17 UTC
[cloudstack] branch 4.14 updated: server: Fix update capacity for
hosts take long time if there are many service offerings (#4623)
This is an automated email from the ASF dual-hosted git repository.
rohit pushed a commit to branch 4.14
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.14 by this push:
new 78f73c1 server: Fix update capacity for hosts take long time if there are many service offerings (#4623)
78f73c1 is described below
commit 78f73c1bc631fdba997fe19d44cdd0ea58243e11
Author: Wei Zhou <w....@global.leaseweb.com>
AuthorDate: Thu Feb 4 10:13:57 2021 +0100
server: Fix update capacity for hosts take long time if there are many service offerings (#4623)
Steps to reproduce the issue:
(1)Create 10000 service offerings (by db changes below or cloudmonkey).
```
DROP PROCEDURE IF EXISTS cloud.insert_service_offering;
DELIMITER $$
CREATE PROCEDURE cloud.insert_service_offering()
BEGIN
DECLARE count INT DEFAULT 10000;
SET @offeringid = (select max(id)+1 from disk_offering);
WHILE count > 0 DO
INSERT INTO disk_offering (id,name,uuid,display_text,disk_size,type,created) values (@offeringid,'test-offering-wei',uuid(), 'test-offering-wei',0,'Service',now());
INSERT INTO service_offering (id,cpu,speed,ram_size) values (@offeringid, 1, 500,256);
SET @offeringid = @offeringid + 1;
SET count = count - 1;
END WHILE;
END $$
DELIMITER ;
CALL cloud.insert_service_offering();
mysql> CALL cloud.insert_service_offering();
Query OK, 0 rows affected (2 min 30.85 sec)
```
(2) Check the total time of periodical capacity check in cloudstack.
Without this patch, it spend 2.5 seconds (2 hosts)
```
2021-01-15 16:10:12,793 DEBUG [c.c.a.AlertManagerImpl] (CapacityChecker:ctx-5d5f3b3b) (logid:f5eb68ba) Running Capacity Checker ...
2021-01-15 16:10:15,287 DEBUG [c.c.a.AlertManagerImpl] (CapacityChecker:ctx-5d5f3b3b) (logid:f5eb68ba) Done running Capacity Checker ...
```
With this patch ,it spend 1.3 seconds (2 hosts)
```
2021-01-15 16:12:43,604 DEBUG [c.c.a.AlertManagerImpl] (CapacityChecker:ctx-a2a7f3f1) (logid:f7e0a4c5) Running Capacity Checker ...
2021-01-15 16:12:44,927 DEBUG [c.c.a.AlertManagerImpl] (CapacityChecker:ctx-a2a7f3f1) (logid:f7e0a4c5) Done running Capacity Checker ...
```
If there are 100 hosts, the total time will be reduced from 100+ seconds to around 10 seconds.
---
.../src/main/java/com/cloud/capacity/CapacityManager.java | 5 +++++
server/src/main/java/com/cloud/alert/AlertManagerImpl.java | 12 +++++++++++-
.../main/java/com/cloud/capacity/CapacityManagerImpl.java | 11 +++++++++++
3 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java
index 98287bb..8827ca4 100644
--- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java
+++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java
@@ -16,10 +16,13 @@
// under the License.
package com.cloud.capacity;
+import java.util.Map;
+
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import com.cloud.host.Host;
+import com.cloud.service.ServiceOfferingVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.vm.VirtualMachine;
@@ -99,6 +102,8 @@ public interface CapacityManager {
void updateCapacityForHost(Host host);
+ void updateCapacityForHost(Host host, Map<Long, ServiceOfferingVO> offeringsMap);
+
/**
* @param pool storage pool
* @param templateForVmCreation template that will be used for vm creation
diff --git a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java
index 5a7a8b4..75b8cc7 100644
--- a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java
+++ b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java
@@ -75,6 +75,8 @@ import com.cloud.host.HostVO;
import com.cloud.network.dao.IPAddressDao;
import com.cloud.org.Grouping.AllocationState;
import com.cloud.resource.ResourceManager;
+import com.cloud.service.ServiceOfferingVO;
+import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.storage.StorageManager;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.component.ManagerBase;
@@ -121,6 +123,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
private ConfigurationManager _configMgr;
@Inject
protected ConfigDepot _configDepot;
+ @Inject
+ ServiceOfferingDao _offeringsDao;
private Timer _timer = null;
private long _capacityCheckPeriod = 60L * 60L * 1000L; // One hour by default.
@@ -275,8 +279,14 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
// get all hosts...even if they are not in 'UP' state
List<HostVO> hosts = _resourceMgr.listAllNotInMaintenanceHostsInOneZone(Host.Type.Routing, null);
if (hosts != null) {
+ // prepare the service offerings
+ List<ServiceOfferingVO> offerings = _offeringsDao.listAllIncludingRemoved();
+ Map<Long, ServiceOfferingVO> offeringsMap = new HashMap<Long, ServiceOfferingVO>();
+ for (ServiceOfferingVO offering : offerings) {
+ offeringsMap.put(offering.getId(), offering);
+ }
for (HostVO host : hosts) {
- _capacityMgr.updateCapacityForHost(host);
+ _capacityMgr.updateCapacityForHost(host, offeringsMap);
}
}
if (s_logger.isDebugEnabled()) {
diff --git a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java
index 5530e53..f851471 100644
--- a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java
+++ b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java
@@ -627,7 +627,12 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
for (ServiceOfferingVO offering : offerings) {
offeringsMap.put(offering.getId(), offering);
}
+ updateCapacityForHost(host, offeringsMap);
+ }
+ @DB
+ @Override
+ public void updateCapacityForHost(final Host host, final Map<Long, ServiceOfferingVO> offeringsMap) {
long usedCpuCore = 0;
long reservedCpuCore = 0;
long usedCpu = 0;
@@ -664,6 +669,9 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
ramOvercommitRatio = Float.parseFloat(vmDetailRam);
}
ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
+ if (so == null) {
+ so = _offeringsDao.findByIdIncludingRemoved(vm.getServiceOfferingId());
+ }
if (so.isDynamic()) {
usedMemory +=
((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.memory.name())) * 1024L * 1024L) / ramOvercommitRatio) *
@@ -703,6 +711,9 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
}
ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
Map<String, String> vmDetails = _userVmDetailsDao.listDetailsKeyPairs(vm.getId());
+ if (so == null) {
+ so = _offeringsDao.findByIdIncludingRemoved(vm.getServiceOfferingId());
+ }
if (so.isDynamic()) {
reservedMemory +=
((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.memory.name())) * 1024L * 1024L) / ramOvercommitRatio) *