You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ni...@apache.org on 2015/01/26 19:48:27 UTC
git commit: updated refs/heads/master to abf4e5c
Repository: cloudstack
Updated Branches:
refs/heads/master 29aeed71f -> abf4e5c64
CLOUDSTACK-8181: Introducing a new allocator called firstfitleastconsumed. The purpose of this allocator is to find hosts/pools with least capacity usage (in terms of percentage) within the cluster and use those resources first before others. This allocator can be used changing vm.allocation.algorithm. For hosts it would decide the least consumed host through the setting host.capacityType.to.order.clusters to base the usage on cpu or ram.
Reviewed-by: Prachi
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/abf4e5c6
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/abf4e5c6
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/abf4e5c6
Branch: refs/heads/master
Commit: abf4e5c646316269072edd47d693e98182a663c6
Parents: 29aeed7
Author: Nitin Mehta <ni...@citrix.com>
Authored: Mon Jan 26 10:41:46 2015 -0800
Committer: Nitin Mehta <ni...@citrix.com>
Committed: Mon Jan 26 10:48:06 2015 -0800
----------------------------------------------------------------------
.../src/com/cloud/capacity/dao/CapacityDao.java | 2 +
.../com/cloud/capacity/dao/CapacityDaoImpl.java | 25 +++++++++++++
.../allocator/AbstractStoragePoolAllocator.java | 39 ++++++++++++++++++++
.../allocator/impl/FirstFitAllocator.java | 38 +++++++++++++++++++
server/src/com/cloud/configuration/Config.java | 2 +-
5 files changed, 105 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/abf4e5c6/engine/schema/src/com/cloud/capacity/dao/CapacityDao.java
----------------------------------------------------------------------
diff --git a/engine/schema/src/com/cloud/capacity/dao/CapacityDao.java b/engine/schema/src/com/cloud/capacity/dao/CapacityDao.java
index a900e64..50d6052 100644
--- a/engine/schema/src/com/cloud/capacity/dao/CapacityDao.java
+++ b/engine/schema/src/com/cloud/capacity/dao/CapacityDao.java
@@ -56,4 +56,6 @@ public interface CapacityDao extends GenericDao<CapacityVO, Long> {
List<Long> listClustersCrossingThreshold(short capacityType, Long zoneId, String configName, long computeRequested);
float findClusterConsumption(Long clusterId, short capacityType, long computeRequested);
+
+ List<Long> orderHostsByFreeCapacity(Long clusterId, short capacityType);
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/abf4e5c6/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java b/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java
index d9e6184..2bd6bcc 100644
--- a/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java
+++ b/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java
@@ -98,6 +98,8 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
private static final String ORDER_PODS_BY_AGGREGATE_OVERCOMMIT_CAPACITY =
"SELECT capacity.pod_id, SUM(used_capacity+reserved_capacity)/SUM(total_capacity * cluster_details.value) FROM `cloud`.`op_host_capacity` capacity INNER JOIN `cloud`.`cluster_details` cluster_details ON (capacity.cluster_id = cluster_details.cluster_id) WHERE data_center_id=? AND capacity_type = ? AND cluster_details.name = ? GROUP BY capacity.pod_id ORDER BY SUM(used_capacity+reserved_capacity)/SUM(total_capacity * cluster_details.value) ASC";
+ private static final String ORDER_HOSTS_BY_FREE_CAPACITY = "SELECT host_id, SUM(total_capacity - (used_capacity+reserved_capacity))/SUM(total_capacity) FROM `cloud`.`op_host_capacity` WHERE "
+ + " cluster_id = ? AND capacity_type = ? GROUP BY host_id ORDER BY SUM(total_capacity - (used_capacity+reserved_capacity))/SUM(total_capacity) DESC ";
private static final String LIST_CAPACITY_BY_RESOURCE_STATE =
"SELECT capacity.data_center_id, sum(capacity.used_capacity), sum(capacity.reserved_quantity), sum(capacity.total_capacity), capacity_capacity_type "
@@ -858,6 +860,29 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
}
@Override
+ public List<Long> orderHostsByFreeCapacity(Long clusterId, short capacityTypeForOrdering){
+ TransactionLegacy txn = TransactionLegacy.currentTxn();
+ PreparedStatement pstmt = null;
+ List<Long> result = new ArrayList<Long>();
+ StringBuilder sql = new StringBuilder(ORDER_HOSTS_BY_FREE_CAPACITY);
+
+ try {
+ pstmt = txn.prepareAutoCloseStatement(sql.toString());
+ pstmt.setLong(1, clusterId);
+ pstmt.setShort(2, capacityTypeForOrdering);
+ ResultSet rs = pstmt.executeQuery();
+ while (rs.next()) {
+ result.add(rs.getLong(1));
+ }
+ return result;
+ } catch (SQLException e) {
+ throw new CloudRuntimeException("DB Exception on: " + sql, e);
+ } catch (Throwable e) {
+ throw new CloudRuntimeException("Caught: " + sql, e);
+ }
+ }
+
+ @Override
public List<Long> listPodsByHostCapacities(long zoneId, int requiredCpu, long requiredRam, short capacityType) {
TransactionLegacy txn = TransactionLegacy.currentTxn();
PreparedStatement pstmt = null;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/abf4e5c6/engine/storage/src/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java
index 1bb85c1..73a8544 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java
@@ -35,6 +35,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import com.cloud.capacity.Capacity;
+import com.cloud.capacity.dao.CapacityDao;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.deploy.DeploymentPlan;
@@ -72,6 +74,8 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement
protected String _allocationAlgorithm = "random";
@Inject
DiskOfferingDao _diskOfferingDao;
+ @Inject
+ CapacityDao _capacityDao;
@Override
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
@@ -100,6 +104,39 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement
return reOrder(pools, vmProfile, plan);
}
+ protected List<StoragePool> reorderPoolsByCapacity(DeploymentPlan plan,
+ List<StoragePool> pools) {
+ Long clusterId = plan.getClusterId();
+ short capacityType;
+ if(pools != null && pools.size() != 0){
+ capacityType = pools.get(0).getPoolType().isShared() == true ?
+ Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED : Capacity.CAPACITY_TYPE_LOCAL_STORAGE;
+ } else{
+ return null;
+ }
+
+ List<Long> poolIdsByCapacity = _capacityDao.orderHostsByFreeCapacity(clusterId, capacityType);
+ if (s_logger.isDebugEnabled()) {
+ s_logger.debug("List of pools in descending order of free capacity: "+ poolIdsByCapacity);
+ }
+
+ //now filter the given list of Pools by this ordered list
+ Map<Long, StoragePool> poolMap = new HashMap<Long, StoragePool>();
+ for (StoragePool pool : pools) {
+ poolMap.put(pool.getId(), pool);
+ }
+ List<Long> matchingPoolIds = new ArrayList<Long>(poolMap.keySet());
+
+ poolIdsByCapacity.retainAll(matchingPoolIds);
+
+ List<StoragePool> reorderedPools = new ArrayList<StoragePool>();
+ for(Long id: poolIdsByCapacity){
+ reorderedPools.add(poolMap.get(id));
+ }
+
+ return reorderedPools;
+ }
+
protected List<StoragePool> reorderPoolsByNumberOfVolumes(DeploymentPlan plan, List<StoragePool> pools, Account account) {
if (account == null) {
return pools;
@@ -144,6 +181,8 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement
Collections.shuffle(pools);
} else if (_allocationAlgorithm.equals("userdispersing")) {
pools = reorderPoolsByNumberOfVolumes(plan, pools, account);
+ } else if(_allocationAlgorithm.equals("firstfitleastconsumed")){
+ pools = reorderPoolsByCapacity(plan, pools);
}
return pools;
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/abf4e5c6/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java
index a746eb7..44552af 100644
--- a/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java
+++ b/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java
@@ -32,6 +32,9 @@ import org.springframework.stereotype.Component;
import com.cloud.agent.manager.allocator.HostAllocator;
import com.cloud.capacity.CapacityManager;
+import com.cloud.capacity.CapacityVO;
+import com.cloud.capacity.dao.CapacityDao;
+import com.cloud.configuration.Config;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.dao.ClusterDao;
@@ -89,6 +92,8 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator {
ServiceOfferingDetailsDao _serviceOfferingDetailsDao;
@Inject
CapacityManager _capacityMgr;
+ @Inject
+ CapacityDao _capacityDao;
boolean _checkHvm = true;
protected String _allocationAlgorithm = "random";
@@ -235,6 +240,8 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator {
Collections.shuffle(hosts);
} else if (_allocationAlgorithm.equals("userdispersing")) {
hosts = reorderHostsByNumberOfVms(plan, hosts, account);
+ }else if(_allocationAlgorithm.equals("firstfitleastconsumed")){
+ hosts = reorderHostsByCapacity(plan, hosts);
}
if (s_logger.isDebugEnabled()) {
@@ -318,6 +325,37 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator {
return suitableHosts;
}
+ // Reorder hosts in the decreasing order of free capacity.
+ private List<? extends Host> reorderHostsByCapacity(DeploymentPlan plan, List<? extends Host> hosts) {
+ Long clusterId = plan.getClusterId();
+ //Get capacity by which we should reorder
+ String capacityTypeToOrder = _configDao.getValue(Config.HostCapacityTypeToOrderClusters.key());
+ short capacityType = CapacityVO.CAPACITY_TYPE_CPU;
+ if("RAM".equalsIgnoreCase(capacityTypeToOrder)){
+ capacityType = CapacityVO.CAPACITY_TYPE_MEMORY;
+ }
+ List<Long> hostIdsByFreeCapacity = _capacityDao.orderHostsByFreeCapacity(clusterId, capacityType);
+ if (s_logger.isDebugEnabled()) {
+ s_logger.debug("List of hosts in descending order of free capacity in the cluster: "+ hostIdsByFreeCapacity);
+ }
+
+ //now filter the given list of Hosts by this ordered list
+ Map<Long, Host> hostMap = new HashMap<Long, Host>();
+ for (Host host : hosts) {
+ hostMap.put(host.getId(), host);
+ }
+ List<Long> matchingHostIds = new ArrayList<Long>(hostMap.keySet());
+
+ hostIdsByFreeCapacity.retainAll(matchingHostIds);
+
+ List<Host> reorderedHosts = new ArrayList<Host>();
+ for(Long id: hostIdsByFreeCapacity){
+ reorderedHosts.add(hostMap.get(id));
+ }
+
+ return reorderedHosts;
+ }
+
private List<? extends Host> reorderHostsByNumberOfVms(DeploymentPlan plan, List<? extends Host> hosts, Account account) {
if (account == null) {
return hosts;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/abf4e5c6/server/src/com/cloud/configuration/Config.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java
index f9e9bdd..453a3bc 100644
--- a/server/src/com/cloud/configuration/Config.java
+++ b/server/src/com/cloud/configuration/Config.java
@@ -980,7 +980,7 @@ public enum Config {
String.class,
"vm.allocation.algorithm",
"random",
- "'random', 'firstfit', 'userdispersing', 'userconcentratedpod_random', 'userconcentratedpod_firstfit' : Order in which hosts within a cluster will be considered for VM/volume allocation.",
+ "'random', 'firstfit', 'userdispersing', 'userconcentratedpod_random', 'userconcentratedpod_firstfit', 'firstfitleastconsumed' : Order in which hosts within a cluster will be considered for VM/volume allocation.",
null),
VmDeploymentPlanner(
"Advanced",