You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by hu...@apache.org on 2014/02/14 18:38:12 UTC
[12/22] git commit: updated refs/heads/master to 443acac
Fix findbugs findings in cloud-plugin-network-elb
Findings:
- comparing Longs by reference
- possible null pointer dereference
Signed-off-by: Hugo Trippaers <ht...@schubergphilis.com>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/8649fa00
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/8649fa00
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/8649fa00
Branch: refs/heads/master
Commit: 8649fa005a94f0e50f7268c7eccdf003e1107672
Parents: 5a6ad11
Author: Miguel Ferreira <mf...@shubergphilis.com>
Authored: Mon Feb 10 20:05:23 2014 +0100
Committer: Hugo Trippaers <ht...@schubergphilis.com>
Committed: Fri Feb 14 18:37:46 2014 +0100
----------------------------------------------------------------------
.../lb/ElasticLoadBalancerManagerImpl.java | 486 +++----------------
.../network/lb/LoadBalanceRuleHandler.java | 466 ++++++++++++++++++
.../lb/ElasticLoadBalancerManagerImplTest.java | 101 ++++
.../network/lb/LoadBalanceRuleHandlerTest.java | 215 ++++++++
4 files changed, 847 insertions(+), 421 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8649fa00/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java
----------------------------------------------------------------------
diff --git a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java
index 6dd0f8a..2a7bcac 100644
--- a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java
+++ b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java
@@ -18,12 +18,9 @@ package com.cloud.network.lb;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
@@ -36,8 +33,6 @@ import javax.naming.ConfigurationException;
import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd;
import org.apache.cloudstack.config.ApiServiceConfiguration;
-import org.apache.cloudstack.context.CallContext;
-import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.log4j.Logger;
@@ -56,54 +51,31 @@ import com.cloud.configuration.Config;
import com.cloud.dc.DataCenter;
import com.cloud.dc.DataCenter.NetworkType;
import com.cloud.dc.DataCenterVO;
-import com.cloud.dc.Pod;
-import com.cloud.dc.PodVlanMapVO;
-import com.cloud.dc.Vlan.VlanType;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.dc.dao.HostPodDao;
-import com.cloud.dc.dao.PodVlanMapDao;
-import com.cloud.dc.dao.VlanDao;
-import com.cloud.deploy.DataCenterDeployment;
import com.cloud.deploy.DeployDestination;
import com.cloud.exception.AgentUnavailableException;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientAddressCapacityException;
-import com.cloud.exception.InsufficientCapacityException;
-import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.OperationTimedoutException;
import com.cloud.exception.ResourceUnavailableException;
-import com.cloud.exception.StorageUnavailableException;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.network.ElasticLbVmMapVO;
-import com.cloud.network.IpAddressManager;
import com.cloud.network.Network;
-import com.cloud.network.Network.Provider;
-import com.cloud.network.Network.Service;
import com.cloud.network.NetworkModel;
import com.cloud.network.Networks.TrafficType;
-import com.cloud.network.PhysicalNetworkServiceProvider;
-import com.cloud.network.VirtualRouterProvider;
-import com.cloud.network.VirtualRouterProvider.Type;
-import com.cloud.network.addr.PublicIp;
-import com.cloud.network.dao.IPAddressDao;
-import com.cloud.network.dao.IPAddressVO;
import com.cloud.network.dao.LoadBalancerDao;
import com.cloud.network.dao.LoadBalancerVO;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkVO;
-import com.cloud.network.dao.PhysicalNetworkServiceProviderDao;
-import com.cloud.network.dao.VirtualRouterProviderDao;
import com.cloud.network.lb.LoadBalancingRule.LbDestination;
import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy;
import com.cloud.network.lb.LoadBalancingRule.LbSslCert;
import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy;
import com.cloud.network.lb.dao.ElasticLbVmMapDao;
import com.cloud.network.router.VirtualRouter;
-import com.cloud.network.router.VirtualRouter.RedundantState;
-import com.cloud.network.router.VirtualRouter.Role;
-import com.cloud.network.router.VpcVirtualNetworkApplianceManager;
import com.cloud.network.rules.FirewallRule;
import com.cloud.network.rules.LoadBalancer;
import com.cloud.offering.NetworkOffering;
@@ -111,21 +83,11 @@ import com.cloud.offering.ServiceOffering;
import com.cloud.offerings.dao.NetworkOfferingDao;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
-import com.cloud.storage.VMTemplateVO;
-import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.user.Account;
import com.cloud.user.AccountService;
-import com.cloud.user.User;
-import com.cloud.user.dao.AccountDao;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
-import com.cloud.utils.db.DB;
-import com.cloud.utils.db.SearchBuilder;
-import com.cloud.utils.db.SearchCriteria;
-import com.cloud.utils.db.Transaction;
-import com.cloud.utils.db.TransactionCallbackWithException;
-import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.net.Ip;
import com.cloud.vm.DomainRouterVO;
@@ -135,9 +97,7 @@ import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachine.State;
import com.cloud.vm.VirtualMachineGuru;
import com.cloud.vm.VirtualMachineManager;
-import com.cloud.vm.VirtualMachineName;
import com.cloud.vm.VirtualMachineProfile;
-import com.cloud.vm.VirtualMachineProfile.Param;
import com.cloud.vm.dao.DomainRouterDao;
import com.cloud.vm.dao.NicDao;
@@ -147,64 +107,40 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
private static final Logger s_logger = Logger.getLogger(ElasticLoadBalancerManagerImpl.class);
@Inject
- IPAddressDao _ipAddressDao;
+ private AgentManager _agentMgr;
@Inject
- AgentManager _agentMgr;
+ private NetworkModel _networkModel;
@Inject
- NetworkModel _networkModel;
+ private LoadBalancingRulesManager _lbMgr;
@Inject
- NetworkOrchestrationService _networkMgr;
- @Inject
- LoadBalancerDao _loadBalancerDao = null;
- @Inject
- LoadBalancingRulesManager _lbMgr;
- @Inject
- VpcVirtualNetworkApplianceManager _routerMgr;
- @Inject
- DomainRouterDao _routerDao = null;
+ private final DomainRouterDao _routerDao = null;
@Inject
protected HostPodDao _podDao = null;
@Inject
protected ClusterDao _clusterDao;
@Inject
- DataCenterDao _dcDao = null;
- @Inject
- IpAddressManager _ipAddrMgr;
+ private final DataCenterDao _dcDao = null;
@Inject
protected NetworkDao _networkDao;
@Inject
protected NetworkOfferingDao _networkOfferingDao;
@Inject
- VMTemplateDao _templateDao = null;
- @Inject
- VirtualMachineManager _itMgr;
- @Inject
- ConfigurationDao _configDao;
- @Inject
- ServiceOfferingDao _serviceOfferingDao = null;
+ private VirtualMachineManager _itMgr;
@Inject
- AccountService _accountService;
+ private ConfigurationDao _configDao;
@Inject
- LoadBalancerDao _lbDao;
+ private final ServiceOfferingDao _serviceOfferingDao = null;
@Inject
- VlanDao _vlanDao;
+ private AccountService _accountService;
@Inject
- PodVlanMapDao _podVlanMapDao;
+ private LoadBalancerDao _lbDao;
@Inject
- ElasticLbVmMapDao _elbVmMapDao;
+ private ElasticLbVmMapDao _elbVmMapDao;
@Inject
- NetworkDao _networksDao;
- @Inject
- AccountDao _accountDao;
- @Inject
- PhysicalNetworkServiceProviderDao _physicalProviderDao;
- @Inject
- VirtualRouterProviderDao _vrProviderDao;
- @Inject
- NicDao _nicDao;
+ private NicDao _nicDao;
String _instance;
- static final private String ElbVmNamePrefix = "l";
+
static final private String SystemVmType = "elbvm";
boolean _enabled;
@@ -221,41 +157,7 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
int _elasticLbvmCpuMHz;
int _elasticLbvmNumCpu;
- private Long getPodIdForDirectIp(IPAddressVO ipAddr) {
- PodVlanMapVO podVlanMaps = _podVlanMapDao.listPodVlanMapsByVlan(ipAddr.getVlanId());
- if (podVlanMaps == null) {
- return null;
- } else {
- return podVlanMaps.getPodId();
- }
- }
-
- public DomainRouterVO deployLoadBalancerVM(Long networkId, IPAddressVO ipAddr, Long accountId) {
- NetworkVO network = _networkDao.findById(networkId);
- DataCenter dc = _dcDao.findById(network.getDataCenterId());
- Long podId = getPodIdForDirectIp(ipAddr);
- Pod pod = podId == null ? null : _podDao.findById(podId);
- Map<VirtualMachineProfile.Param, Object> params = new HashMap<VirtualMachineProfile.Param, Object>(1);
- params.put(VirtualMachineProfile.Param.ReProgramGuestNetworks, true);
- Account owner = _accountService.getActiveAccountByName("system", new Long(1));
- DeployDestination dest = new DeployDestination(dc, pod, null, null);
- s_logger.debug("About to deploy ELB vm ");
-
- try {
- DomainRouterVO elbVm = deployELBVm(network, dest, owner, params);
- if (elbVm == null) {
- throw new InvalidParameterValueException("Could not deploy or find existing ELB VM");
- }
- s_logger.debug("Deployed ELB vm = " + elbVm);
-
- return elbVm;
-
- } catch (Throwable t) {
- s_logger.warn("Error while deploying ELB VM: ", t);
- return null;
- }
-
- }
+ private LoadBalanceRuleHandler loadBalanceRuleHandler;
private boolean sendCommandsToRouter(final DomainRouterVO elbVm, Commands cmds) throws AgentUnavailableException {
Answer[] answers = null;
@@ -307,9 +209,8 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
} else {
maxconn = offering.getConcurrentConnections().toString();
}
- LoadBalancerConfigCommand cmd =
- new LoadBalancerConfigCommand(lbs, elbVm.getPublicIpAddress(), _nicDao.getIpAddress(guestNetworkId, elbVm.getId()), elbVm.getPrivateIpAddress(), null, null,
- maxconn, offering.isKeepAliveEnabled());
+ LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs, elbVm.getPublicIpAddress(), _nicDao.getIpAddress(guestNetworkId, elbVm.getId()),
+ elbVm.getPrivateIpAddress(), null, null, maxconn, offering.isKeepAliveEnabled());
cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, elbVm.getPrivateIpAddress());
cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, elbVm.getInstanceName());
//FIXME: why are we setting attributes directly? Ick!! There should be accessors and
@@ -393,9 +294,8 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
_elasticLbVmRamSize = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmMemory.key()), DEFAULT_ELB_VM_RAMSIZE);
_elasticLbvmCpuMHz = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmCpuMhz.key()), DEFAULT_ELB_VM_CPU_MHZ);
_elasticLbvmNumCpu = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmNumVcpu.key()), 1);
- _elasticLbVmOffering =
- new ServiceOfferingVO("System Offering For Elastic LB VM", _elasticLbvmNumCpu, _elasticLbVmRamSize, _elasticLbvmCpuMHz, 0, 0, true, null, useLocalStorage,
- true, null, true, VirtualMachine.Type.ElasticLoadBalancerVm, true);
+ _elasticLbVmOffering = new ServiceOfferingVO("System Offering For Elastic LB VM", _elasticLbvmNumCpu, _elasticLbVmRamSize, _elasticLbvmCpuMHz, 0, 0, true, null,
+ useLocalStorage, true, null, true, VirtualMachine.Type.ElasticLoadBalancerVm, true);
_elasticLbVmOffering.setUniqueName(ServiceOffering.elbVmDefaultOffUniqueName);
_elasticLbVmOffering = _serviceOfferingDao.persistSystemServiceOffering(_elasticLbVmOffering);
@@ -420,110 +320,12 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
_itMgr.registerGuru(VirtualMachine.Type.ElasticLoadBalancerVm, this);
}
- return true;
- }
-
- private DomainRouterVO findELBVmWithCapacity(Network guestNetwork, IPAddressVO ipAddr) {
- List<DomainRouterVO> unusedElbVms = _elbVmMapDao.listUnusedElbVms();
- if (unusedElbVms.size() > 0) {
- List<DomainRouterVO> candidateVms = new ArrayList<DomainRouterVO>();
- for (DomainRouterVO candidateVm : unusedElbVms) {
- if (candidateVm.getPodIdToDeployIn() == getPodIdForDirectIp(ipAddr))
- candidateVms.add(candidateVm);
- }
- return candidateVms.size() == 0 ? null : candidateVms.get(new Random().nextInt(candidateVms.size()));
- }
- return null;
- }
-
- public DomainRouterVO deployELBVm(Network guestNetwork, DeployDestination dest, Account owner, Map<Param, Object> params) throws ConcurrentOperationException,
- ResourceUnavailableException, InsufficientCapacityException {
- long dcId = dest.getDataCenter().getId();
-
- // lock guest network
- Long guestNetworkId = guestNetwork.getId();
- guestNetwork = _networkDao.acquireInLockTable(guestNetworkId);
-
- if (guestNetwork == null) {
- throw new ConcurrentOperationException("Unable to acquire network lock: " + guestNetworkId);
- }
-
- try {
-
- if (_networkModel.isNetworkSystem(guestNetwork) || guestNetwork.getGuestType() == Network.GuestType.Shared) {
- owner = _accountService.getSystemAccount();
- }
-
- if (s_logger.isDebugEnabled()) {
- s_logger.debug("Starting a ELB vm for network configurations: " + guestNetwork + " in " + dest);
- }
- assert guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup ||
- guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: " + guestNetwork;
-
- DataCenterDeployment plan = null;
- DomainRouterVO elbVm = null;
-
- plan = new DataCenterDeployment(dcId, dest.getPod().getId(), null, null, null, null);
-
- if (elbVm == null) {
- long id = _routerDao.getNextInSequence(Long.class, "id");
- if (s_logger.isDebugEnabled()) {
- s_logger.debug("Creating the ELB vm " + id);
- }
-
- List<? extends NetworkOffering> offerings = _networkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemControlNetwork);
- NetworkOffering controlOffering = offerings.get(0);
- Network controlConfig = _networkMgr.setupNetwork(_systemAcct, controlOffering, plan, null, null, false).get(0);
-
- LinkedHashMap<Network, NicProfile> networks = new LinkedHashMap<Network, NicProfile>(2);
- NicProfile guestNic = new NicProfile();
- guestNic.setDefaultNic(true);
- networks.put(controlConfig, null);
- networks.put(guestNetwork, guestNic);
-
- VMTemplateVO template = _templateDao.findSystemVMTemplate(dcId);
-
- String typeString = "ElasticLoadBalancerVm";
- Long physicalNetworkId = _networkModel.getPhysicalNetworkId(guestNetwork);
- PhysicalNetworkServiceProvider provider = _physicalProviderDao.findByServiceProvider(physicalNetworkId, typeString);
- if (provider == null) {
- throw new CloudRuntimeException("Cannot find service provider " + typeString + " in physical network " + physicalNetworkId);
- }
- VirtualRouterProvider vrProvider = _vrProviderDao.findByNspIdAndType(provider.getId(), Type.ElasticLoadBalancerVm);
- if (vrProvider == null) {
- throw new CloudRuntimeException("Cannot find virtual router provider " + typeString + " as service provider " + provider.getId());
- }
-
- elbVm =
- new DomainRouterVO(id, _elasticLbVmOffering.getId(), vrProvider.getId(), VirtualMachineName.getSystemVmName(id, _instance, ElbVmNamePrefix),
- template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), false, 0, false,
- RedundantState.UNKNOWN, _elasticLbVmOffering.getOfferHA(), false, VirtualMachine.Type.ElasticLoadBalancerVm, null);
- elbVm.setRole(Role.LB);
- elbVm = _routerDao.persist(elbVm);
- _itMgr.allocate(elbVm.getInstanceName(), template, _elasticLbVmOffering, networks, plan, null);
- elbVm = _routerDao.findById(elbVm.getId());
- //TODO: create usage stats
- }
-
- State state = elbVm.getState();
- if (state != State.Running) {
- elbVm = this.start(elbVm, _accountService.getSystemUser(), _accountService.getSystemAccount(), params);
- }
-
- return elbVm;
- } finally {
- _networkDao.releaseFromLockTable(guestNetworkId);
- }
- }
+ loadBalanceRuleHandler = new LoadBalanceRuleHandler(_elasticLbVmOffering, _instance, _systemAcct);
- private DomainRouterVO start(DomainRouterVO elbVm, User user, Account caller, Map<Param, Object> params) throws StorageUnavailableException,
- InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException {
- s_logger.debug("Starting ELB VM " + elbVm);
- _itMgr.start(elbVm.getUuid(), params);
- return _routerDao.findById(elbVm.getId());
+ return true;
}
- private DomainRouterVO stop(DomainRouterVO elbVm, boolean forced, User user, Account caller) throws ConcurrentOperationException, ResourceUnavailableException {
+ private DomainRouterVO stop(DomainRouterVO elbVm, boolean forced) throws ConcurrentOperationException, ResourceUnavailableException {
s_logger.debug("Stopping ELB vm " + elbVm);
try {
_itMgr.advanceStop(elbVm.getUuid(), forced);
@@ -533,208 +335,55 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
}
}
- protected List<LoadBalancerVO> findExistingLoadBalancers(String lbName, Long ipId, Long accountId, Long domainId, Integer publicPort) {
- SearchBuilder<LoadBalancerVO> sb = _lbDao.createSearchBuilder();
- sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ);
- sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ);
- sb.and("publicPort", sb.entity().getSourcePortStart(), SearchCriteria.Op.EQ);
- if (ipId != null) {
- sb.and("sourceIpAddress", sb.entity().getSourceIpAddressId(), SearchCriteria.Op.EQ);
- }
- if (domainId != null) {
- sb.and("domainId", sb.entity().getDomainId(), SearchCriteria.Op.EQ);
- }
- if (publicPort != null) {
- sb.and("publicPort", sb.entity().getSourcePortStart(), SearchCriteria.Op.EQ);
- }
- SearchCriteria<LoadBalancerVO> sc = sb.create();
- sc.setParameters("name", lbName);
- sc.setParameters("accountId", accountId);
- if (ipId != null) {
- sc.setParameters("sourceIpAddress", ipId);
- }
- if (domainId != null) {
- sc.setParameters("domainId", domainId);
- }
- if (publicPort != null) {
- sc.setParameters("publicPort", publicPort);
- }
- List<LoadBalancerVO> lbs = _lbDao.search(sc, null);
-
- return lbs == null || lbs.size() == 0 ? null : lbs;
- }
-
- @DB
- public PublicIp allocDirectIp(final Account account, final long guestNetworkId) throws InsufficientAddressCapacityException {
- return Transaction.execute(new TransactionCallbackWithException<PublicIp, InsufficientAddressCapacityException>() {
- @Override
- public PublicIp doInTransaction(TransactionStatus status) throws InsufficientAddressCapacityException {
- Network frontEndNetwork = _networkModel.getNetwork(guestNetworkId);
-
- PublicIp ip =
- _ipAddrMgr.assignPublicIpAddress(frontEndNetwork.getDataCenterId(), null, account, VlanType.DirectAttached, frontEndNetwork.getId(), null, true);
- IPAddressVO ipvo = _ipAddressDao.findById(ip.getId());
- ipvo.setAssociatedWithNetworkId(frontEndNetwork.getId());
- _ipAddressDao.update(ipvo.getId(), ipvo);
- s_logger.info("Acquired frontend IP for ELB " + ip);
-
- return ip;
- }
- });
- }
-
- public void releaseIp(long ipId, long userId, Account caller) {
- s_logger.info("ELB: Release public IP for loadbalancing " + ipId);
- IPAddressVO ipvo = _ipAddressDao.findById(ipId);
- ipvo.setAssociatedWithNetworkId(null);
- _ipAddressDao.update(ipvo.getId(), ipvo);
- _ipAddrMgr.disassociatePublicIpAddress(ipId, userId, caller);
- _ipAddressDao.unassignIpAddress(ipId);
- }
-
@Override
- @DB
public LoadBalancer handleCreateLoadBalancerRule(CreateLoadBalancerRuleCmd lb, Account account, long networkId) throws InsufficientAddressCapacityException,
- NetworkRuleConflictException {
- //this part of code is executed when the LB provider is Elastic Load Balancer vm
- if (!_networkModel.isProviderSupportServiceInNetwork(lb.getNetworkId(), Service.Lb, Provider.ElasticLoadBalancerVm)) {
- return null;
- }
-
- Long ipId = lb.getSourceIpAddressId();
- if (ipId != null) {
- return null;
- }
- boolean newIp = false;
- account = _accountDao.acquireInLockTable(account.getId());
- if (account == null) {
- s_logger.warn("ELB: CreateLoadBalancer: Failed to acquire lock on account");
- throw new CloudRuntimeException("Failed to acquire lock on account");
- }
- try {
- List<LoadBalancerVO> existingLbs =
- findExistingLoadBalancers(lb.getName(), lb.getSourceIpAddressId(), lb.getAccountId(), lb.getDomainId(), lb.getSourcePortStart());
- if (existingLbs == null) {
- existingLbs = findExistingLoadBalancers(lb.getName(), lb.getSourceIpAddressId(), lb.getAccountId(), lb.getDomainId(), null);
- if (existingLbs == null) {
- if (lb.getSourceIpAddressId() != null) {
- existingLbs = findExistingLoadBalancers(lb.getName(), null, lb.getAccountId(), lb.getDomainId(), null);
- if (existingLbs != null) {
- throw new InvalidParameterValueException("Supplied LB name " + lb.getName() + " is not associated with IP " + lb.getSourceIpAddressId());
- }
- } else {
- s_logger.debug("Could not find any existing frontend ips for this account for this LB rule, acquiring a new frontent IP for ELB");
- PublicIp ip = allocDirectIp(account, networkId);
- ipId = ip.getId();
- newIp = true;
- }
- } else {
- ipId = existingLbs.get(0).getSourceIpAddressId();
- s_logger.debug("ELB: Found existing frontend ip for this account for this LB rule " + ipId);
- }
- } else {
- s_logger.warn("ELB: Found existing load balancers matching requested new LB");
- throw new NetworkRuleConflictException("ELB: Found existing load balancers matching requested new LB");
- }
-
- Network network = _networkModel.getNetwork(networkId);
- IPAddressVO ipAddr = _ipAddressDao.findById(ipId);
-
- LoadBalancer result = null;
- try {
- lb.setSourceIpAddressId(ipId);
-
- result =
- _lbMgr.createPublicLoadBalancer(lb.getXid(), lb.getName(), lb.getDescription(), lb.getSourcePortStart(), lb.getDefaultPortStart(), ipId.longValue(),
- lb.getProtocol(), lb.getAlgorithm(), false, CallContext.current(), lb.getLbProtocol());
- } catch (NetworkRuleConflictException e) {
- s_logger.warn("Failed to create LB rule, not continuing with ELB deployment");
- if (newIp) {
- releaseIp(ipId, CallContext.current().getCallingUserId(), account);
- }
- throw e;
- }
-
- DomainRouterVO elbVm = null;
-
- if (existingLbs == null) {
- elbVm = findELBVmWithCapacity(network, ipAddr);
- if (elbVm == null) {
- elbVm = deployLoadBalancerVM(networkId, ipAddr, account.getId());
- if (elbVm == null) {
- s_logger.warn("Failed to deploy a new ELB vm for ip " + ipAddr + " in network " + network + "lb name=" + lb.getName());
- if (newIp)
- releaseIp(ipId, CallContext.current().getCallingUserId(), account);
- }
- }
-
- } else {
- ElasticLbVmMapVO elbVmMap = _elbVmMapDao.findOneByIp(ipId);
- if (elbVmMap != null) {
- elbVm = _routerDao.findById(elbVmMap.getElbVmId());
- }
- }
-
- if (elbVm == null) {
- s_logger.warn("No ELB VM can be found or deployed");
- s_logger.warn("Deleting LB since we failed to deploy ELB VM");
- _lbDao.remove(result.getId());
- return null;
- }
-
- ElasticLbVmMapVO mapping = new ElasticLbVmMapVO(ipId, elbVm.getId(), result.getId());
- _elbVmMapDao.persist(mapping);
- return result;
-
- } finally {
- if (account != null) {
- _accountDao.releaseFromLockTable(account.getId());
- }
- }
-
+ NetworkRuleConflictException {
+ return loadBalanceRuleHandler.handleCreateLoadBalancerRule(lb, account, networkId);
}
void garbageCollectUnusedElbVms() {
List<DomainRouterVO> unusedElbVms = _elbVmMapDao.listUnusedElbVms();
- if (unusedElbVms != null && unusedElbVms.size() > 0)
- s_logger.info("Found " + unusedElbVms.size() + " unused ELB vms");
- Set<Long> currentGcCandidates = new HashSet<Long>();
- for (DomainRouterVO elbVm : unusedElbVms) {
- currentGcCandidates.add(elbVm.getId());
- }
- _gcCandidateElbVmIds.retainAll(currentGcCandidates);
- currentGcCandidates.removeAll(_gcCandidateElbVmIds);
- User user = _accountService.getSystemUser();
- for (Long elbVmId : _gcCandidateElbVmIds) {
- DomainRouterVO elbVm = _routerDao.findById(elbVmId);
- boolean gceed = false;
-
- try {
- s_logger.info("Attempting to stop ELB VM: " + elbVm);
- stop(elbVm, true, user, _systemAcct);
- gceed = true;
- } catch (ConcurrentOperationException e) {
- s_logger.warn("Unable to stop unused ELB vm " + elbVm + " due to ", e);
- } catch (ResourceUnavailableException e) {
- s_logger.warn("Unable to stop unused ELB vm " + elbVm + " due to ", e);
- continue;
+ if (unusedElbVms != null) {
+ if (unusedElbVms.size() > 0) {
+ s_logger.info("Found " + unusedElbVms.size() + " unused ELB vms");
+ }
+ Set<Long> currentGcCandidates = new HashSet<Long>();
+ for (DomainRouterVO elbVm : unusedElbVms) {
+ currentGcCandidates.add(elbVm.getId());
}
- if (gceed) {
+ _gcCandidateElbVmIds.retainAll(currentGcCandidates);
+ currentGcCandidates.removeAll(_gcCandidateElbVmIds);
+ for (Long elbVmId : _gcCandidateElbVmIds) {
+ DomainRouterVO elbVm = _routerDao.findById(elbVmId);
+ boolean gceed = false;
+
try {
- s_logger.info("Attempting to destroy ELB VM: " + elbVm);
- _itMgr.expunge(elbVm.getUuid());
- _routerDao.remove(elbVm.getId());
+ s_logger.info("Attempting to stop ELB VM: " + elbVm);
+ stop(elbVm, true);
+ gceed = true;
+ } catch (ConcurrentOperationException e) {
+ s_logger.warn("Unable to stop unused ELB vm " + elbVm + " due to ", e);
} catch (ResourceUnavailableException e) {
- s_logger.warn("Unable to destroy unused ELB vm " + elbVm + " due to ", e);
- gceed = false;
+ s_logger.warn("Unable to stop unused ELB vm " + elbVm + " due to ", e);
+ continue;
+ }
+ if (gceed) {
+ try {
+ s_logger.info("Attempting to destroy ELB VM: " + elbVm);
+ _itMgr.expunge(elbVm.getUuid());
+ _routerDao.remove(elbVm.getId());
+ } catch (ResourceUnavailableException e) {
+ s_logger.warn("Unable to destroy unused ELB vm " + elbVm + " due to ", e);
+ gceed = false;
+ }
+ }
+ if (!gceed) {
+ currentGcCandidates.add(elbVm.getId());
}
- }
- if (!gceed) {
- currentGcCandidates.add(elbVm.getId());
- }
+ }
+ _gcCandidateElbVmIds = currentGcCandidates;
}
- _gcCandidateElbVmIds = currentGcCandidates;
}
public class CleanupThread extends ManagedContextRunnable {
@@ -754,11 +403,7 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
if (!_enabled) {
return;
}
- List<LoadBalancerVO> remainingLbs = _loadBalancerDao.listByIpAddress(lb.getSourceIpAddressId());
- if (remainingLbs.size() == 0) {
- s_logger.debug("ELB mgr: releasing ip " + lb.getSourceIpAddressId() + " since no LB rules remain for this ip address");
- releaseIp(lb.getSourceIpAddressId(), userId, caller);
- }
+ loadBalanceRuleHandler.handleDeleteLoadBalancerRule(lb, userId, caller);
}
@Override
@@ -799,9 +444,9 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
// control command is sent over management network in VMware
if (dest.getHost().getHypervisorType() == HypervisorType.VMware) {
if (s_logger.isInfoEnabled()) {
- s_logger.info("Check if we need to add management server explicit route to ELB vm. pod cidr: " + dest.getPod().getCidrAddress() + "/" +
- dest.getPod().getCidrSize() + ", pod gateway: " + dest.getPod().getGateway() + ", management host: " +
- ApiServiceConfiguration.ManagementHostIPAdr.value());
+ s_logger.info("Check if we need to add management server explicit route to ELB vm. pod cidr: " + dest.getPod().getCidrAddress() + "/"
+ + dest.getPod().getCidrSize() + ", pod gateway: " + dest.getPod().getGateway() + ", management host: "
+ + ApiServiceConfiguration.ManagementHostIPAdr.value());
}
if (s_logger.isDebugEnabled()) {
@@ -843,8 +488,7 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
}
@Override
- public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context)
- throws ResourceUnavailableException {
+ public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) throws ResourceUnavailableException {
DomainRouterVO elbVm = _routerDao.findById(profile.getVirtualMachine().getId());
List<NicProfile> nics = profile.getNics();
@@ -868,7 +512,7 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Commands cmds, ReservationContext context) {
CheckSshAnswer answer = (CheckSshAnswer)cmds.getAnswer("checkSsh");
if (answer == null || !answer.getResult()) {
- s_logger.warn("Unable to ssh to the ELB VM: " + answer.getDetails());
+ s_logger.warn("Unable to ssh to the ELB VM: " + (answer != null ? answer.getDetails() : "No answer (answer for \"checkSsh\" was null)"));
return false;
}
@@ -937,6 +581,7 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
}
}
+ @SuppressWarnings("unused")
public void processStopOrRebootAnswer(final DomainRouterVO elbVm, Answer answer) {
//TODO: process network usage stats
}
@@ -944,7 +589,6 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
@Override
public void finalizeExpunge(VirtualMachine vm) {
// no-op
-
}
@Override
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8649fa00/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/LoadBalanceRuleHandler.java
----------------------------------------------------------------------
diff --git a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/LoadBalanceRuleHandler.java b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/LoadBalanceRuleHandler.java
new file mode 100644
index 0000000..967e68f
--- /dev/null
+++ b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/LoadBalanceRuleHandler.java
@@ -0,0 +1,466 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.network.lb;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.log4j.Logger;
+
+import com.cloud.dc.DataCenter;
+import com.cloud.dc.Pod;
+import com.cloud.dc.PodVlanMapVO;
+import com.cloud.dc.Vlan.VlanType;
+import com.cloud.dc.dao.ClusterDao;
+import com.cloud.dc.dao.DataCenterDao;
+import com.cloud.dc.dao.HostPodDao;
+import com.cloud.dc.dao.PodVlanMapDao;
+import com.cloud.deploy.DataCenterDeployment;
+import com.cloud.deploy.DeployDestination;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientAddressCapacityException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.network.ElasticLbVmMapVO;
+import com.cloud.network.IpAddressManager;
+import com.cloud.network.Network;
+import com.cloud.network.Network.Provider;
+import com.cloud.network.Network.Service;
+import com.cloud.network.NetworkModel;
+import com.cloud.network.PhysicalNetworkServiceProvider;
+import com.cloud.network.VirtualRouterProvider;
+import com.cloud.network.VirtualRouterProvider.Type;
+import com.cloud.network.addr.PublicIp;
+import com.cloud.network.dao.IPAddressDao;
+import com.cloud.network.dao.IPAddressVO;
+import com.cloud.network.dao.LoadBalancerDao;
+import com.cloud.network.dao.LoadBalancerVO;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
+import com.cloud.network.dao.PhysicalNetworkServiceProviderDao;
+import com.cloud.network.dao.VirtualRouterProviderDao;
+import com.cloud.network.lb.dao.ElasticLbVmMapDao;
+import com.cloud.network.router.VirtualRouter.RedundantState;
+import com.cloud.network.router.VirtualRouter.Role;
+import com.cloud.network.rules.LoadBalancer;
+import com.cloud.offering.NetworkOffering;
+import com.cloud.offerings.dao.NetworkOfferingDao;
+import com.cloud.service.ServiceOfferingVO;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.user.Account;
+import com.cloud.user.AccountService;
+import com.cloud.user.dao.AccountDao;
+import com.cloud.utils.db.DB;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.TransactionCallbackWithException;
+import com.cloud.utils.db.TransactionStatus;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.DomainRouterVO;
+import com.cloud.vm.NicProfile;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.VirtualMachineManager;
+import com.cloud.vm.VirtualMachineName;
+import com.cloud.vm.VirtualMachineProfile;
+import com.cloud.vm.VirtualMachineProfile.Param;
+import com.cloud.vm.dao.DomainRouterDao;
+
+public class LoadBalanceRuleHandler {
+
+ private static final Logger s_logger = Logger.getLogger(LoadBalanceRuleHandler.class);
+
+ @Inject
+ private IPAddressDao _ipAddressDao;
+ @Inject
+ private NetworkModel _networkModel;
+ @Inject
+ private NetworkOrchestrationService _networkMgr;
+ @Inject
+ private final LoadBalancerDao _loadBalancerDao = null;
+ @Inject
+ private LoadBalancingRulesManager _lbMgr;
+ @Inject
+ private final DomainRouterDao _routerDao = null;
+ @Inject
+ protected HostPodDao _podDao = null;
+ @Inject
+ protected ClusterDao _clusterDao;
+ @Inject
+ private final DataCenterDao _dcDao = null;
+ @Inject
+ private IpAddressManager _ipAddrMgr;
+ @Inject
+ protected NetworkDao _networkDao;
+ @Inject
+ protected NetworkOfferingDao _networkOfferingDao;
+ @Inject
+ private final VMTemplateDao _templateDao = null;
+ @Inject
+ private VirtualMachineManager _itMgr;
+ @Inject
+ private AccountService _accountService;
+ @Inject
+ private LoadBalancerDao _lbDao;
+ @Inject
+ private PodVlanMapDao _podVlanMapDao;
+ @Inject
+ private ElasticLbVmMapDao _elbVmMapDao;
+ @Inject
+ private AccountDao _accountDao;
+ @Inject
+ private PhysicalNetworkServiceProviderDao _physicalProviderDao;
+ @Inject
+ private VirtualRouterProviderDao _vrProviderDao;
+
+ static final private String ELB_VM_NAME_PREFIX = "l";
+
+ private final ServiceOfferingVO _elasticLbVmOffering;
+ private final String _instance;
+ private final Account _systemAcct;
+
+ public LoadBalanceRuleHandler(ServiceOfferingVO elasticLbVmOffering, String instance, Account systemAcct) {
+ this._elasticLbVmOffering = elasticLbVmOffering;
+ this._instance = instance;
+ this._systemAcct = systemAcct;
+ }
+
+ public void handleDeleteLoadBalancerRule(LoadBalancer lb, long userId, Account caller) {
+ List<LoadBalancerVO> remainingLbs = _loadBalancerDao.listByIpAddress(lb.getSourceIpAddressId());
+ if (remainingLbs.size() == 0) {
+ s_logger.debug("ELB mgr: releasing ip " + lb.getSourceIpAddressId() + " since no LB rules remain for this ip address");
+ releaseIp(lb.getSourceIpAddressId(), userId, caller);
+ }
+ }
+
+ public LoadBalancer handleCreateLoadBalancerRule(CreateLoadBalancerRuleCmd lb, Account account, long networkId) throws InsufficientAddressCapacityException,
+ NetworkRuleConflictException {
+ //this part of code is executed when the LB provider is Elastic Load Balancer vm
+ if (!_networkModel.isProviderSupportServiceInNetwork(lb.getNetworkId(), Service.Lb, Provider.ElasticLoadBalancerVm)) {
+ return null;
+ }
+
+ Long ipId = lb.getSourceIpAddressId();
+ if (ipId != null) {
+ return null;
+ }
+
+ account = _accountDao.acquireInLockTable(account.getId());
+ if (account == null) {
+ s_logger.warn("ELB: CreateLoadBalancer: Failed to acquire lock on account");
+ throw new CloudRuntimeException("Failed to acquire lock on account");
+ }
+ try {
+ return handleCreateLoadBalancerRuleWithLock(lb, account, networkId);
+ } finally {
+ if (account != null) {
+ _accountDao.releaseFromLockTable(account.getId());
+ }
+ }
+ }
+
+ private DomainRouterVO deployLoadBalancerVM(Long networkId, IPAddressVO ipAddr) {
+ NetworkVO network = _networkDao.findById(networkId);
+ DataCenter dc = _dcDao.findById(network.getDataCenterId());
+ Long podId = getPodIdForDirectIp(ipAddr);
+ Pod pod = podId == null ? null : _podDao.findById(podId);
+ Map<VirtualMachineProfile.Param, Object> params = new HashMap<VirtualMachineProfile.Param, Object>(1);
+ params.put(VirtualMachineProfile.Param.ReProgramGuestNetworks, true);
+ Account owner = _accountService.getActiveAccountByName("system", new Long(1));
+ DeployDestination dest = new DeployDestination(dc, pod, null, null);
+ s_logger.debug("About to deploy ELB vm ");
+
+ try {
+ DomainRouterVO elbVm = deployELBVm(network, dest, owner, params);
+ if (elbVm == null) {
+ throw new InvalidParameterValueException("Could not deploy or find existing ELB VM");
+ }
+ s_logger.debug("Deployed ELB vm = " + elbVm);
+
+ return elbVm;
+
+ } catch (Throwable t) {
+ s_logger.warn("Error while deploying ELB VM: ", t);
+ return null;
+ }
+
+ }
+
+ private DomainRouterVO deployELBVm(Network guestNetwork, DeployDestination dest, Account owner, Map<Param, Object> params) throws ConcurrentOperationException,
+ InsufficientCapacityException {
+ long dcId = dest.getDataCenter().getId();
+
+ // lock guest network
+ Long guestNetworkId = guestNetwork.getId();
+ guestNetwork = _networkDao.acquireInLockTable(guestNetworkId);
+
+ if (guestNetwork == null) {
+ throw new ConcurrentOperationException("Unable to acquire network lock: " + guestNetworkId);
+ }
+
+ try {
+
+ if (_networkModel.isNetworkSystem(guestNetwork) || guestNetwork.getGuestType() == Network.GuestType.Shared) {
+ owner = _accountService.getSystemAccount();
+ }
+
+ if (s_logger.isDebugEnabled()) {
+ s_logger.debug("Starting a ELB vm for network configurations: " + guestNetwork + " in " + dest);
+ }
+ assert guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup || guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: "
+ + guestNetwork;
+
+ DataCenterDeployment plan = null;
+ DomainRouterVO elbVm = null;
+
+ plan = new DataCenterDeployment(dcId, dest.getPod().getId(), null, null, null, null);
+
+ if (elbVm == null) {
+ long id = _routerDao.getNextInSequence(Long.class, "id");
+ if (s_logger.isDebugEnabled()) {
+ s_logger.debug("Creating the ELB vm " + id);
+ }
+
+ List<? extends NetworkOffering> offerings = _networkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemControlNetwork);
+ NetworkOffering controlOffering = offerings.get(0);
+ Network controlConfig = _networkMgr.setupNetwork(_systemAcct, controlOffering, plan, null, null, false).get(0);
+
+ LinkedHashMap<Network, NicProfile> networks = new LinkedHashMap<Network, NicProfile>(2);
+ NicProfile guestNic = new NicProfile();
+ guestNic.setDefaultNic(true);
+ networks.put(controlConfig, null);
+ networks.put(guestNetwork, guestNic);
+
+ VMTemplateVO template = _templateDao.findSystemVMTemplate(dcId);
+
+ String typeString = "ElasticLoadBalancerVm";
+ Long physicalNetworkId = _networkModel.getPhysicalNetworkId(guestNetwork);
+ PhysicalNetworkServiceProvider provider = _physicalProviderDao.findByServiceProvider(physicalNetworkId, typeString);
+ if (provider == null) {
+ throw new CloudRuntimeException("Cannot find service provider " + typeString + " in physical network " + physicalNetworkId);
+ }
+ VirtualRouterProvider vrProvider = _vrProviderDao.findByNspIdAndType(provider.getId(), Type.ElasticLoadBalancerVm);
+ if (vrProvider == null) {
+ throw new CloudRuntimeException("Cannot find virtual router provider " + typeString + " as service provider " + provider.getId());
+ }
+
+ elbVm = new DomainRouterVO(id, _elasticLbVmOffering.getId(), vrProvider.getId(), VirtualMachineName.getSystemVmName(id, _instance, ELB_VM_NAME_PREFIX),
+ template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), false, 0, false, RedundantState.UNKNOWN,
+ _elasticLbVmOffering.getOfferHA(), false, VirtualMachine.Type.ElasticLoadBalancerVm, null);
+ elbVm.setRole(Role.LB);
+ elbVm = _routerDao.persist(elbVm);
+ _itMgr.allocate(elbVm.getInstanceName(), template, _elasticLbVmOffering, networks, plan, null);
+ elbVm = _routerDao.findById(elbVm.getId());
+ //TODO: create usage stats
+ }
+
+ State state = elbVm.getState();
+ if (state != State.Running) {
+ elbVm = this.start(elbVm, params);
+ }
+
+ return elbVm;
+ } finally {
+ _networkDao.releaseFromLockTable(guestNetworkId);
+ }
+ }
+
+ private void releaseIp(long ipId, long userId, Account caller) {
+ s_logger.info("ELB: Release public IP for loadbalancing " + ipId);
+ IPAddressVO ipvo = _ipAddressDao.findById(ipId);
+ ipvo.setAssociatedWithNetworkId(null);
+ _ipAddressDao.update(ipvo.getId(), ipvo);
+ _ipAddrMgr.disassociatePublicIpAddress(ipId, userId, caller);
+ _ipAddressDao.unassignIpAddress(ipId);
+ }
+
+ protected Long getPodIdForDirectIp(IPAddressVO ipAddr) {
+ PodVlanMapVO podVlanMaps = _podVlanMapDao.listPodVlanMapsByVlan(ipAddr.getVlanId());
+ if (podVlanMaps == null) {
+ return null;
+ } else {
+ return podVlanMaps.getPodId();
+ }
+ }
+
+ private LoadBalancer handleCreateLoadBalancerRuleWithLock(CreateLoadBalancerRuleCmd lb, Account account, long networkId) throws InsufficientAddressCapacityException,
+ NetworkRuleConflictException {
+ Long ipId = null;
+ boolean newIp = false;
+ List<LoadBalancerVO> existingLbs = findExistingLoadBalancers(lb.getName(), lb.getSourceIpAddressId(), lb.getAccountId(), lb.getDomainId(), lb.getSourcePortStart());
+ if (existingLbs == null) {
+ existingLbs = findExistingLoadBalancers(lb.getName(), lb.getSourceIpAddressId(), lb.getAccountId(), lb.getDomainId(), null);
+ if (existingLbs == null) {
+ if (lb.getSourceIpAddressId() != null) {
+ throwExceptionIfSuppliedlLbNameIsNotAssociatedWithIpAddress(lb);
+ } else {
+ s_logger.debug("Could not find any existing frontend ips for this account for this LB rule, acquiring a new frontent IP for ELB");
+ PublicIp ip = allocDirectIp(account, networkId);
+ ipId = ip.getId();
+ newIp = true;
+ }
+ } else {
+ ipId = existingLbs.get(0).getSourceIpAddressId();
+ s_logger.debug("ELB: Found existing frontend ip for this account for this LB rule " + ipId);
+ }
+ } else {
+ s_logger.warn("ELB: Found existing load balancers matching requested new LB");
+ throw new NetworkRuleConflictException("ELB: Found existing load balancers matching requested new LB");
+ }
+
+ IPAddressVO ipAddr = _ipAddressDao.findById(ipId);
+
+ LoadBalancer result = null;
+ try {
+ lb.setSourceIpAddressId(ipId);
+
+ result = _lbMgr.createPublicLoadBalancer(lb.getXid(), lb.getName(), lb.getDescription(), lb.getSourcePortStart(), lb.getDefaultPortStart(), ipId.longValue(),
+ lb.getProtocol(), lb.getAlgorithm(), false, CallContext.current(), lb.getLbProtocol());
+ } catch (NetworkRuleConflictException e) {
+ s_logger.warn("Failed to create LB rule, not continuing with ELB deployment");
+ if (newIp) {
+ releaseIp(ipId, CallContext.current().getCallingUserId(), account);
+ }
+ throw e;
+ }
+
+ DomainRouterVO elbVm = null;
+
+ if (existingLbs == null) {
+ elbVm = findElbVmWithCapacity(ipAddr);
+ if (elbVm == null) {
+ elbVm = deployLoadBalancerVM(networkId, ipAddr);
+ if (elbVm == null) {
+ Network network = _networkModel.getNetwork(networkId);
+ s_logger.warn("Failed to deploy a new ELB vm for ip " + ipAddr + " in network " + network + "lb name=" + lb.getName());
+ if (newIp)
+ releaseIp(ipId, CallContext.current().getCallingUserId(), account);
+ }
+ }
+
+ } else {
+ ElasticLbVmMapVO elbVmMap = _elbVmMapDao.findOneByIp(ipId);
+ if (elbVmMap != null) {
+ elbVm = _routerDao.findById(elbVmMap.getElbVmId());
+ }
+ }
+
+ if (elbVm == null) {
+ s_logger.warn("No ELB VM can be found or deployed");
+ s_logger.warn("Deleting LB since we failed to deploy ELB VM");
+ _lbDao.remove(result.getId());
+ return null;
+ }
+
+ ElasticLbVmMapVO mapping = new ElasticLbVmMapVO(ipId, elbVm.getId(), result.getId());
+ _elbVmMapDao.persist(mapping);
+ return result;
+ }
+
+ private void throwExceptionIfSuppliedlLbNameIsNotAssociatedWithIpAddress(CreateLoadBalancerRuleCmd lb) {
+ List<LoadBalancerVO> existingLbs = findExistingLoadBalancers(lb.getName(), null, lb.getAccountId(), lb.getDomainId(), null);
+ if (existingLbs != null) {
+ throw new InvalidParameterValueException("Supplied LB name " + lb.getName() + " is not associated with IP " + lb.getSourceIpAddressId());
+ }
+ }
+
+ protected List<LoadBalancerVO> findExistingLoadBalancers(String lbName, Long ipId, Long accountId, Long domainId, Integer publicPort) {
+ SearchBuilder<LoadBalancerVO> sb = _lbDao.createSearchBuilder();
+ sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ);
+ sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ);
+ sb.and("publicPort", sb.entity().getSourcePortStart(), SearchCriteria.Op.EQ);
+ if (ipId != null) {
+ sb.and("sourceIpAddress", sb.entity().getSourceIpAddressId(), SearchCriteria.Op.EQ);
+ }
+ if (domainId != null) {
+ sb.and("domainId", sb.entity().getDomainId(), SearchCriteria.Op.EQ);
+ }
+ if (publicPort != null) {
+ sb.and("publicPort", sb.entity().getSourcePortStart(), SearchCriteria.Op.EQ);
+ }
+ SearchCriteria<LoadBalancerVO> sc = sb.create();
+ sc.setParameters("name", lbName);
+ sc.setParameters("accountId", accountId);
+ if (ipId != null) {
+ sc.setParameters("sourceIpAddress", ipId);
+ }
+ if (domainId != null) {
+ sc.setParameters("domainId", domainId);
+ }
+ if (publicPort != null) {
+ sc.setParameters("publicPort", publicPort);
+ }
+ List<LoadBalancerVO> lbs = _lbDao.search(sc, null);
+
+ return lbs == null || lbs.size() == 0 ? null : lbs;
+ }
+
+ @DB
+ private PublicIp allocDirectIp(final Account account, final long guestNetworkId) throws InsufficientAddressCapacityException {
+ return Transaction.execute(new TransactionCallbackWithException<PublicIp, InsufficientAddressCapacityException>() {
+ @Override
+ public PublicIp doInTransaction(TransactionStatus status) throws InsufficientAddressCapacityException {
+ Network frontEndNetwork = _networkModel.getNetwork(guestNetworkId);
+
+ PublicIp ip = _ipAddrMgr.assignPublicIpAddress(frontEndNetwork.getDataCenterId(), null, account, VlanType.DirectAttached, frontEndNetwork.getId(), null, true);
+ IPAddressVO ipvo = _ipAddressDao.findById(ip.getId());
+ ipvo.setAssociatedWithNetworkId(frontEndNetwork.getId());
+ _ipAddressDao.update(ipvo.getId(), ipvo);
+ s_logger.info("Acquired frontend IP for ELB " + ip);
+
+ return ip;
+ }
+ });
+ }
+
+ protected DomainRouterVO findElbVmWithCapacity(IPAddressVO ipAddr) {
+ List<DomainRouterVO> unusedElbVms = _elbVmMapDao.listUnusedElbVms();
+ if (unusedElbVms.size() > 0) {
+ List<DomainRouterVO> candidateVms = new ArrayList<DomainRouterVO>();
+ for (DomainRouterVO candidateVm : unusedElbVms) {
+ addCandidateVmIsPodIpMatches(candidateVm, getPodIdForDirectIp(ipAddr), candidateVms);
+ }
+ return candidateVms.size() == 0 ? null : candidateVms.get(new Random().nextInt(candidateVms.size()));
+ }
+ return null;
+ }
+
+ protected static void addCandidateVmIsPodIpMatches(DomainRouterVO candidateVm, Long podIdForDirectIp, List<DomainRouterVO> candidateVms) {
+ if (candidateVm.getPodIdToDeployIn().equals(podIdForDirectIp)) {
+ candidateVms.add(candidateVm);
+ }
+ }
+
+ protected DomainRouterVO start(DomainRouterVO elbVm, Map<Param, Object> params) throws ConcurrentOperationException {
+ s_logger.debug("Starting ELB VM " + elbVm);
+ _itMgr.start(elbVm.getUuid(), params);
+ return _routerDao.findById(elbVm.getId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8649fa00/plugins/network-elements/elastic-loadbalancer/test/com/cloud/network/lb/ElasticLoadBalancerManagerImplTest.java
----------------------------------------------------------------------
diff --git a/plugins/network-elements/elastic-loadbalancer/test/com/cloud/network/lb/ElasticLoadBalancerManagerImplTest.java b/plugins/network-elements/elastic-loadbalancer/test/com/cloud/network/lb/ElasticLoadBalancerManagerImplTest.java
new file mode 100644
index 0000000..8928fd9
--- /dev/null
+++ b/plugins/network-elements/elastic-loadbalancer/test/com/cloud/network/lb/ElasticLoadBalancerManagerImplTest.java
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.network.lb;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.internal.util.reflection.Whitebox;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.cloud.agent.api.check.CheckSshAnswer;
+import com.cloud.agent.manager.Commands;
+import com.cloud.network.lb.dao.ElasticLbVmMapDao;
+import com.cloud.vm.ReservationContext;
+import com.cloud.vm.VirtualMachineProfile;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ElasticLoadBalancerManagerImplTest {
+
+ @InjectMocks
+ private ElasticLoadBalancerManagerImpl elasticLoadBalancerManagerImpl;
+
+ @Test
+ public void testFinalizeStartWhenCmdsAnswerIsNull() throws Exception {
+ VirtualMachineProfile profileMock = mock(VirtualMachineProfile.class);
+ long hostId = 1L;
+ Commands cmds = mock(Commands.class);
+ when(cmds.getAnswer("checkSsh")).thenReturn(null);
+ ReservationContext context = mock(ReservationContext.class);
+
+ boolean expected = false;
+ boolean actual = elasticLoadBalancerManagerImpl.finalizeStart(profileMock, hostId, cmds, context);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testFinalizeStartWhenCmdsAnswerIsNotNullButAnswerResultIsFalse() throws Exception {
+ CheckSshAnswer answerMock = mock(CheckSshAnswer.class);
+ when(answerMock.getResult()).thenReturn(false);
+ VirtualMachineProfile profileMock = mock(VirtualMachineProfile.class);
+ long hostId = 1L;
+ Commands cmds = mock(Commands.class);
+ when(cmds.getAnswer("checkSsh")).thenReturn(answerMock);
+ ReservationContext context = mock(ReservationContext.class);
+
+ boolean expected = false;
+ boolean actual = elasticLoadBalancerManagerImpl.finalizeStart(profileMock, hostId, cmds, context);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testFinalizeStartWhenCmdsAnswerIsNotNullAndAnswerResultIsTrue() throws Exception {
+ CheckSshAnswer answerMock = mock(CheckSshAnswer.class);
+ when(answerMock.getResult()).thenReturn(true);
+ VirtualMachineProfile profileMock = mock(VirtualMachineProfile.class);
+ long hostId = 1L;
+ Commands cmds = mock(Commands.class);
+ when(cmds.getAnswer("checkSsh")).thenReturn(answerMock);
+ ReservationContext context = mock(ReservationContext.class);
+
+ boolean expected = true;
+ boolean actual = elasticLoadBalancerManagerImpl.finalizeStart(profileMock, hostId, cmds, context);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testGarbageCollectUnusedElbVmsWhenVariableUnusedElbVmsIsNull() throws Exception {
+ ElasticLbVmMapDao elasticLbVmMapDaoMock = mock(ElasticLbVmMapDao.class);
+ when(elasticLbVmMapDaoMock.listUnusedElbVms()).thenReturn(null);
+ Whitebox.setInternalState(elasticLoadBalancerManagerImpl, "_elbVmMapDao", elasticLbVmMapDaoMock);
+
+ try {
+ elasticLoadBalancerManagerImpl.garbageCollectUnusedElbVms();
+ } catch (NullPointerException e) {
+ fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8649fa00/plugins/network-elements/elastic-loadbalancer/test/com/cloud/network/lb/LoadBalanceRuleHandlerTest.java
----------------------------------------------------------------------
diff --git a/plugins/network-elements/elastic-loadbalancer/test/com/cloud/network/lb/LoadBalanceRuleHandlerTest.java b/plugins/network-elements/elastic-loadbalancer/test/com/cloud/network/lb/LoadBalanceRuleHandlerTest.java
new file mode 100644
index 0000000..17bae63
--- /dev/null
+++ b/plugins/network-elements/elastic-loadbalancer/test/com/cloud/network/lb/LoadBalanceRuleHandlerTest.java
@@ -0,0 +1,215 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.network.lb;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.internal.util.reflection.Whitebox;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.cloud.dc.PodVlanMapVO;
+import com.cloud.dc.dao.PodVlanMapDao;
+import com.cloud.network.dao.IPAddressVO;
+import com.cloud.network.lb.dao.ElasticLbVmMapDao;
+import com.cloud.vm.DomainRouterVO;
+import com.cloud.vm.VirtualMachineManager;
+import com.cloud.vm.VirtualMachineProfile.Param;
+import com.cloud.vm.dao.DomainRouterDao;
+
+@RunWith(MockitoJUnitRunner.class)
+public class LoadBalanceRuleHandlerTest {
+
+ @InjectMocks
+ private LoadBalanceRuleHandler loadBalanceRuleHandler;
+
+ @Mock
+ private VirtualMachineManager virtualMachineManagerMock;
+
+ @Mock
+ private DomainRouterDao domainRouterDaoMock;
+
+ @Mock
+ private ElasticLbVmMapDao elasticLbVmMapDao;
+
+ @Mock
+ private PodVlanMapDao podVlanMapDao;
+
+ @Before
+ public void setup() {
+ Whitebox.setInternalState(loadBalanceRuleHandler, "_itMgr", virtualMachineManagerMock);
+ Whitebox.setInternalState(loadBalanceRuleHandler, "_routerDao", domainRouterDaoMock);
+ Whitebox.setInternalState(loadBalanceRuleHandler, "_elbVmMapDao", elasticLbVmMapDao);
+ Whitebox.setInternalState(loadBalanceRuleHandler, "_podVlanMapDao", podVlanMapDao);
+ }
+
+ @Test
+ public void testAddCandidateVmIsPodIpMatchesWhenIdsHaveSameValue() throws Exception {
+ DomainRouterVO candidateVmMock = mock(DomainRouterVO.class);
+ when(candidateVmMock.getPodIdToDeployIn()).thenReturn(new Long(1));
+ Long podIdForDirectIp = new Long(1);
+ List<DomainRouterVO> candidateVms = new ArrayList<DomainRouterVO>();
+
+ LoadBalanceRuleHandler.addCandidateVmIsPodIpMatches(candidateVmMock, podIdForDirectIp, candidateVms);
+
+ assertEquals(1, candidateVms.size());
+ }
+
+ @Test
+ public void testAddCandidateVmIsPodIpMatchesWhenPodIdForDirectIpIsNull() throws Exception {
+ DomainRouterVO candidateVmMock = mock(DomainRouterVO.class);
+ when(candidateVmMock.getPodIdToDeployIn()).thenReturn(new Long(1));
+ Long podIdForDirectIp = null;
+ List<DomainRouterVO> candidateVms = new ArrayList<DomainRouterVO>();
+
+ LoadBalanceRuleHandler.addCandidateVmIsPodIpMatches(candidateVmMock, podIdForDirectIp, candidateVms);
+
+ assertEquals(0, candidateVms.size());
+ }
+
+ // PodIdToDeployIn should not be null according to column specification in DomainRouterVO
+ @Test(expected = NullPointerException.class)
+ public void testAddCandidateVmIsPodIpMatchesWhenPodIdToDeployInIsNull() throws Exception {
+ DomainRouterVO candidateVmMock = mock(DomainRouterVO.class);
+ when(candidateVmMock.getPodIdToDeployIn()).thenReturn(null);
+ Long podIdForDirectIp = new Long(1);
+ List<DomainRouterVO> candidateVms = new ArrayList<DomainRouterVO>();
+
+ LoadBalanceRuleHandler.addCandidateVmIsPodIpMatches(candidateVmMock, podIdForDirectIp, candidateVms);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testAddCandidateVmIsPodIpMatchesWhenCandidateVmsIsNull() throws Exception {
+ DomainRouterVO candidateVmMock = mock(DomainRouterVO.class);
+ when(candidateVmMock.getPodIdToDeployIn()).thenReturn(new Long(1));
+ Long podIdForDirectIp = new Long(1);
+ List<DomainRouterVO> candidateVms = null;
+
+ LoadBalanceRuleHandler.addCandidateVmIsPodIpMatches(candidateVmMock, podIdForDirectIp, candidateVms);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testStartWhenElbVmIsNull() throws Exception {
+ DomainRouterVO elbVm = null;
+ Map<Param, Object> params = new HashMap<Param, Object>();
+
+ loadBalanceRuleHandler.start(elbVm, params);
+ }
+
+ @Test
+ public void testStartWhenParamsIsNull() throws Exception {
+ DomainRouterVO elbVmMock = mock(DomainRouterVO.class);
+ String uuid = "uuid";
+ when(elbVmMock.getUuid()).thenReturn(uuid);
+ long id = 1L;
+ when(elbVmMock.getId()).thenReturn(id);
+ Map<Param, Object> params = null;
+
+ loadBalanceRuleHandler.start(elbVmMock, params);
+
+ verify(virtualMachineManagerMock, times(1)).start(uuid, params);
+ verify(domainRouterDaoMock, times(1)).findById(id);
+ }
+
+ @Test
+ public void testStartWhenParamsIsEmpty() throws Exception {
+ DomainRouterVO elbVmMock = mock(DomainRouterVO.class);
+ String uuid = "uuid";
+ when(elbVmMock.getUuid()).thenReturn(uuid);
+ long id = 1L;
+ when(elbVmMock.getId()).thenReturn(id);
+ Map<Param, Object> params = new HashMap<Param, Object>();
+
+ loadBalanceRuleHandler.start(elbVmMock, params);
+
+ verify(virtualMachineManagerMock, times(1)).start(uuid, params);
+ verify(domainRouterDaoMock, times(1)).findById(id);
+ }
+
+ @Test
+ public void testStart() throws Exception {
+ DomainRouterVO elbVmMock = mock(DomainRouterVO.class);
+ String uuid = "uuid";
+ when(elbVmMock.getUuid()).thenReturn(uuid);
+ long id = 1L;
+ when(elbVmMock.getId()).thenReturn(id);
+ Map<Param, Object> params = new HashMap<Param, Object>();
+ params.put(mock(Param.class), new Object());
+
+ loadBalanceRuleHandler.start(elbVmMock, params);
+
+ verify(virtualMachineManagerMock, times(1)).start(uuid, params);
+ verify(domainRouterDaoMock, times(1)).findById(id);
+ }
+
+ @Test
+ public void testFindElbVmWithCapacityWhenIpAddrIsNull() throws Exception {
+ IPAddressVO ipAddr = null;
+
+ DomainRouterVO actual = loadBalanceRuleHandler.findElbVmWithCapacity(ipAddr);
+
+ assertNull(actual);
+ }
+
+ @Test
+ public void testFindElbVmWithCapacityWhenThereAreNoUnusedElbVms() throws Exception {
+ IPAddressVO ipAddr = mock(IPAddressVO.class);
+ when(this.elasticLbVmMapDao.listUnusedElbVms()).thenReturn(new ArrayList<DomainRouterVO>(1));
+
+ DomainRouterVO actual = loadBalanceRuleHandler.findElbVmWithCapacity(ipAddr);
+
+ assertNull(actual);
+ }
+
+ @Test
+ public void testFindElbVmWithCapacityWhenThereAreUnusedElbVmsAndOneMatchesThePodId() throws Exception {
+ Long podId = 1L;
+ IPAddressVO ipAddrMock = mock(IPAddressVO.class);
+ when(ipAddrMock.getVlanId()).thenReturn(podId);
+ PodVlanMapVO podVlanMapVoMock = mock(PodVlanMapVO.class);
+ when(podVlanMapVoMock.getPodId()).thenReturn(podId);
+ when(podVlanMapDao.listPodVlanMapsByVlan(podId)).thenReturn(podVlanMapVoMock);
+ DomainRouterVO unusedElbVmThatMatchesPodId = mock(DomainRouterVO.class);
+ when(unusedElbVmThatMatchesPodId.getPodIdToDeployIn()).thenReturn(podId);
+ List<DomainRouterVO> unusedElbVms = Arrays.asList(new DomainRouterVO[] {unusedElbVmThatMatchesPodId, mock(DomainRouterVO.class)});
+ when(this.elasticLbVmMapDao.listUnusedElbVms()).thenReturn(unusedElbVms);
+
+ DomainRouterVO expected = unusedElbVmThatMatchesPodId;
+ DomainRouterVO actual = loadBalanceRuleHandler.findElbVmWithCapacity(ipAddrMock);
+
+ assertNotNull(actual);
+ assertEquals(expected, actual);
+ }
+
+}