You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by pe...@apache.org on 2024/02/06 17:41:04 UTC
(cloudstack) branch fr03-nsx-reorder-acl-rules created (now c839cd2645b)
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a change to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
at c839cd2645b tweak reorder logic
This branch includes the following new commits:
new 85bf6f239b4 Externalise a few timeouts & fix timeout for hostSupportsUefi in libvirt ready command wrapper (#8547)
new 228240fa49d NSX: Fix custom ACL check (#2)
new bce5ebb6806 Nsx vpc routed mode (#5)
new 2ba97ab5f4c Nsx: Support internal LB (#4)
new 4e12e7f62d5 NSX: Improve NSX resource cleanup process (#3)
new b0989f670d4 Fix unit test
new 54c8672eb0e NSX: Add SourceNAT service to the default Routed offering for VPC (#13)
new 44fa33c18db Fix VPC restart with cleanup (#12)
new 9d5c8740699 NSX: Fix ACL rule removal on replacement and fix rule order (#11)
new a1990e51f74 NSX: fix smoke test failure for ACLs (#9)
new 8721912b0ed Fix unit tests
new 6a55d1d7fc9 [WIP] NSX: Add support to re-order ACL rules (NSX FW rules)
new 788680855c0 fix reordering of acl rules on all networks that it is associated to
new 08ed58066d2 clean up and attempt test fix
new a428933b14a Fix tests
new 71a15f4e6b8 Remove unused import
new c839cd2645b tweak reorder logic
The 17 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(cloudstack) 06/17: Fix unit test
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit b0989f670d42e3b9dc48b83aac42f2d69a771b95
Author: nvazquez <ni...@gmail.com>
AuthorDate: Mon Jan 29 13:36:39 2024 -0300
Fix unit test
---
.../java/org/apache/cloudstack/service/NsxGuestNetworkGuruTest.java | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxGuestNetworkGuruTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxGuestNetworkGuruTest.java
index e6efc8f0503..66b9684203b 100644
--- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxGuestNetworkGuruTest.java
+++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxGuestNetworkGuruTest.java
@@ -37,6 +37,7 @@ import com.cloud.network.guru.GuestNetworkGuru;
import com.cloud.network.vpc.VpcVO;
import com.cloud.network.vpc.dao.VpcDao;
import com.cloud.offering.NetworkOffering;
+import com.cloud.offerings.NetworkOfferingVO;
import com.cloud.offerings.dao.NetworkOfferingDao;
import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
import com.cloud.user.Account;
@@ -307,6 +308,7 @@ public class NsxGuestNetworkGuruTest {
@Test
public void testCreateNsxSegmentForIsolatedNetwork() {
NetworkVO networkVO = Mockito.mock(NetworkVO.class);
+ NetworkOfferingVO offeringVO = Mockito.mock(NetworkOfferingVO.class);
DataCenter dataCenter = Mockito.mock(DataCenter.class);
when(networkVO.getAccountId()).thenReturn(1L);
@@ -315,6 +317,9 @@ public class NsxGuestNetworkGuruTest {
anyLong())).thenReturn(new NsxAnswer(new NsxCommand(), true, ""));
when(nsxControllerUtils.sendNsxCommand(any(CreateNsxSegmentCommand.class),
anyLong())).thenReturn(new NsxAnswer(new NsxCommand(), true, ""));
+ when(networkVO.getNetworkOfferingId()).thenReturn(1L);
+ when(networkOfferingDao.findById(1L)).thenReturn(offeringVO);
+ when(offeringVO.getNsxMode()).thenReturn(NetworkOffering.NsxMode.NATTED.name());
guru.createNsxSegment(networkVO, dataCenter);
verify(nsxControllerUtils, times(1)).sendNsxCommand(any(CreateNsxTier1GatewayCommand.class),
anyLong());
(cloudstack) 08/17: Fix VPC restart with cleanup (#12)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 44fa33c18db499f4dabefa4dc76b895cff188c5e
Author: Nicolas Vazquez <ni...@gmail.com>
AuthorDate: Fri Feb 2 11:58:48 2024 -0300
Fix VPC restart with cleanup (#12)
---
engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java | 2 ++
.../schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java | 15 ++++++++++++---
.../com/cloud/network/router/NicProfileHelperImpl.java | 10 +++++++++-
3 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java
index 68f57329d77..23c26ea0718 100644
--- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java
+++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java
@@ -44,6 +44,8 @@ public interface NicDao extends GenericDao<NicVO, Long> {
NicVO findByNetworkIdAndType(long networkId, VirtualMachine.Type vmType);
+ NicVO findByNetworkIdAndTypeIncludingRemoved(long networkId, VirtualMachine.Type vmType);
+
NicVO findByIp4AddressAndNetworkId(String ip4Address, long networkId);
NicVO findByNetworkIdAndMacAddress(long networkId, String mac);
diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java
index 59d2417b073..3eee1d4e749 100644
--- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java
@@ -176,12 +176,21 @@ public class NicDaoImpl extends GenericDaoBase<NicVO, Long> implements NicDao {
return findOneIncludingRemovedBy(sc);
}
- @Override
- public NicVO findByNetworkIdAndType(long networkId, VirtualMachine.Type vmType) {
+ private NicVO findByNetworkIdAndTypeInternal(long networkId, VirtualMachine.Type vmType, boolean includingRemoved) {
SearchCriteria<NicVO> sc = AllFieldsSearch.create();
sc.setParameters("network", networkId);
sc.setParameters("vmType", vmType);
- return findOneBy(sc);
+ return includingRemoved ? findOneIncludingRemovedBy(sc) : findOneBy(sc);
+ }
+
+ @Override
+ public NicVO findByNetworkIdAndType(long networkId, VirtualMachine.Type vmType) {
+ return findByNetworkIdAndTypeInternal(networkId, vmType, false);
+ }
+
+ @Override
+ public NicVO findByNetworkIdAndTypeIncludingRemoved(long networkId, VirtualMachine.Type vmType) {
+ return findByNetworkIdAndTypeInternal(networkId, vmType, true);
}
@Override
diff --git a/server/src/main/java/com/cloud/network/router/NicProfileHelperImpl.java b/server/src/main/java/com/cloud/network/router/NicProfileHelperImpl.java
index 15c8a2b4155..399019db3e2 100644
--- a/server/src/main/java/com/cloud/network/router/NicProfileHelperImpl.java
+++ b/server/src/main/java/com/cloud/network/router/NicProfileHelperImpl.java
@@ -21,6 +21,8 @@ import java.net.URI;
import javax.inject.Inject;
+import com.cloud.vm.NicVO;
+import com.cloud.vm.VirtualMachine;
import org.apache.cloudstack.network.router.deployment.RouterDeploymentDefinition;
import com.cloud.network.IpAddressManager;
@@ -118,7 +120,13 @@ public class NicProfileHelperImpl implements NicProfileHelper {
public NicProfile createGuestNicProfileForVpcRouter(final RouterDeploymentDefinition vpcRouterDeploymentDefinition, final Network guestNetwork) {
final NicProfile guestNic = new NicProfile();
- if (vpcRouterDeploymentDefinition.isRedundant()) {
+ if (BroadcastDomainType.NSX == guestNetwork.getBroadcastDomainType()) {
+ NicVO vrNic = _nicDao.findByNetworkIdAndTypeIncludingRemoved(guestNetwork.getId(), VirtualMachine.Type.DomainRouter);
+ if (vrNic != null) {
+ guestNic.setIPv4Address(vrNic.getIPv4Address());
+ guestNic.setIPv4Gateway(vrNic.getIPv4Gateway());
+ }
+ } else if (vpcRouterDeploymentDefinition.isRedundant()) {
guestNic.setIPv4Address(this.acquireGuestIpAddressForVrouterRedundant(guestNetwork));
} else {
guestNic.setIPv4Address(guestNetwork.getGateway());
(cloudstack) 16/17: Remove unused import
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 71a15f4e6b8b11f35d24bd1df8d08892ba46f281
Author: nvazquez <ni...@gmail.com>
AuthorDate: Tue Feb 6 13:03:33 2024 -0300
Remove unused import
---
.../src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java b/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java
index 938879d8f9f..729b3f802d7 100644
--- a/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java
+++ b/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java
@@ -24,7 +24,6 @@ import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
(cloudstack) 12/17: [WIP] NSX: Add support to re-order ACL rules (NSX FW rules)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 6a55d1d7fc9fef0413c662863df9bf4d8981b849
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Mon Feb 5 23:32:38 2024 -0500
[WIP] NSX: Add support to re-order ACL rules (NSX FW rules)
---
.../network/element/NetworkACLServiceProvider.java | 3 ++
.../com/cloud/network/vpc/NetworkACLManager.java | 2 +
.../cloud/network/element/BigSwitchBcfElement.java | 5 +++
.../management/ContrailVpcElementImpl.java | 5 +++
.../org/apache/cloudstack/service/NsxElement.java | 44 +++++++++++++++-------
.../network/element/VpcVirtualRouterElement.java | 5 +++
.../cloud/network/vpc/NetworkACLManagerImpl.java | 15 ++++++++
.../cloud/network/vpc/NetworkACLServiceImpl.java | 15 +++++++-
8 files changed, 79 insertions(+), 15 deletions(-)
diff --git a/api/src/main/java/com/cloud/network/element/NetworkACLServiceProvider.java b/api/src/main/java/com/cloud/network/element/NetworkACLServiceProvider.java
index 8c3243c99f4..92d3519d596 100644
--- a/api/src/main/java/com/cloud/network/element/NetworkACLServiceProvider.java
+++ b/api/src/main/java/com/cloud/network/element/NetworkACLServiceProvider.java
@@ -21,6 +21,7 @@ import java.util.List;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.network.Network;
import com.cloud.network.vpc.NetworkACLItem;
+import com.cloud.network.vpc.Vpc;
public interface NetworkACLServiceProvider extends NetworkElement {
@@ -32,4 +33,6 @@ public interface NetworkACLServiceProvider extends NetworkElement {
*/
boolean applyNetworkACLs(Network config, List<? extends NetworkACLItem> rules) throws ResourceUnavailableException;
+ boolean reorderAclRules(Vpc vpc, List<? extends NetworkACLItem> networkACLItems);
+
}
diff --git a/engine/components-api/src/main/java/com/cloud/network/vpc/NetworkACLManager.java b/engine/components-api/src/main/java/com/cloud/network/vpc/NetworkACLManager.java
index 4200ea8c601..9d1caa1c031 100644
--- a/engine/components-api/src/main/java/com/cloud/network/vpc/NetworkACLManager.java
+++ b/engine/components-api/src/main/java/com/cloud/network/vpc/NetworkACLManager.java
@@ -91,4 +91,6 @@ public interface NetworkACLManager {
boolean revokeACLItemsForPrivateGw(PrivateGateway gateway) throws ResourceUnavailableException;
boolean applyACLToPrivateGw(PrivateGateway gateway) throws ResourceUnavailableException;
+
+ boolean reorderAclRules(VpcVO vpc, List<? extends NetworkACLItem> networkACLItems);
}
diff --git a/plugins/network-elements/bigswitch/src/main/java/com/cloud/network/element/BigSwitchBcfElement.java b/plugins/network-elements/bigswitch/src/main/java/com/cloud/network/element/BigSwitchBcfElement.java
index 776f76fabca..c4b8de2ed95 100644
--- a/plugins/network-elements/bigswitch/src/main/java/com/cloud/network/element/BigSwitchBcfElement.java
+++ b/plugins/network-elements/bigswitch/src/main/java/com/cloud/network/element/BigSwitchBcfElement.java
@@ -700,6 +700,11 @@ NetworkACLServiceProvider, FirewallServiceProvider, ResourceStateAdapter {
return true;
}
+ @Override
+ public boolean reorderAclRules(Vpc vpc, List<? extends NetworkACLItem> networkACLItems) {
+ return true;
+ }
+
@Override
public boolean applyFWRules(Network network,
List<? extends FirewallRule> rules)
diff --git a/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/management/ContrailVpcElementImpl.java b/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/management/ContrailVpcElementImpl.java
index 689b252b2a7..58169f22d7b 100644
--- a/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/management/ContrailVpcElementImpl.java
+++ b/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/management/ContrailVpcElementImpl.java
@@ -185,6 +185,11 @@ public class ContrailVpcElementImpl extends ContrailElementImpl implements Netwo
return true;
}
+ @Override
+ public boolean reorderAclRules(Vpc vpc, List<? extends NetworkACLItem> networkACLItems) {
+ return true;
+ }
+
@Override
public boolean applyACLItemsToPrivateGw(PrivateGateway privateGateway,
List<? extends NetworkACLItem> rules)
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
index 05a7d141a86..99beb099bed 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
@@ -72,6 +72,7 @@ import com.cloud.network.rules.LoadBalancerContainer;
import com.cloud.network.rules.PortForwardingRule;
import com.cloud.network.rules.StaticNat;
import com.cloud.network.vpc.NetworkACLItem;
+import com.cloud.network.vpc.NetworkACLItemVO;
import com.cloud.network.vpc.PrivateGateway;
import com.cloud.network.vpc.StaticRouteProfile;
import com.cloud.network.vpc.Vpc;
@@ -97,7 +98,9 @@ import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachineProfile;
import com.cloud.vm.dao.VMInstanceDao;
import net.sf.ehcache.config.InvalidConfigurationException;
+import org.apache.cloudstack.NsxAnswer;
import org.apache.cloudstack.StartupNsxCommand;
+import org.apache.cloudstack.agent.api.DeleteNsxDistributedFirewallRulesCommand;
import org.apache.cloudstack.api.command.admin.internallb.ConfigureInternalLoadBalancerElementCmd;
import org.apache.cloudstack.api.command.admin.internallb.CreateInternalLoadBalancerElementCmd;
import org.apache.cloudstack.api.command.admin.internallb.ListInternalLoadBalancerElementsCmd;
@@ -712,18 +715,7 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
boolean success = true;
for (NetworkACLItem rule : rules) {
String privatePort = getPrivatePortRangeForACLRule(rule);
- NsxNetworkRule networkRule = new NsxNetworkRule.Builder()
- .setRuleId(rule.getId())
- .setSourceCidrList(Objects.nonNull(rule.getSourceCidrList()) ? transformCidrListValues(rule.getSourceCidrList()) : List.of("ANY"))
- .setAclAction(transformActionValue(rule.getAction()))
- .setTrafficType(rule.getTrafficType().toString())
- .setProtocol(rule.getProtocol().toUpperCase())
- .setPublicPort(String.valueOf(rule.getSourcePortStart()))
- .setPrivatePort(privatePort)
- .setIcmpCode(rule.getIcmpCode())
- .setIcmpType(rule.getIcmpType())
- .setService(Network.Service.NetworkACL)
- .build();
+ NsxNetworkRule networkRule = getNsxNetworkRuleForAcl(rule, privatePort);
if (Arrays.asList(NetworkACLItem.State.Active, NetworkACLItem.State.Add).contains(rule.getState())) {
success = success && nsxService.addFirewallRules(network, List.of(networkRule));
} else if (NetworkACLItem.State.Revoke == rule.getState()) {
@@ -740,9 +732,33 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
return success;
}
- private void reorderRules(List<? extends NetworkACLItem> rules) {
- rules.sort((Comparator) (r1, r2) -> ((NetworkACLItem) r2).getNumber() - ((NetworkACLItem) r1).getNumber());
+ @Override
+ public boolean reorderAclRules(Vpc vpc, List<? extends NetworkACLItem> networkACLItems) {
+ List<NsxNetworkRule> aclRulesList = new ArrayList<>();
+ for (NetworkACLItem rule : networkACLItems) {
+ String privatePort = getPrivatePortRangeForACLRule(rule);
+ aclRulesList.add(getNsxNetworkRuleForAcl(rule, privatePort));
+ }
+ DeleteNsxDistributedFirewallRulesCommand command = new DeleteNsxDistributedFirewallRulesCommand(vpc.getDomainId(),
+ vpc.getAccountId(), vpc.getZoneId(), vpc.getId(), network.getId(), netRules);
+ NsxAnswer result = nsxControllerUtils.sendNsxCommand(command, network.getDataCenterId());
+ return result.getResult();
+ return true;
+ }
+ private NsxNetworkRule getNsxNetworkRuleForAcl(NetworkACLItem rule, String privatePort) {
+ return new NsxNetworkRule.Builder()
+ .setRuleId(rule.getId())
+ .setSourceCidrList(Objects.nonNull(rule.getSourceCidrList()) ? transformCidrListValues(rule.getSourceCidrList()) : List.of("ANY"))
+ .setAclAction(transformActionValue(rule.getAction()))
+ .setTrafficType(rule.getTrafficType().toString())
+ .setProtocol(rule.getProtocol().toUpperCase())
+ .setPublicPort(String.valueOf(rule.getSourcePortStart()))
+ .setPrivatePort(privatePort)
+ .setIcmpCode(rule.getIcmpCode())
+ .setIcmpType(rule.getIcmpType())
+ .setService(Network.Service.NetworkACL)
+ .build();
}
@Override
public boolean applyFWRules(Network network, List<? extends FirewallRule> rules) throws ResourceUnavailableException {
diff --git a/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java
index d740f80bd25..6b79950949d 100644
--- a/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java
+++ b/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java
@@ -531,6 +531,11 @@ public class VpcVirtualRouterElement extends VirtualRouterElement implements Vpc
return result;
}
+ @Override
+ public boolean reorderAclRules(Vpc vpc, List<? extends NetworkACLItem> networkACLItems) {
+ return true;
+ }
+
@Override
protected Type getVirtualRouterProvider() {
return Type.VPCVirtualRouter;
diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java
index 2e54474772d..16b797fd94f 100644
--- a/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java
@@ -21,6 +21,7 @@ import java.util.List;
import javax.inject.Inject;
+import com.cloud.network.nsx.NsxProvider;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
@@ -370,6 +371,20 @@ public class NetworkACLManagerImpl extends ManagerBase implements NetworkACLMana
return applyACLToPrivateGw(gateway, rules);
}
+ @Override
+ public boolean reorderAclRules(VpcVO vpc, List<? extends NetworkACLItem> networkACLItems) {
+ List<NetworkACLServiceProvider> nsxElements = new ArrayList<>();
+ nsxElements.add((NetworkACLServiceProvider) _ntwkModel.getElementImplementingProvider(Network.Provider.Nsx.getName()));
+ try {
+ for (final NetworkACLServiceProvider provider : nsxElements) {
+ return provider.reorderAclRules(networkACLItems);
+ }
+ } catch (final Exception ex) {
+ s_logger.debug("Failed to reorder ACLs on NSX due to: " + ex.getLocalizedMessage());
+ }
+ return false;
+ }
+
private boolean applyACLToPrivateGw(final PrivateGateway gateway, final List<? extends NetworkACLItem> rules) throws ResourceUnavailableException {
List<VpcProvider> vpcElements = new ArrayList<VpcProvider>();
vpcElements.add((VpcProvider)_ntwkModel.getElementImplementingProvider(Network.Provider.VPCVirtualRouter.getName()));
diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
index 59e7d441e69..59b7eb62a2c 100644
--- a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
@@ -1000,7 +1000,15 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
if (nextRule == null) {
return moveRuleToTheBottom(ruleBeingMoved, allAclRules);
}
- return moveRuleBetweenAclRules(ruleBeingMoved, allAclRules, previousRule, nextRule);
+ NetworkACLItem networkACLItem = moveRuleBetweenAclRules(ruleBeingMoved, allAclRules, previousRule, nextRule);
+ VpcVO vpc = _vpcDao.findById(lockedAcl.getVpcId());
+ final DataCenter dc = _entityMgr.findById(DataCenter.class, vpc.getZoneId());
+ final NsxProviderVO nsxProvider = nsxProviderDao.findByZoneId(dc.getId());
+ if (Objects.nonNull(nsxProvider)) {
+ allAclRules = getAllAclRulesSortedByNumber(lockedAcl.getId());
+ _networkAclMgr.reorderAclRules(vpc, allAclRules);
+ }
+ return networkACLItem;
} finally {
_networkACLDao.releaseFromLockTable(ruleBeingMoved.getAclId());
}
@@ -1064,6 +1072,11 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
return allAclRules;
}
+ protected List<NetworkACLItem> getAclRulesSortedByNumber(long aclId) {
+ List<NetworkACLItemVO> allAclRules = getAllAclRulesSortedByNumber(aclId);
+ return new ArrayList<>(allAclRules);
+ }
+
/**
* Moves an ACL to the space between to other rules. If there is already enough room to accommodate the ACL rule being moved, we simply get the 'number' field from the previous ACL rule and add one, and then define this new value as the 'number' value for the ACL rule being moved.
* Otherwise, we will need to make room. This process is executed via {@link #updateAclRuleToNewPositionAndExecuteShiftIfNecessary(NetworkACLItem, int, List, int)}, which will create the space between ACL rules if necessary. This involves shifting ACL rules to accommodate the rule being moved.
(cloudstack) 02/17: NSX: Fix custom ACL check (#2)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 228240fa49dc6770a0f278dee9b34155f19be7d0
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Mon Jan 29 10:48:17 2024 -0500
NSX: Fix custom ACL check (#2)
* NSX: Fix custom ACL check
* NSX: Fix custom ACL check
---
.../com/cloud/network/vpc/NetworkACLServiceImpl.java | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
index 52d92f16117..d05ead6c143 100644
--- a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
@@ -20,6 +20,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
@@ -344,7 +345,7 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
if (isGlobalAcl(acl.getVpcId()) && !Account.Type.ADMIN.equals(caller.getType())) {
throw new PermissionDeniedException("Only Root Admins can create rules for a global ACL.");
}
- validateNsxConstraints(acl.getVpcId(), icmpType);
+ validateNsxConstraints(acl.getVpcId(), protocol, icmpType, icmpCode, sourcePortStart, sourcePortEnd);
validateAclRuleNumber(createNetworkACLCmd, acl);
NetworkACLItem.Action ruleAction = validateAndCreateNetworkAclRuleAction(action);
@@ -435,18 +436,27 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
}
}
- private void validateNsxConstraints(Long vpcId, Integer icpmType) {
+ private void validateNsxConstraints(long vpcId, String protocol, Integer icmpType,
+ Integer icmpCode, Integer sourcePortStart, Integer sourcePortEnd) {
VpcVO vpc = _vpcDao.findById(vpcId);
final DataCenter dc = _entityMgr.findById(DataCenter.class, vpc.getZoneId());
final NsxProviderVO nsxProvider = nsxProviderDao.findByZoneId(dc.getId());
if (Objects.isNull(nsxProvider)) {
return;
}
- if (icpmType == -1) {
+
+ if (NetUtils.ICMP_PROTO.equals(protocol.toLowerCase(Locale.ROOT)) && (icmpType == -1 || icmpCode == -1)) {
String errorMsg = "Passing -1 for ICMP type is not supported for NSX enabled zones";
s_logger.error(errorMsg);
throw new InvalidParameterValueException(errorMsg);
}
+
+ if (List.of(NetUtils.TCP_PROTO, NetUtils.UDP_PROTO).contains(protocol.toLowerCase(Locale.ROOT)) &&
+ (Objects.isNull(sourcePortStart) || Objects.isNull(sourcePortEnd))) {
+ String errorMsg = "Source start and end ports are required to be passed";
+ s_logger.error(errorMsg);
+ throw new InvalidParameterValueException(errorMsg);
+ }
}
/**
@@ -838,7 +848,8 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
NetworkACL acl = _networkAclMgr.getNetworkACL(networkACLItemVo.getAclId());
validateNetworkAcl(acl);
- validateNsxConstraints(acl.getVpcId(), networkACLItemVo.getIcmpType());
+ validateNsxConstraints(acl.getVpcId(), networkACLItemVo.getProtocol(), networkACLItemVo.getIcmpType(),
+ networkACLItemVo.getIcmpCode(), networkACLItemVo.getSourcePortStart(), networkACLItemVo.getSourcePortEnd());
Account account = CallContext.current().getCallingAccount();
validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account, "Only Root Admins can update global ACLs.");
(cloudstack) 03/17: Nsx vpc routed mode (#5)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit bce5ebb6806ec7745f23cbbb7ff846cbd30d3db9
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Mon Jan 29 10:50:57 2024 -0500
Nsx vpc routed mode (#5)
* NSX: Fix VPC routed mode
* NSX: VPC route mode
* remove unnecessary changes
---
.../java/org/apache/cloudstack/service/NsxGuestNetworkGuru.java | 5 ++++-
.../java/org/apache/cloudstack/service/NsxPublicNetworkGuru.java | 6 +++++-
server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java | 6 +++---
3 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxGuestNetworkGuru.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxGuestNetworkGuru.java
index 9c153283b80..1482f8fdc6f 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxGuestNetworkGuru.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxGuestNetworkGuru.java
@@ -319,7 +319,10 @@ public class NsxGuestNetworkGuru extends GuestNetworkGuru implements NetworkMigr
vpcName = vpc.getName();
} else {
LOGGER.debug(String.format("Creating a Tier 1 Gateway for the network %s before creating the NSX segment", networkVO.getName()));
- boolean isSourceNatSupported = networkOfferingServiceMapDao.areServicesSupportedByNetworkOffering(networkVO.getNetworkOfferingId(), Network.Service.SourceNat);
+ long networkOfferingId = networkVO.getNetworkOfferingId();
+ NetworkOfferingVO networkOfferingVO = networkOfferingDao.findById(networkOfferingId);
+ boolean isSourceNatSupported = !NetworkOffering.NsxMode.ROUTED.name().equals(networkOfferingVO.getNsxMode()) &&
+ networkOfferingServiceMapDao.areServicesSupportedByNetworkOffering(networkVO.getNetworkOfferingId(), Network.Service.SourceNat);
CreateNsxTier1GatewayCommand nsxTier1GatewayCommand = new CreateNsxTier1GatewayCommand(domain.getId(), account.getId(), zone.getId(), networkVO.getId(), networkVO.getName(), false, isSourceNatSupported);
NsxAnswer nsxAnswer = nsxControllerUtils.sendNsxCommand(nsxTier1GatewayCommand, zone.getId());
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxPublicNetworkGuru.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxPublicNetworkGuru.java
index 2e8127526b3..1b92a694317 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxPublicNetworkGuru.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxPublicNetworkGuru.java
@@ -29,6 +29,7 @@ import com.cloud.network.dao.IPAddressVO;
import com.cloud.network.dao.NetworkVO;
import com.cloud.network.guru.PublicNetworkGuru;
import com.cloud.network.vpc.VpcOffering;
+import com.cloud.network.vpc.VpcOfferingVO;
import com.cloud.network.vpc.VpcVO;
import com.cloud.network.vpc.dao.VpcDao;
import com.cloud.network.vpc.dao.VpcOfferingDao;
@@ -129,7 +130,10 @@ public class NsxPublicNetworkGuru extends PublicNetworkGuru {
long dataCenterId = vpc.getZoneId();
long resourceId = vpc.getId();
Network.Service[] services = { Network.Service.SourceNat };
- boolean sourceNatEnabled = vpcOfferingServiceMapDao.areServicesSupportedByVpcOffering(vpc.getVpcOfferingId(), services);
+ long networkOfferingId = vpc.getVpcOfferingId();
+ VpcOfferingVO vpcVO = vpcOfferingDao.findById(networkOfferingId);
+ boolean sourceNatEnabled = !NetworkOffering.NsxMode.ROUTED.name().equals(vpcVO.getNsxMode()) &&
+ vpcOfferingServiceMapDao.areServicesSupportedByVpcOffering(vpc.getVpcOfferingId(), services);
s_logger.info(String.format("Creating Tier 1 Gateway for VPC %s", vpc.getName()));
boolean result = nsxService.createVpcNetwork(dataCenterId, accountId, domainId, resourceId, vpc.getName(), sourceNatEnabled);
diff --git a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java
index 17277b2f7dd..5f22e8470ef 100644
--- a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java
@@ -403,7 +403,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
if (List.of(Service.UserData, Service.Dhcp, Service.Dns).contains(svc)) {
final Set<Provider> userDataProvider = Set.of(Provider.VPCVirtualRouter);
svcProviderMap.put(svc, userDataProvider);
- } else {
+ } else if (Service.NetworkACL.equals(svc)){
svcProviderMap.put(svc, defaultProviders);
}
}
@@ -3189,8 +3189,8 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
@Override
public boolean isSrcNatIpRequired(long vpcOfferingId) {
final Map<Network.Service, Set<Network.Provider>> vpcOffSvcProvidersMap = getVpcOffSvcProvidersMap(vpcOfferingId);
- return vpcOffSvcProvidersMap.get(Network.Service.SourceNat).contains(Network.Provider.VPCVirtualRouter) ||
- vpcOffSvcProvidersMap.get(Service.SourceNat).contains(Provider.Nsx);
+ return Objects.nonNull(vpcOffSvcProvidersMap.get(Network.Service.SourceNat)) && (vpcOffSvcProvidersMap.get(Network.Service.SourceNat).contains(Network.Provider.VPCVirtualRouter) ||
+ vpcOffSvcProvidersMap.get(Service.SourceNat).contains(Provider.Nsx));
}
/**
(cloudstack) 10/17: NSX: fix smoke test failure for ACLs (#9)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit a1990e51f74a2d72bdecf9763798930ff6e55cf4
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Fri Feb 2 10:21:06 2024 -0500
NSX: fix smoke test failure for ACLs (#9)
---
server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
index d05ead6c143..59e7d441e69 100644
--- a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
@@ -439,6 +439,9 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
private void validateNsxConstraints(long vpcId, String protocol, Integer icmpType,
Integer icmpCode, Integer sourcePortStart, Integer sourcePortEnd) {
VpcVO vpc = _vpcDao.findById(vpcId);
+ if (Objects.isNull(vpc)) {
+ return;
+ }
final DataCenter dc = _entityMgr.findById(DataCenter.class, vpc.getZoneId());
final NsxProviderVO nsxProvider = nsxProviderDao.findByZoneId(dc.getId());
if (Objects.isNull(nsxProvider)) {
(cloudstack) 09/17: NSX: Fix ACL rule removal on replacement and fix rule order (#11)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 9d5c8740699b000fa69d060d3ec04f8a7d50d230
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Fri Feb 2 10:16:08 2024 -0500
NSX: Fix ACL rule removal on replacement and fix rule order (#11)
---
.../java/org/apache/cloudstack/service/NsxElement.java | 16 +++++++++++-----
.../com/cloud/network/vpc/NetworkACLManagerImpl.java | 1 +
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
index dfb075aa312..05a7d141a86 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
@@ -112,6 +112,7 @@ import javax.inject.Inject;
import javax.naming.ConfigurationException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -706,8 +707,9 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
if (!canHandle(network, Network.Service.NetworkACL)) {
return false;
}
- List<NsxNetworkRule> nsxAddNetworkRules = new ArrayList<>();
+
List<NsxNetworkRule> nsxDelNetworkRules = new ArrayList<>();
+ boolean success = true;
for (NetworkACLItem rule : rules) {
String privatePort = getPrivatePortRangeForACLRule(rule);
NsxNetworkRule networkRule = new NsxNetworkRule.Builder()
@@ -723,22 +725,26 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
.setService(Network.Service.NetworkACL)
.build();
if (Arrays.asList(NetworkACLItem.State.Active, NetworkACLItem.State.Add).contains(rule.getState())) {
- nsxAddNetworkRules.add(networkRule);
+ success = success && nsxService.addFirewallRules(network, List.of(networkRule));
} else if (NetworkACLItem.State.Revoke == rule.getState()) {
nsxDelNetworkRules.add(networkRule);
}
}
- boolean success = true;
+
if (!nsxDelNetworkRules.isEmpty()) {
success = nsxService.deleteFirewallRules(network, nsxDelNetworkRules);
if (!success) {
LOGGER.warn("Not all firewall rules were successfully deleted");
}
}
- return success && nsxService.addFirewallRules(network, nsxAddNetworkRules);
+ return success;
}
- @Override
+ private void reorderRules(List<? extends NetworkACLItem> rules) {
+ rules.sort((Comparator) (r1, r2) -> ((NetworkACLItem) r2).getNumber() - ((NetworkACLItem) r1).getNumber());
+
+ }
+ @Override
public boolean applyFWRules(Network network, List<? extends FirewallRule> rules) throws ResourceUnavailableException {
if (!canHandle(network, Network.Service.Firewall)) {
diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java
index d95cf9ac7af..2e54474772d 100644
--- a/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java
@@ -206,6 +206,7 @@ public class NetworkACLManagerImpl extends ManagerBase implements NetworkACLMana
final List<NetworkACLItemVO> aclItems = _networkACLItemDao.listByACL(acl.getId());
if (aclItems == null || aclItems.isEmpty()) {
s_logger.debug("New network ACL is empty. Revoke existing rules before applying ACL");
+ } else {
if (!revokeACLItemsForNetwork(network.getId())) {
throw new CloudRuntimeException("Failed to replace network ACL. Error while removing existing ACL items for network: " + network.getId());
}
(cloudstack) 05/17: NSX: Improve NSX resource cleanup process (#3)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 4e12e7f62d510274d841d606e35120ee35eedfe4
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Mon Jan 29 10:53:33 2024 -0500
NSX: Improve NSX resource cleanup process (#3)
---
.../org/apache/cloudstack/service/NsxApiClient.java | 17 ++++++++++++++---
.../com/cloud/network/firewall/FirewallManagerImpl.java | 3 ++-
2 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java
index cb81a736c37..0ad9f94512a 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java
@@ -100,6 +100,7 @@ import static org.apache.cloudstack.utils.NsxControllerUtils.getServiceEntryName
import static org.apache.cloudstack.utils.NsxControllerUtils.getLoadBalancerName;
import static org.apache.cloudstack.utils.NsxControllerUtils.getLoadBalancerAlgorithm;
import static org.apache.cloudstack.utils.NsxControllerUtils.getActiveMonitorProfileName;
+import static org.apache.cloudstack.utils.NsxControllerUtils.getTier1GatewayName;
public class NsxApiClient {
@@ -429,6 +430,10 @@ public class NsxApiClient {
public void deleteSegment(long zoneId, long domainId, long accountId, Long vpcId, long networkId, String segmentName) {
try {
removeSegmentDistributedFirewallRules(segmentName);
+ if (Objects.isNull(vpcId)) {
+ String t1GatewayName = getTier1GatewayName(domainId, accountId, zoneId, networkId, false);
+ deleteLoadBalancer(getLoadBalancerName(t1GatewayName));
+ }
removeSegment(segmentName);
DhcpRelayConfigs dhcpRelayConfig = (DhcpRelayConfigs) nsxService.apply(DhcpRelayConfigs.class);
String dhcpRelayConfigId = NsxControllerUtils.getNsxDhcpRelayConfigId(zoneId, domainId, accountId, vpcId, networkId);
@@ -445,9 +450,15 @@ public class NsxApiClient {
protected void removeSegment(String segmentName) {
LOGGER.debug(String.format("Removing the segment with ID %s", segmentName));
Segments segmentService = (Segments) nsxService.apply(Segments.class);
- Segment segment = segmentService.get(segmentName);
- if (segment == null) {
- LOGGER.error(String.format("The segment with ID %s is not found, skipping removal", segmentName));
+ String errMsg = String.format("The segment with ID %s is not found, skipping removal", segmentName);
+ try {
+ Segment segment = segmentService.get(segmentName);
+ if (segment == null) {
+ LOGGER.warn(errMsg);
+ return;
+ }
+ } catch (Exception e) {
+ LOGGER.warn(errMsg);
return;
}
String siteId = getDefaultSiteId();
diff --git a/server/src/main/java/com/cloud/network/firewall/FirewallManagerImpl.java b/server/src/main/java/com/cloud/network/firewall/FirewallManagerImpl.java
index 4be933aaf11..0aae8f5beb6 100644
--- a/server/src/main/java/com/cloud/network/firewall/FirewallManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/firewall/FirewallManagerImpl.java
@@ -732,7 +732,8 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
return;
}
- if (NetUtils.ICMP_PROTO.equals(protocol.toLowerCase(Locale.ROOT)) && (rule.getIcmpType() == -1 || rule.getIcmpCode() == -1)) {
+ if (NetUtils.ICMP_PROTO.equals(protocol.toLowerCase(Locale.ROOT)) && (rule.getIcmpType() == -1 || rule.getIcmpCode() == -1)
+ && State.Add.equals(rule.getState())) {
String errorMsg = "Passing -1 for ICMP type is not supported for NSX enabled zones";
s_logger.error(errorMsg);
throw new InvalidParameterValueException(errorMsg);
(cloudstack) 07/17: NSX: Add SourceNAT service to the default Routed offering for VPC (#13)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 54c8672eb0ed86591e73d17df41eed89ba311fa1
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Fri Feb 2 07:21:42 2024 -0500
NSX: Add SourceNAT service to the default Routed offering for VPC (#13)
---
server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java
index 5f22e8470ef..02b00878f5d 100644
--- a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java
@@ -403,7 +403,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
if (List.of(Service.UserData, Service.Dhcp, Service.Dns).contains(svc)) {
final Set<Provider> userDataProvider = Set.of(Provider.VPCVirtualRouter);
svcProviderMap.put(svc, userDataProvider);
- } else if (Service.NetworkACL.equals(svc)){
+ } else if (List.of(Service.SourceNat, Service.NetworkACL).contains(svc)){
svcProviderMap.put(svc, defaultProviders);
}
}
(cloudstack) 14/17: clean up and attempt test fix
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 08ed58066d2cdb0892448e0d243fccc4f9bdba6c
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Tue Feb 6 10:41:36 2024 -0500
clean up and attempt test fix
---
.../cloud/network/vpc/NetworkACLServiceImpl.java | 8 +++---
.../network/vpc/NetworkACLServiceImplTest.java | 30 ++++++++++++----------
2 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
index 5b64fc24c03..0f7b99754c3 100644
--- a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
@@ -1002,6 +1002,9 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
}
NetworkACLItem networkACLItem = moveRuleBetweenAclRules(ruleBeingMoved, allAclRules, previousRule, nextRule);
VpcVO vpc = _vpcDao.findById(lockedAcl.getVpcId());
+ if (Objects.isNull(vpc)) {
+ return networkACLItem;
+ }
final DataCenter dc = _entityMgr.findById(DataCenter.class, vpc.getZoneId());
final NsxProviderVO nsxProvider = nsxProviderDao.findByZoneId(dc.getId());
List<NetworkVO> networks = _networkDao.listByAclId(lockedAcl.getId());
@@ -1073,11 +1076,6 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
return allAclRules;
}
- protected List<NetworkACLItem> getAclRulesSortedByNumber(long aclId) {
- List<NetworkACLItemVO> allAclRules = getAllAclRulesSortedByNumber(aclId);
- return new ArrayList<>(allAclRules);
- }
-
/**
* Moves an ACL to the space between to other rules. If there is already enough room to accommodate the ACL rule being moved, we simply get the 'number' field from the previous ACL rule and add one, and then define this new value as the 'number' value for the ACL rule being moved.
* Otherwise, we will need to make room. This process is executed via {@link #updateAclRuleToNewPositionAndExecuteShiftIfNecessary(NetworkACLItem, int, List, int)}, which will create the space between ACL rules if necessary. This involves shifting ACL rules to accommodate the rule being moved.
diff --git a/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java b/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java
index afd2b61960e..e9fcf32431c 100644
--- a/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java
+++ b/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java
@@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
@@ -138,8 +139,6 @@ public class NetworkACLServiceImplTest {
private VpcVO vpcVOMock;
@Mock
DataCenter dataCenterVO;
- @Mock
- NsxProviderVO nsxProviderVO;
@Mock
private Account accountMock;
@@ -148,23 +147,25 @@ public class NetworkACLServiceImplTest {
@Before
public void setup() {
- callContextMocked = Mockito.mockStatic(CallContext.class);
- Mockito.when(CallContext.current()).thenReturn(callContextMock);
- Mockito.doReturn(Mockito.mock(User.class)).when(callContextMock).getCallingUser();
- Mockito.doReturn(Mockito.mock(Account.class)).when(callContextMock).getCallingAccount();
+ try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
+ CallContext callContextMock = Mockito.mock(CallContext.class);
+ callContextMocked.when(CallContext::current).thenReturn(callContextMock);
+ Mockito.doReturn(Mockito.mock(User.class)).when(callContextMock).getCallingUser();
+ Mockito.doReturn(Mockito.mock(Account.class)).when(callContextMock).getCallingAccount();
- Mockito.when(networkAclDaoMock.findById(networkAclListId)).thenReturn(networkACLVOMock);
- Mockito.when(createNetworkAclCmdMock.getNetworkId()).thenReturn(1L);
- Mockito.when(createNetworkAclCmdMock.getProtocol()).thenReturn("tcp");
+ Mockito.when(networkAclDaoMock.findById(networkAclListId)).thenReturn(networkACLVOMock);
+ Mockito.when(createNetworkAclCmdMock.getNetworkId()).thenReturn(1L);
+ Mockito.when(createNetworkAclCmdMock.getProtocol()).thenReturn("tcp");
- Mockito.when(networkMock.getNetworkOfferingId()).thenReturn(networkOfferingMockId);
- Mockito.when(networkMock.getVpcId()).thenReturn(networkMockVpcMockId);
+ Mockito.when(networkMock.getNetworkOfferingId()).thenReturn(networkOfferingMockId);
+ Mockito.when(networkMock.getVpcId()).thenReturn(networkMockVpcMockId);
- Mockito.when(moveNetworkAclItemCmdMock.getUuidRuleBeingMoved()).thenReturn(uuidAclRuleBeingMoved);
+ Mockito.when(moveNetworkAclItemCmdMock.getUuidRuleBeingMoved()).thenReturn(uuidAclRuleBeingMoved);
- Mockito.when(aclRuleBeingMovedMock.getAclId()).thenReturn(networkAclMockId);
+ Mockito.when(aclRuleBeingMovedMock.getAclId()).thenReturn(networkAclMockId);
- Mockito.when(networkAclMock.getVpcId()).thenReturn(networkMockVpcMockId);
+ Mockito.when(networkAclMock.getVpcId()).thenReturn(networkMockVpcMockId);
+ }
}
@After
@@ -1013,6 +1014,7 @@ public class NetworkACLServiceImplTest {
Mockito.doReturn(previousAclRuleMock).when(networkAclServiceImpl).retrieveAndValidateAclRule(previousAclRuleUuid);
Mockito.doReturn(nextAclRuleMock).when(networkAclServiceImpl).retrieveAndValidateAclRule(nextAclRuleUuid);
+ when(nsxProviderDao.findByZoneId(anyLong())).then(null);
configureMoveMethodsToDoNothing();
(cloudstack) 01/17: Externalise a few timeouts & fix timeout for hostSupportsUefi in libvirt ready command wrapper (#8547)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 85bf6f239b4eac0f5e1478f82e7dfb1d8455196e
Author: Vishesh <vi...@gmail.com>
AuthorDate: Sat Jan 27 23:36:13 2024 +0530
Externalise a few timeouts & fix timeout for hostSupportsUefi in libvirt ready command wrapper (#8547)
This PR fixes bug introduced in #8502. Timeout for script execution was set to 60 ms instead of 60s which resulted in host not getting UEFI enabled. This is a blocker for 4.19 release.
We do this by introducing a new agent parameter `agent.script.timeout` (default - 60 seconds) to use as a timeout for the script checking host's UEFI status.
We also externalize the timeout for the ReadyCommand by introducing a new global setting `ready.command.wait` (default - 60 seconds).
For ModifyStoragePoolCommand, we don't externalize the timeout to avoid confusion for the user. Since, the required timeout can vary depending on the provider in use and we are only setting the wait for default host listener for now. Instead, we reuse the global `wait` setting by dividing it by `5` making the default value of 6 minutes (1800/5 = 360s) for ModifyStoragePoolCommand.
Note: the actual time, the MS waits is twice the wait set for a Command. Check reference code below.
https://github.com/apache/cloudstack/blob/19250403e645c76f60b17aa4aeb4dc915f5ca206/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java#L406-L442
---
agent/conf/agent.properties | 8 ++++++--
.../java/com/cloud/agent/properties/AgentProperties.java | 8 ++++++++
.../src/main/java/com/cloud/agent/AgentManager.java | 3 +++
.../main/java/com/cloud/agent/manager/AgentManagerImpl.java | 4 ++--
.../storage/datastore/provider/DefaultHostListener.java | 11 +++++++++--
.../kvm/resource/wrapper/LibvirtReadyCommandWrapper.java | 7 +++++--
utils/src/main/java/com/cloud/utils/script/Script.java | 13 +++++++++++++
7 files changed, 46 insertions(+), 8 deletions(-)
diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties
index b15534a0be6..6adde7435ab 100644
--- a/agent/conf/agent.properties
+++ b/agent/conf/agent.properties
@@ -5,9 +5,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -378,6 +378,10 @@ iscsi.session.cleanup.enabled=false
# If the time is exceeded shutdown will be forced.
#stop.script.timeout=120
+# Time (in seconds) to wait for scripts to complete.
+# This is currently used only while checking if the host supports UEFI.
+#agent.script.timeout=60
+
# Definition of VMs video model type.
#vm.video.hardware=
diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
index 0e979d370f4..322a0ed2444 100644
--- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
+++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
@@ -659,6 +659,14 @@ public class AgentProperties{
*/
public static final Property<Integer> STOP_SCRIPT_TIMEOUT = new Property<>("stop.script.timeout", 120);
+ /**
+ * Time (in seconds) to wait for scripts to complete.<br>
+ * This is currently used only while checking if the host supports UEFI.<br>
+ * Data type: Integer.<br>
+ * Default value: <code>60</code>
+ */
+ public static final Property<Integer> AGENT_SCRIPT_TIMEOUT = new Property<>("agent.script.timeout", 60);
+
/**
* Definition of VMs video model type.<br>
* Data type: String.<br>
diff --git a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java
index 6ba0c3b4fa0..2182dfc542d 100644
--- a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java
+++ b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java
@@ -47,6 +47,9 @@ public interface AgentManager {
"according to the hosts health check results",
true, ConfigKey.Scope.Cluster, null);
+ ConfigKey<Integer> ReadyCommandWait = new ConfigKey<Integer>("Advanced", Integer.class, "ready.command.wait",
+ "60", "Time in seconds to wait for Ready command to return", true);
+
public enum TapAgentsAction {
Add, Del, Contains,
}
diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java
index d8671ed29df..606a902dce7 100644
--- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java
+++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java
@@ -596,7 +596,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
final Long dcId = host.getDataCenterId();
final ReadyCommand ready = new ReadyCommand(dcId, host.getId(), NumbersUtil.enableHumanReadableSizes);
- ready.setWait(60);
+ ready.setWait(ReadyCommandWait.value());
final Answer answer = easySend(hostId, ready);
if (answer == null || !answer.getResult()) {
// this is tricky part for secondary storage
@@ -1838,7 +1838,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] { CheckTxnBeforeSending, Workers, Port, Wait, AlertWait, DirectAgentLoadSize,
- DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable };
+ DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait };
}
protected class SetHostParamsListener implements Listener {
diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java
index 90e8742c84d..a453f2d48c2 100644
--- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java
+++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java
@@ -57,6 +57,12 @@ import java.util.List;
public class DefaultHostListener implements HypervisorHostListener {
private static final Logger s_logger = Logger.getLogger(DefaultHostListener.class);
+
+ /**
+ * Wait time for modify storage pool command to complete. We should wait for 5 minutes for the command to complete.
+ * This should ideally be externalised as a global configuration parameter in the future (See #8506).
+ **/
+ private final int modifyStoragePoolCommandWait = 300; // 5 minutes
@Inject
AgentManager agentMgr;
@Inject
@@ -84,7 +90,6 @@ public class DefaultHostListener implements HypervisorHostListener {
@Inject
NetworkDao networkDao;
-
@Override
public boolean hostAdded(long hostId) {
return true;
@@ -121,7 +126,9 @@ public class DefaultHostListener implements HypervisorHostListener {
public boolean hostConnect(long hostId, long poolId) throws StorageConflictException {
StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool);
- cmd.setWait(60);
+ cmd.setWait(modifyStoragePoolCommandWait);
+ s_logger.debug(String.format("Sending modify storage pool command to agent: %d for storage pool: %d with timeout %d seconds",
+ hostId, poolId, cmd.getWait()));
final Answer answer = agentMgr.easySend(hostId, cmd);
if (answer == null) {
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java
index 4df74decdea..c0089c0e3a6 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java
@@ -25,6 +25,8 @@ import java.util.Map;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.ReadyAnswer;
import com.cloud.agent.api.ReadyCommand;
+import com.cloud.agent.properties.AgentProperties;
+import com.cloud.agent.properties.AgentPropertiesFileHandler;
import com.cloud.host.Host;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.resource.CommandWrapper;
@@ -51,11 +53,12 @@ public final class LibvirtReadyCommandWrapper extends CommandWrapper<ReadyComman
private boolean hostSupportsUefi(boolean isUbuntuHost) {
String cmd = "rpm -qa | grep -i ovmf";
+ int timeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.AGENT_SCRIPT_TIMEOUT) * 1000; // Get property value & convert to milliseconds
if (isUbuntuHost) {
cmd = "dpkg -l ovmf";
}
- s_logger.debug("Running command : " + cmd);
- int result = Script.runSimpleBashScriptForExitValue(cmd, 60, false);
+ s_logger.debug("Running command : [" + cmd + "] with timeout : " + timeout + " ms");
+ int result = Script.runSimpleBashScriptForExitValue(cmd, timeout, false);
s_logger.debug("Got result : " + result);
return result == 0;
}
diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java
index b42acb2e3f4..6af08a981d1 100644
--- a/utils/src/main/java/com/cloud/utils/script/Script.java
+++ b/utils/src/main/java/com/cloud/utils/script/Script.java
@@ -531,6 +531,19 @@ public class Script implements Callable<String> {
return runSimpleBashScriptForExitValue(command, 0, true);
}
+ /**
+ * Executes a bash script and returns the exit value of the script.
+ *
+ * @param command
+ * The bash command to be executed.
+ * @param timeout
+ * The maximum time (in milliseconds) that the script is allowed to run before it is forcibly terminated.
+ * @param avoidLogging
+ * If set to true, some logging is avoided.
+ *
+ * @return The exit value of the script. Returns -1 if the result is null or empty, or if it cannot be parsed into
+ * an integer which can happen in case of a timeout.
+ */
public static int runSimpleBashScriptForExitValue(String command, int timeout, boolean avoidLogging) {
Script s = new Script("/bin/bash", timeout);
(cloudstack) 17/17: tweak reorder logic
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit c839cd2645bbd6e1b05ce9db4fd18be857327e10
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Tue Feb 6 12:40:17 2024 -0500
tweak reorder logic
---
.../java/com/cloud/network/vpc/NetworkACLServiceImpl.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
index 0f7b99754c3..aa4e2cde563 100644
--- a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
@@ -993,14 +993,14 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
NetworkACLVO lockedAcl = _networkACLDao.acquireInLockTable(ruleBeingMoved.getAclId());
List<NetworkACLItemVO> allAclRules = getAllAclRulesSortedByNumber(lockedAcl.getId());
validateAclConsistency(moveNetworkAclItemCmd, lockedAcl, allAclRules);
-
+ NetworkACLItem networkACLItem = null;
if (previousRule == null) {
- return moveRuleToTheTop(ruleBeingMoved, allAclRules);
- }
- if (nextRule == null) {
- return moveRuleToTheBottom(ruleBeingMoved, allAclRules);
+ networkACLItem = moveRuleToTheTop(ruleBeingMoved, allAclRules);
+ } else if (nextRule == null) {
+ networkACLItem = moveRuleToTheBottom(ruleBeingMoved, allAclRules);
+ } else {
+ networkACLItem = moveRuleBetweenAclRules(ruleBeingMoved, allAclRules, previousRule, nextRule);
}
- NetworkACLItem networkACLItem = moveRuleBetweenAclRules(ruleBeingMoved, allAclRules, previousRule, nextRule);
VpcVO vpc = _vpcDao.findById(lockedAcl.getVpcId());
if (Objects.isNull(vpc)) {
return networkACLItem;
(cloudstack) 13/17: fix reordering of acl rules on all networks that it is associated to
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 788680855c06b15967b998765d6bf4650d955c28
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Tue Feb 6 08:51:23 2024 -0500
fix reordering of acl rules on all networks that it is associated to
---
.../network/element/NetworkACLServiceProvider.java | 2 +-
.../com/cloud/network/vpc/NetworkACLManager.java | 3 ++-
.../cloud/network/element/BigSwitchBcfElement.java | 2 +-
.../contrail/management/ContrailVpcElementImpl.java | 2 +-
.../org/apache/cloudstack/service/NsxElement.java | 21 +++++++++++----------
.../network/element/VpcVirtualRouterElement.java | 2 +-
.../cloud/network/vpc/NetworkACLManagerImpl.java | 5 ++---
.../cloud/network/vpc/NetworkACLServiceImpl.java | 5 +++--
8 files changed, 22 insertions(+), 20 deletions(-)
diff --git a/api/src/main/java/com/cloud/network/element/NetworkACLServiceProvider.java b/api/src/main/java/com/cloud/network/element/NetworkACLServiceProvider.java
index 92d3519d596..852a650cfcd 100644
--- a/api/src/main/java/com/cloud/network/element/NetworkACLServiceProvider.java
+++ b/api/src/main/java/com/cloud/network/element/NetworkACLServiceProvider.java
@@ -33,6 +33,6 @@ public interface NetworkACLServiceProvider extends NetworkElement {
*/
boolean applyNetworkACLs(Network config, List<? extends NetworkACLItem> rules) throws ResourceUnavailableException;
- boolean reorderAclRules(Vpc vpc, List<? extends NetworkACLItem> networkACLItems);
+ boolean reorderAclRules(Vpc vpc, List<? extends Network> networks, List<? extends NetworkACLItem> networkACLItems);
}
diff --git a/engine/components-api/src/main/java/com/cloud/network/vpc/NetworkACLManager.java b/engine/components-api/src/main/java/com/cloud/network/vpc/NetworkACLManager.java
index 9d1caa1c031..de69b894183 100644
--- a/engine/components-api/src/main/java/com/cloud/network/vpc/NetworkACLManager.java
+++ b/engine/components-api/src/main/java/com/cloud/network/vpc/NetworkACLManager.java
@@ -19,6 +19,7 @@ package com.cloud.network.vpc;
import java.util.List;
import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.network.Network;
import com.cloud.network.dao.NetworkVO;
public interface NetworkACLManager {
@@ -92,5 +93,5 @@ public interface NetworkACLManager {
boolean applyACLToPrivateGw(PrivateGateway gateway) throws ResourceUnavailableException;
- boolean reorderAclRules(VpcVO vpc, List<? extends NetworkACLItem> networkACLItems);
+ boolean reorderAclRules(VpcVO vpc, List<? extends Network> networks, List<? extends NetworkACLItem> networkACLItems);
}
diff --git a/plugins/network-elements/bigswitch/src/main/java/com/cloud/network/element/BigSwitchBcfElement.java b/plugins/network-elements/bigswitch/src/main/java/com/cloud/network/element/BigSwitchBcfElement.java
index c4b8de2ed95..9677d3b13ab 100644
--- a/plugins/network-elements/bigswitch/src/main/java/com/cloud/network/element/BigSwitchBcfElement.java
+++ b/plugins/network-elements/bigswitch/src/main/java/com/cloud/network/element/BigSwitchBcfElement.java
@@ -701,7 +701,7 @@ NetworkACLServiceProvider, FirewallServiceProvider, ResourceStateAdapter {
}
@Override
- public boolean reorderAclRules(Vpc vpc, List<? extends NetworkACLItem> networkACLItems) {
+ public boolean reorderAclRules(Vpc vpc, List<? extends Network> networks, List<? extends NetworkACLItem> networkACLItems) {
return true;
}
diff --git a/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/management/ContrailVpcElementImpl.java b/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/management/ContrailVpcElementImpl.java
index 58169f22d7b..cdbda514d88 100644
--- a/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/management/ContrailVpcElementImpl.java
+++ b/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/management/ContrailVpcElementImpl.java
@@ -186,7 +186,7 @@ public class ContrailVpcElementImpl extends ContrailElementImpl implements Netwo
}
@Override
- public boolean reorderAclRules(Vpc vpc, List<? extends NetworkACLItem> networkACLItems) {
+ public boolean reorderAclRules(Vpc vpc, List<? extends Network> networks, List<? extends NetworkACLItem> networkACLItems) {
return true;
}
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
index 99beb099bed..85f66cf4776 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
@@ -72,7 +72,6 @@ import com.cloud.network.rules.LoadBalancerContainer;
import com.cloud.network.rules.PortForwardingRule;
import com.cloud.network.rules.StaticNat;
import com.cloud.network.vpc.NetworkACLItem;
-import com.cloud.network.vpc.NetworkACLItemVO;
import com.cloud.network.vpc.PrivateGateway;
import com.cloud.network.vpc.StaticRouteProfile;
import com.cloud.network.vpc.Vpc;
@@ -98,9 +97,7 @@ import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachineProfile;
import com.cloud.vm.dao.VMInstanceDao;
import net.sf.ehcache.config.InvalidConfigurationException;
-import org.apache.cloudstack.NsxAnswer;
import org.apache.cloudstack.StartupNsxCommand;
-import org.apache.cloudstack.agent.api.DeleteNsxDistributedFirewallRulesCommand;
import org.apache.cloudstack.api.command.admin.internallb.ConfigureInternalLoadBalancerElementCmd;
import org.apache.cloudstack.api.command.admin.internallb.CreateInternalLoadBalancerElementCmd;
import org.apache.cloudstack.api.command.admin.internallb.ListInternalLoadBalancerElementsCmd;
@@ -115,7 +112,6 @@ import javax.inject.Inject;
import javax.naming.ConfigurationException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -733,17 +729,22 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
}
@Override
- public boolean reorderAclRules(Vpc vpc, List<? extends NetworkACLItem> networkACLItems) {
+ public boolean reorderAclRules(Vpc vpc, List<? extends Network> networks, List<? extends NetworkACLItem> networkACLItems) {
List<NsxNetworkRule> aclRulesList = new ArrayList<>();
for (NetworkACLItem rule : networkACLItems) {
String privatePort = getPrivatePortRangeForACLRule(rule);
aclRulesList.add(getNsxNetworkRuleForAcl(rule, privatePort));
}
- DeleteNsxDistributedFirewallRulesCommand command = new DeleteNsxDistributedFirewallRulesCommand(vpc.getDomainId(),
- vpc.getAccountId(), vpc.getZoneId(), vpc.getId(), network.getId(), netRules);
- NsxAnswer result = nsxControllerUtils.sendNsxCommand(command, network.getDataCenterId());
- return result.getResult();
- return true;
+ for (Network network: networks) {
+ nsxService.deleteFirewallRules(network, aclRulesList);
+ }
+ boolean success = true;
+ for (Network network : networks) {
+ for (NsxNetworkRule aclRule : aclRulesList) {
+ success = success && nsxService.addFirewallRules(network, List.of(aclRule));
+ }
+ }
+ return success;
}
private NsxNetworkRule getNsxNetworkRuleForAcl(NetworkACLItem rule, String privatePort) {
diff --git a/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java
index 6b79950949d..acd58400dc4 100644
--- a/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java
+++ b/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java
@@ -532,7 +532,7 @@ public class VpcVirtualRouterElement extends VirtualRouterElement implements Vpc
}
@Override
- public boolean reorderAclRules(Vpc vpc, List<? extends NetworkACLItem> networkACLItems) {
+ public boolean reorderAclRules(Vpc vpc, List<? extends Network> networks, List<? extends NetworkACLItem> networkACLItems) {
return true;
}
diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java
index 16b797fd94f..0ab4f149bdb 100644
--- a/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLManagerImpl.java
@@ -21,7 +21,6 @@ import java.util.List;
import javax.inject.Inject;
-import com.cloud.network.nsx.NsxProvider;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
@@ -372,12 +371,12 @@ public class NetworkACLManagerImpl extends ManagerBase implements NetworkACLMana
}
@Override
- public boolean reorderAclRules(VpcVO vpc, List<? extends NetworkACLItem> networkACLItems) {
+ public boolean reorderAclRules(VpcVO vpc, List<? extends Network> networks, List<? extends NetworkACLItem> networkACLItems) {
List<NetworkACLServiceProvider> nsxElements = new ArrayList<>();
nsxElements.add((NetworkACLServiceProvider) _ntwkModel.getElementImplementingProvider(Network.Provider.Nsx.getName()));
try {
for (final NetworkACLServiceProvider provider : nsxElements) {
- return provider.reorderAclRules(networkACLItems);
+ return provider.reorderAclRules(vpc, networks, networkACLItems);
}
} catch (final Exception ex) {
s_logger.debug("Failed to reorder ACLs on NSX due to: " + ex.getLocalizedMessage());
diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
index 59b7eb62a2c..5b64fc24c03 100644
--- a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
@@ -1004,9 +1004,10 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
VpcVO vpc = _vpcDao.findById(lockedAcl.getVpcId());
final DataCenter dc = _entityMgr.findById(DataCenter.class, vpc.getZoneId());
final NsxProviderVO nsxProvider = nsxProviderDao.findByZoneId(dc.getId());
- if (Objects.nonNull(nsxProvider)) {
+ List<NetworkVO> networks = _networkDao.listByAclId(lockedAcl.getId());
+ if (Objects.nonNull(nsxProvider) && !networks.isEmpty()) {
allAclRules = getAllAclRulesSortedByNumber(lockedAcl.getId());
- _networkAclMgr.reorderAclRules(vpc, allAclRules);
+ _networkAclMgr.reorderAclRules(vpc, networks, allAclRules);
}
return networkACLItem;
} finally {
(cloudstack) 04/17: Nsx: Support internal LB (#4)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 2ba97ab5f4c5860dd031a9ed8c4489d2a9bf1315
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Mon Jan 29 10:52:16 2024 -0500
Nsx: Support internal LB (#4)
* NSX: Support internal LB service in NSX
* add lb removal logic
* Fix UI issue hiding internal LB tab
* Refactor method name
---------
Co-authored-by: nvazquez <ni...@gmail.com>
---
.../com/cloud/network/VirtualRouterProvider.java | 2 +-
.../java/com/cloud/offering/NetworkOffering.java | 1 +
.../org/apache/cloudstack/api/ApiConstants.java | 2 +
.../admin/network/CreateNetworkOfferingCmd.java | 10 ++
.../api/response/NetworkOfferingResponse.java | 8 ++
.../apache/cloudstack/resource/NsxResource.java | 2 +
.../apache/cloudstack/service/NsxApiClient.java | 8 +-
.../org/apache/cloudstack/service/NsxElement.java | 102 ++++++++++++++++++++-
.../api/query/dao/NetworkOfferingJoinDaoImpl.java | 1 +
.../configuration/ConfigurationManagerImpl.java | 6 ++
.../com/cloud/server/ConfigurationServerImpl.java | 22 +++--
ui/public/locales/en.json | 1 +
ui/src/views/network/VpcTiersTab.vue | 20 +++-
ui/src/views/offering/AddNetworkOffering.vue | 9 ++
14 files changed, 176 insertions(+), 18 deletions(-)
diff --git a/api/src/main/java/com/cloud/network/VirtualRouterProvider.java b/api/src/main/java/com/cloud/network/VirtualRouterProvider.java
index aca526b1832..98410ca09f8 100644
--- a/api/src/main/java/com/cloud/network/VirtualRouterProvider.java
+++ b/api/src/main/java/com/cloud/network/VirtualRouterProvider.java
@@ -21,7 +21,7 @@ import org.apache.cloudstack.api.InternalIdentity;
public interface VirtualRouterProvider extends InternalIdentity, Identity {
public enum Type {
- VirtualRouter, ElasticLoadBalancerVm, VPCVirtualRouter, InternalLbVm, NetScalerVm
+ VirtualRouter, ElasticLoadBalancerVm, VPCVirtualRouter, InternalLbVm, NetScalerVm, Nsx
}
public Type getType();
diff --git a/api/src/main/java/com/cloud/offering/NetworkOffering.java b/api/src/main/java/com/cloud/offering/NetworkOffering.java
index 064b962fd2f..cf01fbf30e2 100644
--- a/api/src/main/java/com/cloud/offering/NetworkOffering.java
+++ b/api/src/main/java/com/cloud/offering/NetworkOffering.java
@@ -58,6 +58,7 @@ public interface NetworkOffering extends InfrastructureEntity, InternalIdentity,
public final static String DefaultSharedNetworkOfferingWithSGService = "DefaultSharedNetworkOfferingWithSGService";
public static final String DEFAULT_TUNGSTEN_SHARED_NETWORK_OFFERING_WITH_SGSERVICE = "DefaultTungstenSharedNetworkOfferingWithSGService";
public static final String DEFAULT_NAT_NSX_OFFERING_FOR_VPC = "DefaultNATNSXNetworkOfferingForVpc";
+ public static final String DEFAULT_NAT_NSX_OFFERING_FOR_VPC_WITH_ILB = "DefaultNATNSXNetworkOfferingForVpcWithInternalLB";
public static final String DEFAULT_ROUTED_NSX_OFFERING_FOR_VPC = "DefaultRoutedNSXNetworkOfferingForVpc";
public static final String DEFAULT_NAT_NSX_OFFERING = "DefaultNATNSXNetworkOffering";
public static final String DEFAULT_ROUTED_NSX_OFFERING = "DefaultRoutedNSXNetworkOffering";
diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
index 51523b1863e..775ab83e9c0 100644
--- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
@@ -825,6 +825,7 @@ public class ApiConstants {
public static final String FOR_VPC = "forvpc";
public static final String FOR_NSX = "fornsx";
public static final String NSX_SUPPORT_LB = "nsxsupportlb";
+ public static final String NSX_SUPPORTS_INTERNAL_LB = "nsxsupportsinternallb";
public static final String FOR_TUNGSTEN = "fortungsten";
public static final String SHRINK_OK = "shrinkok";
public static final String NICIRA_NVP_DEVICE_ID = "nvpdeviceid";
@@ -963,6 +964,7 @@ public class ApiConstants {
public static final String SUPPORTS_REGION_LEVEL_VPC = "supportsregionLevelvpc";
public static final String SUPPORTS_STRECHED_L2_SUBNET = "supportsstrechedl2subnet";
public static final String SUPPORTS_PUBLIC_ACCESS = "supportspublicaccess";
+ public static final String SUPPORTS_INTERNAL_LB = "supportsinternallb";
public static final String SUPPORTS_VM_AUTOSCALING = "supportsvmautoscaling";
public static final String REGION_LEVEL_VPC = "regionlevelvpc";
public static final String STRECHED_L2_SUBNET = "strechedl2subnet";
diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java
index e2ae2b42861..d4cf3660af1 100644
--- a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java
@@ -160,6 +160,12 @@ public class CreateNetworkOfferingCmd extends BaseCmd {
since = "4.20.0")
private Boolean nsxSupportsLbService;
+ @Parameter(name = ApiConstants.NSX_SUPPORTS_INTERNAL_LB,
+ type = CommandType.BOOLEAN,
+ description = "true if network offering for NSX network offering supports Internal Load balancer service.",
+ since = "4.20.0")
+ private Boolean nsxSupportsInternalLbService;
+
@Parameter(name = ApiConstants.FOR_TUNGSTEN,
type = CommandType.BOOLEAN,
description = "true if network offering is meant to be used for Tungsten-Fabric, false otherwise.")
@@ -306,6 +312,10 @@ public class CreateNetworkOfferingCmd extends BaseCmd {
return BooleanUtils.isTrue(nsxSupportsLbService);
}
+ public boolean getNsxSupportsInternalLbService() {
+ return BooleanUtils.isTrue(nsxSupportsInternalLbService);
+ }
+
public Boolean getForTungsten() {
return forTungsten;
}
diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NetworkOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NetworkOfferingResponse.java
index 2594488c5be..b73163a5d05 100644
--- a/api/src/main/java/org/apache/cloudstack/api/response/NetworkOfferingResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/NetworkOfferingResponse.java
@@ -135,6 +135,10 @@ public class NetworkOfferingResponse extends BaseResponseWithAnnotations {
@Param(description = "true if network offering supports public access for guest networks", since = "4.10.0")
private Boolean supportsPublicAccess;
+ @SerializedName(ApiConstants.SUPPORTS_INTERNAL_LB)
+ @Param(description = "true if network offering supports public access for guest networks", since = "4.20.0")
+ private Boolean supportsInternalLb;
+
@SerializedName(ApiConstants.DOMAIN_ID)
@Param(description = "the domain ID(s) this disk offering belongs to. Ignore this information as it is not currently applicable.")
private String domainId;
@@ -259,6 +263,10 @@ public class NetworkOfferingResponse extends BaseResponseWithAnnotations {
this.supportsPublicAccess = supportsPublicAccess;
}
+ public void setSupportsInternalLb(Boolean supportsInternalLb) {
+ this.supportsInternalLb = supportsInternalLb;
+ }
+
public String getDomainId() {
return domainId;
}
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java
index ec4de6b3ef7..bf16b056e6c 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java
@@ -308,7 +308,9 @@ public class NsxResource implements ServerResource {
private Answer executeRequest(DeleteNsxTier1GatewayCommand cmd) {
String tier1Id = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), cmd.getNetworkResourceId(), cmd.isResourceVpc());
+ String lbName = NsxControllerUtils.getLoadBalancerName(tier1Id);
try {
+ nsxApiClient.deleteLoadBalancer(lbName);
nsxApiClient.deleteTier1Gateway(tier1Id);
} catch (Exception e) {
return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage()));
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java
index cdcb83ec572..cb81a736c37 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java
@@ -707,8 +707,7 @@ public class NsxApiClient {
LBPoolListResult lbPoolListResult = lbPools.list(null, null, null, null, null, null);
if (CollectionUtils.isEmpty(lbVsListResult.getResults()) && CollectionUtils.isEmpty(lbPoolListResult.getResults())) {
String lbName = getLoadBalancerName(tier1GatewayName);
- LbServices lbServices = (LbServices) nsxService.apply(LbServices.class);
- lbServices.delete(lbName, true);
+ deleteLoadBalancer(lbName);
}
} catch (Error error) {
@@ -719,6 +718,11 @@ public class NsxApiClient {
}
}
+ public void deleteLoadBalancer(String lbName) {
+ LbServices lbServices = (LbServices) nsxService.apply(LbServices.class);
+ lbServices.delete(lbName, true);
+ }
+
private String getLbPoolPath(String lbPoolName) {
try {
LbPools lbPools = (LbPools) nsxService.apply(LbPools.class);
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
index 571e68496d6..dfb075aa312 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
@@ -44,6 +44,7 @@ import com.cloud.network.NetworkModel;
import com.cloud.network.Networks;
import com.cloud.network.PhysicalNetworkServiceProvider;
import com.cloud.network.PublicIpAddress;
+import com.cloud.network.VirtualRouterProvider;
import com.cloud.network.dao.IPAddressDao;
import com.cloud.network.dao.IPAddressVO;
import com.cloud.network.dao.LoadBalancerVMMapDao;
@@ -51,7 +52,9 @@ import com.cloud.network.dao.LoadBalancerVMMapVO;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkVO;
import com.cloud.network.dao.PhysicalNetworkDao;
+import com.cloud.network.dao.PhysicalNetworkServiceProviderDao;
import com.cloud.network.dao.PhysicalNetworkVO;
+import com.cloud.network.dao.VirtualRouterProviderDao;
import com.cloud.network.element.DhcpServiceProvider;
import com.cloud.network.element.DnsServiceProvider;
import com.cloud.network.element.FirewallServiceProvider;
@@ -60,9 +63,12 @@ import com.cloud.network.element.LoadBalancingServiceProvider;
import com.cloud.network.element.NetworkACLServiceProvider;
import com.cloud.network.element.PortForwardingServiceProvider;
import com.cloud.network.element.StaticNatServiceProvider;
+import com.cloud.network.element.VirtualRouterElement;
+import com.cloud.network.element.VirtualRouterProviderVO;
import com.cloud.network.element.VpcProvider;
import com.cloud.network.lb.LoadBalancingRule;
import com.cloud.network.rules.FirewallRule;
+import com.cloud.network.rules.LoadBalancerContainer;
import com.cloud.network.rules.PortForwardingRule;
import com.cloud.network.rules.StaticNat;
import com.cloud.network.vpc.NetworkACLItem;
@@ -82,6 +88,8 @@ import com.cloud.user.AccountManager;
import com.cloud.uservm.UserVm;
import com.cloud.utils.Pair;
import com.cloud.utils.component.AdapterBase;
+import com.cloud.utils.db.QueryBuilder;
+import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.NicProfile;
import com.cloud.vm.ReservationContext;
@@ -90,6 +98,10 @@ import com.cloud.vm.VirtualMachineProfile;
import com.cloud.vm.dao.VMInstanceDao;
import net.sf.ehcache.config.InvalidConfigurationException;
import org.apache.cloudstack.StartupNsxCommand;
+import org.apache.cloudstack.api.command.admin.internallb.ConfigureInternalLoadBalancerElementCmd;
+import org.apache.cloudstack.api.command.admin.internallb.CreateInternalLoadBalancerElementCmd;
+import org.apache.cloudstack.api.command.admin.internallb.ListInternalLoadBalancerElementsCmd;
+import org.apache.cloudstack.network.element.InternalLoadBalancerElementService;
import org.apache.cloudstack.resource.NsxLoadBalancerMember;
import org.apache.cloudstack.resource.NsxNetworkRule;
import org.apache.cloudstack.resource.NsxOpObject;
@@ -111,7 +123,7 @@ import java.util.function.LongFunction;
@Component
public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsServiceProvider, VpcProvider,
StaticNatServiceProvider, IpDeployer, PortForwardingServiceProvider, NetworkACLServiceProvider,
- LoadBalancingServiceProvider, FirewallServiceProvider, ResourceStateAdapter, Listener {
+ LoadBalancingServiceProvider, FirewallServiceProvider, InternalLoadBalancerElementService, ResourceStateAdapter, Listener {
@Inject
@@ -142,6 +154,10 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
VpcDao vpcDao;
@Inject
LoadBalancerVMMapDao lbVmMapDao;
+ @Inject
+ VirtualRouterProviderDao vrProviderDao;
+ @Inject
+ PhysicalNetworkServiceProviderDao pNtwkSvcProviderDao;
private static final Logger LOGGER = Logger.getLogger(NsxElement.class);
@@ -159,7 +175,16 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
capabilities.put(Network.Service.Dns, dnsCapabilities);
capabilities.put(Network.Service.StaticNat, null);
- capabilities.put(Network.Service.Lb, null);
+
+ // Set capabilities for LB service
+ Map<Network.Capability, String> lbCapabilities = new HashMap<Network.Capability, String>();
+ lbCapabilities.put(Network.Capability.SupportedLBAlgorithms, "roundrobin,leastconn");
+ lbCapabilities.put(Network.Capability.SupportedLBIsolation, "dedicated");
+ lbCapabilities.put(Network.Capability.SupportedProtocols, "tcp, udp");
+ lbCapabilities.put(Network.Capability.SupportedStickinessMethods, VirtualRouterElement.getHAProxyStickinessCapability());
+ lbCapabilities.put(Network.Capability.LbSchemes, String.join(",", LoadBalancerContainer.Scheme.Internal.name(), LoadBalancerContainer.Scheme.Public.name()));
+
+ capabilities.put(Network.Service.Lb, lbCapabilities);
capabilities.put(Network.Service.PortForwarding, null);
capabilities.put(Network.Service.NetworkACL, null);
@@ -633,7 +658,8 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
.setNetworkResourceName(nsxObject.getNetworkResourceName())
.setVpcResource(nsxObject.isVpcResource())
.setMemberList(lbMembers)
- .setPublicIp(publicIp.getAddress().addr())
+ .setPublicIp(LoadBalancerContainer.Scheme.Public == loadBalancingRule.getScheme() ?
+ publicIp.getAddress().addr() : loadBalancingRule.getSourceIp().addr())
.setPublicPort(String.valueOf(loadBalancingRule.getSourcePortStart()))
.setPrivatePort(String.valueOf(loadBalancingRule.getDefaultPortStart()))
.setRuleId(loadBalancingRule.getId())
@@ -778,4 +804,74 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
}
return list;
}
+
+ @Override
+ public VirtualRouterProvider configureInternalLoadBalancerElement(long id, boolean enable) {
+ VirtualRouterProviderVO element = vrProviderDao.findById(id);
+ if (element == null || element.getType() != VirtualRouterProvider.Type.Nsx) {
+ throw new InvalidParameterValueException("Can't find " + getName() + " " +
+ "element with network service provider id " + id + " to be used as a provider for " +
+ getName());
+ }
+
+ element.setEnabled(enable);
+ element = vrProviderDao.persist(element);
+
+ return element;
+ }
+
+ @Override
+ public VirtualRouterProvider addInternalLoadBalancerElement(long ntwkSvcProviderId) {
+ VirtualRouterProviderVO element = vrProviderDao.findByNspIdAndType(ntwkSvcProviderId, VirtualRouterProvider.Type.Nsx);
+ if (element != null) {
+ LOGGER.debug("There is already an " + getName() + " with service provider id " + ntwkSvcProviderId);
+ return null;
+ }
+
+ PhysicalNetworkServiceProvider provider = pNtwkSvcProviderDao.findById(ntwkSvcProviderId);
+ if (provider == null || !provider.getProviderName().equalsIgnoreCase(getName())) {
+ throw new InvalidParameterValueException("Invalid network service provider is specified");
+ }
+
+ element = new VirtualRouterProviderVO(ntwkSvcProviderId, VirtualRouterProvider.Type.Nsx);
+ element = vrProviderDao.persist(element);
+ return element;
+ }
+
+ @Override
+ public VirtualRouterProvider getInternalLoadBalancerElement(long id) {
+ VirtualRouterProvider provider = vrProviderDao.findById(id);
+ if (provider == null || provider.getType() != VirtualRouterProvider.Type.Nsx) {
+ throw new InvalidParameterValueException("Unable to find " + getName() + " by id");
+ }
+ return provider;
+ }
+
+ @Override
+ public List<? extends VirtualRouterProvider> searchForInternalLoadBalancerElements(Long id, Long ntwkSvsProviderId, Boolean enabled) {
+ QueryBuilder<VirtualRouterProviderVO> sc = QueryBuilder.create(VirtualRouterProviderVO.class);
+ if (id != null) {
+ sc.and(sc.entity().getId(), SearchCriteria.Op.EQ, id);
+ }
+ if (ntwkSvsProviderId != null) {
+ sc.and(sc.entity().getNspId(), SearchCriteria.Op.EQ, ntwkSvsProviderId);
+ }
+ if (enabled != null) {
+ sc.and(sc.entity().isEnabled(), SearchCriteria.Op.EQ, enabled);
+ }
+
+ //return only Internal LB elements
+ sc.and(sc.entity().getType(), SearchCriteria.Op.EQ, VirtualRouterProvider.Type.Nsx);
+
+ return sc.list();
+ }
+
+ @Override
+ public List<Class<?>> getCommands() {
+ List<Class<?>> cmdList = new ArrayList<Class<?>>();
+ cmdList.add(CreateInternalLoadBalancerElementCmd.class);
+ cmdList.add(ConfigureInternalLoadBalancerElementCmd.class);
+ cmdList.add(ListInternalLoadBalancerElementsCmd.class);
+ return cmdList;
+ }
}
diff --git a/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java
index d50f1610e69..e291f8d2af0 100644
--- a/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java
+++ b/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java
@@ -108,6 +108,7 @@ public class NetworkOfferingJoinDaoImpl extends GenericDaoBase<NetworkOfferingJo
networkOfferingResponse.setConcurrentConnections(offering.getConcurrentConnections());
networkOfferingResponse.setSupportsStrechedL2Subnet(offering.isSupportingStrechedL2());
networkOfferingResponse.setSupportsPublicAccess(offering.isSupportingPublicAccess());
+ networkOfferingResponse.setSupportsInternalLb(offering.isInternalLb());
networkOfferingResponse.setCreated(offering.getCreated());
if (offering.getGuestType() != null) {
networkOfferingResponse.setGuestIpType(offering.getGuestType().toString());
diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
index 09e7e96bd88..d54fa8b36f9 100644
--- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
+++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
@@ -5956,6 +5956,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
Boolean forNsx = cmd.isForNsx();
Boolean forTungsten = cmd.getForTungsten();
String nsxMode = cmd.getNsxMode();
+ boolean nsxSupportInternalLbSvc = cmd.getNsxSupportsInternalLbService();
Integer maxconn = null;
boolean enableKeepAlive = false;
String servicePackageuuid = cmd.getServicePackageId();
@@ -6274,6 +6275,11 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
final NetworkOfferingVO offering = createNetworkOffering(name, displayText, trafficType, tags, specifyVlan, availability, networkRate, serviceProviderMap, false, guestType, false,
serviceOfferingId, conserveMode, serviceCapabilityMap, specifyIpRanges, isPersistent, details, egressDefaultPolicy, maxconn, enableKeepAlive, forVpc, forTungsten, forNsx, nsxMode, domainIds, zoneIds, enable, internetProtocol);
+ if (Boolean.TRUE.equals(forNsx) && nsxSupportInternalLbSvc) {
+ offering.setInternalLb(true);
+ offering.setPublicLb(false);
+ _networkOfferingDao.update(offering.getId(), offering);
+ }
CallContext.current().setEventDetails(" Id: " + offering.getId() + " Name: " + name);
CallContext.current().putContextParameter(NetworkOffering.class, offering.getId());
return offering;
diff --git a/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java b/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java
index 4669f00070d..039a5edaf68 100644
--- a/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java
+++ b/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java
@@ -1203,35 +1203,41 @@ public class ConfigurationServerImpl extends ManagerBase implements Configuratio
// Offering #9 - network offering for NSX provider - NATTED mode
createAndPersistDefaultNsxOffering(NetworkOffering.DEFAULT_NAT_NSX_OFFERING, "Offering for NSX enabled networks - NAT mode",
- NetworkOffering.NsxMode.NATTED, false);
+ NetworkOffering.NsxMode.NATTED, false, true);
// Offering #10 - network offering for NSX provider - ROUTED mode
createAndPersistDefaultNsxOffering(NetworkOffering.DEFAULT_ROUTED_NSX_OFFERING, "Offering for NSX enabled networks - ROUTED mode",
- NetworkOffering.NsxMode.ROUTED, false);
+ NetworkOffering.NsxMode.ROUTED, false, true);
// Offering #11 - network offering for NSX provider for VPCs - NATTED mode
createAndPersistDefaultNsxOffering(NetworkOffering.DEFAULT_NAT_NSX_OFFERING_FOR_VPC, "Offering for NSX enabled networks on VPCs - NAT mode",
- NetworkOffering.NsxMode.NATTED, true);
+ NetworkOffering.NsxMode.NATTED, true, true);
// Offering #12 - network offering for NSX provider for VPCs - ROUTED mode
createAndPersistDefaultNsxOffering(NetworkOffering.DEFAULT_ROUTED_NSX_OFFERING_FOR_VPC, "Offering for NSX enabled networks on VPCs - ROUTED mode",
- NetworkOffering.NsxMode.ROUTED, true);
+ NetworkOffering.NsxMode.ROUTED, true, true);
+
+ // Offering #13 - network offering for NSX provider for VPCs with Internal LB - NATTED mode
+ createAndPersistDefaultNsxOffering(NetworkOffering.DEFAULT_NAT_NSX_OFFERING_FOR_VPC_WITH_ILB, "Offering for NSX enabled networks on VPCs with internal LB - NAT mode",
+ NetworkOffering.NsxMode.NATTED, true, false);
}
});
}
- private void createAndPersistDefaultNsxOffering(String name, String displayText, NetworkOffering.NsxMode nsxMode, boolean forVpc) {
+ private void createAndPersistDefaultNsxOffering(String name, String displayText, NetworkOffering.NsxMode nsxMode,
+ boolean forVpc, boolean publicLB) {
NetworkOfferingVO defaultNatNSXNetworkOffering =
new NetworkOfferingVO(name, displayText, TrafficType.Guest, false, false, null,
null, true, Availability.Optional, null, GuestType.Isolated, false,
false, false, false, false, forVpc);
- defaultNatNSXNetworkOffering.setPublicLb(true);
+ defaultNatNSXNetworkOffering.setPublicLb(publicLB);
+ defaultNatNSXNetworkOffering.setInternalLb(!publicLB);
defaultNatNSXNetworkOffering.setForNsx(true);
defaultNatNSXNetworkOffering.setNsxMode(nsxMode.name());
defaultNatNSXNetworkOffering.setState(NetworkOffering.State.Enabled);
defaultNatNSXNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(defaultNatNSXNetworkOffering);
- Map<Service, Provider> serviceProviderMap = getServicesAndProvidersForNSXNetwork(nsxMode, forVpc);
+ Map<Service, Provider> serviceProviderMap = getServicesAndProvidersForNSXNetwork(nsxMode, forVpc, publicLB);
for (Map.Entry<Network.Service, Network.Provider> service : serviceProviderMap.entrySet()) {
NetworkOfferingServiceMapVO offService =
new NetworkOfferingServiceMapVO(defaultNatNSXNetworkOffering.getId(), service.getKey(), service.getValue());
@@ -1240,7 +1246,7 @@ public class ConfigurationServerImpl extends ManagerBase implements Configuratio
}
}
- private Map<Service, Provider> getServicesAndProvidersForNSXNetwork(NetworkOffering.NsxMode nsxMode, boolean forVpc) {
+ private Map<Service, Provider> getServicesAndProvidersForNSXNetwork(NetworkOffering.NsxMode nsxMode, boolean forVpc, boolean publicLB) {
final Map<Network.Service, Network.Provider> serviceProviderMap = new HashMap<>();
Provider routerProvider = forVpc ? Provider.VPCVirtualRouter : Provider.VirtualRouter;
serviceProviderMap.put(Service.Dhcp, routerProvider);
diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json
index 6098bec5346..380c94a497f 100644
--- a/ui/public/locales/en.json
+++ b/ui/public/locales/en.json
@@ -1444,6 +1444,7 @@
"label.nsx.provider.edgecluster": "NSX provider edge cluster",
"label.nsx.provider.tier0gateway": "NSX provider tier-0 gateway",
"label.nsx.provider.transportzone": "NSX provider transport zone",
+"label.nsx.supports.internal.lb": "Enable NSX internal LB service",
"label.nsx.supports.lb": "Enable NSX LB service",
"label.num.cpu.cores": "# of CPU cores",
"label.number": "#Rule",
diff --git a/ui/src/views/network/VpcTiersTab.vue b/ui/src/views/network/VpcTiersTab.vue
index 8bb84392bd3..ddfd8fe0e8d 100644
--- a/ui/src/views/network/VpcTiersTab.vue
+++ b/ui/src/views/network/VpcTiersTab.vue
@@ -112,7 +112,7 @@
</template>
</a-pagination>
</a-collapse-panel>
- <a-collapse-panel :header="$t('label.internal.lb')" key="ilb" :style="customStyle" :collapsible="!showIlb(network) ? 'disabled' : null" >
+ <a-collapse-panel :header="$t('label.internal.lb')" key="ilb" :style="customStyle" :collapsible="displayCollapsible[network.id] ? null: 'disabled'" >
<a-button
type="dashed"
style="margin-bottom: 15px; width: 100%"
@@ -438,7 +438,8 @@ export default {
publicLBExists: false,
setMTU: false,
isNsxEnabled: false,
- isOfferingNatMode: false
+ isOfferingNatMode: false,
+ displayCollapsible: []
}
},
created () {
@@ -461,8 +462,8 @@ export default {
this.form = reactive({})
this.rules = reactive({})
},
- showIlb (network) {
- return network.service.filter(s => (s.name === 'Lb') && (s.capability.filter(c => c.name === 'LbSchemes' && c.value === 'Internal').length > 0)).length > 0 || false
+ showIlb (network, networkOffering) {
+ return ((networkOffering.supportsinternallb && network.service.filter(s => (s.name === 'Lb') && (s.capability.filter(c => c.name === 'LbSchemes' && c.value.split(',').includes('Internal')).length > 0)).length > 0)) || false
},
updateMtu () {
if (this.form.privatemtu > this.privateMtuMax) {
@@ -482,6 +483,7 @@ export default {
for (const network of this.networks) {
this.fetchLoadBalancers(network.id)
this.fetchVMs(network.id)
+ this.updateDisplayCollapsible(network.networkofferingid, network)
}
this.publicLBNetworkExists()
},
@@ -519,6 +521,16 @@ export default {
})
})
},
+ updateDisplayCollapsible (offeringId, network) {
+ api('listNetworkOfferings', {
+ id: offeringId
+ }).then(json => {
+ var networkOffering = json.listnetworkofferingsresponse.networkoffering[0]
+ this.displayCollapsible[network.id] = this.showIlb(network, networkOffering)
+ }).catch(e => {
+ this.$notifyError(e)
+ })
+ },
getVpcNetworkOffering () {
return new Promise((resolve, reject) => {
api('listVPCOfferings', {
diff --git a/ui/src/views/offering/AddNetworkOffering.vue b/ui/src/views/offering/AddNetworkOffering.vue
index a9d8bc529ed..500e3dc0a72 100644
--- a/ui/src/views/offering/AddNetworkOffering.vue
+++ b/ui/src/views/offering/AddNetworkOffering.vue
@@ -138,6 +138,14 @@
<a-switch v-model:checked="form.nsxsupportlb" @change="val => { handleNsxLbService(val) }" />
</a-form-item>
</a-col>
+ <a-col :md="12" :lg="12" v-if="form.nsxsupportlb && form.forvpc">
+ <a-form-item name="nsxsupportsinternallb" ref="nsxsupportsinternallb" v-if="guestType === 'isolated'">
+ <template #label>
+ <tooltip-label :title="$t('label.nsx.supports.internal.lb')" :tooltip="apiParams.nsxsupportsinternallb.description"/>
+ </template>
+ <a-switch v-model:checked="form.nsxsupportsinternallb"/>
+ </a-form-item>
+ </a-col>
</a-row>
<a-form-item name="nsxmode" ref="nsxmode" v-if="forNsx">
<template #label>
@@ -1025,6 +1033,7 @@ export default {
params.fornsx = true
params.nsxmode = values.nsxmode
params.nsxsupportlb = values.nsxsupportlb
+ params.nsxsupportsinternallb = values.nsxsupportsinternallb
}
if (values.guestiptype === 'shared' || values.guestiptype === 'isolated') {
if (values.conservemode !== true) {
(cloudstack) 15/17: Fix tests
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit a428933b14a5cad8b1e43ae09b140706681d6930
Author: nvazquez <ni...@gmail.com>
AuthorDate: Tue Feb 6 13:02:41 2024 -0300
Fix tests
---
.../network/vpc/NetworkACLServiceImplTest.java | 28 ++++++++++------------
1 file changed, 12 insertions(+), 16 deletions(-)
diff --git a/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java b/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java
index e9fcf32431c..938879d8f9f 100644
--- a/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java
+++ b/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java
@@ -35,7 +35,6 @@ import java.util.Map;
import com.cloud.dc.DataCenter;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.network.dao.NsxProviderDao;
-import com.cloud.network.element.NsxProviderVO;
import com.cloud.network.vpc.dao.VpcDao;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
import org.apache.cloudstack.api.ServerApiException;
@@ -147,25 +146,23 @@ public class NetworkACLServiceImplTest {
@Before
public void setup() {
- try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
- CallContext callContextMock = Mockito.mock(CallContext.class);
- callContextMocked.when(CallContext::current).thenReturn(callContextMock);
- Mockito.doReturn(Mockito.mock(User.class)).when(callContextMock).getCallingUser();
- Mockito.doReturn(Mockito.mock(Account.class)).when(callContextMock).getCallingAccount();
+ callContextMocked = Mockito.mockStatic(CallContext.class);
+ Mockito.when(CallContext.current()).thenReturn(callContextMock);
+ Mockito.doReturn(Mockito.mock(User.class)).when(callContextMock).getCallingUser();
+ Mockito.doReturn(Mockito.mock(Account.class)).when(callContextMock).getCallingAccount();
- Mockito.when(networkAclDaoMock.findById(networkAclListId)).thenReturn(networkACLVOMock);
- Mockito.when(createNetworkAclCmdMock.getNetworkId()).thenReturn(1L);
- Mockito.when(createNetworkAclCmdMock.getProtocol()).thenReturn("tcp");
+ Mockito.when(networkAclDaoMock.findById(networkAclListId)).thenReturn(networkACLVOMock);
+ Mockito.when(createNetworkAclCmdMock.getNetworkId()).thenReturn(1L);
+ Mockito.when(createNetworkAclCmdMock.getProtocol()).thenReturn("tcp");
- Mockito.when(networkMock.getNetworkOfferingId()).thenReturn(networkOfferingMockId);
- Mockito.when(networkMock.getVpcId()).thenReturn(networkMockVpcMockId);
+ Mockito.when(networkMock.getNetworkOfferingId()).thenReturn(networkOfferingMockId);
+ Mockito.when(networkMock.getVpcId()).thenReturn(networkMockVpcMockId);
- Mockito.when(moveNetworkAclItemCmdMock.getUuidRuleBeingMoved()).thenReturn(uuidAclRuleBeingMoved);
+ Mockito.when(moveNetworkAclItemCmdMock.getUuidRuleBeingMoved()).thenReturn(uuidAclRuleBeingMoved);
- Mockito.when(aclRuleBeingMovedMock.getAclId()).thenReturn(networkAclMockId);
+ Mockito.when(aclRuleBeingMovedMock.getAclId()).thenReturn(networkAclMockId);
- Mockito.when(networkAclMock.getVpcId()).thenReturn(networkMockVpcMockId);
- }
+ Mockito.when(networkAclMock.getVpcId()).thenReturn(networkMockVpcMockId);
}
@After
@@ -1014,7 +1011,6 @@ public class NetworkACLServiceImplTest {
Mockito.doReturn(previousAclRuleMock).when(networkAclServiceImpl).retrieveAndValidateAclRule(previousAclRuleUuid);
Mockito.doReturn(nextAclRuleMock).when(networkAclServiceImpl).retrieveAndValidateAclRule(nextAclRuleUuid);
- when(nsxProviderDao.findByZoneId(anyLong())).then(null);
configureMoveMethodsToDoNothing();
(cloudstack) 11/17: Fix unit tests
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch fr03-nsx-reorder-acl-rules
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 8721912b0ed4cd77ef8878b18a6b37435e01b4fa
Author: nvazquez <ni...@gmail.com>
AuthorDate: Fri Feb 2 12:49:07 2024 -0300
Fix unit tests
---
.../nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java
index 39fb2e06120..318542c40db 100644
--- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java
+++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java
@@ -440,7 +440,6 @@ public class NsxElementTest {
Network.Service service = new Network.Service("service1", new Network.Capability("capability"));
when(nsxElement.canHandle(networkVO, service)).thenReturn(true);
- when(nsxService.addFirewallRules(any(Network.class), any(List.class))).thenReturn(true);
assertTrue(nsxElement.applyNetworkACLs(networkVO, List.of(rule)));
}
@@ -457,7 +456,6 @@ public class NsxElementTest {
when(nsxElement.canHandle(networkVO, service)).thenReturn(true);
when(nsxService.deleteFirewallRules(any(Network.class), any(List.class))).thenReturn(true);
- when(nsxService.addFirewallRules(any(Network.class), any(List.class))).thenReturn(true);
assertTrue(nsxElement.applyNetworkACLs(networkVO, List.of(rule)));
}