You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by pe...@apache.org on 2024/01/17 18:00:33 UTC
(cloudstack) branch nsx-k8s-cleanup created (now d67a89aed23)
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a change to branch nsx-k8s-cleanup
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
at d67a89aed23 NSX: Cleanup NSX resources during k8s cluster cleanup
This branch includes the following new commits:
new d67a89aed23 NSX: Cleanup NSX resources during k8s cluster cleanup
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(cloudstack) 01/01: NSX: Cleanup NSX resources during k8s cluster cleanup
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch nsx-k8s-cleanup
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit d67a89aed23c9855d690a633a8d0a945644578c9
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Wed Jan 17 13:00:22 2024 -0500
NSX: Cleanup NSX resources during k8s cluster cleanup
---
.../cluster/KubernetesClusterHelper.java | 1 +
.../cluster/KubernetesClusterHelperImpl.java | 13 ++++++++++
.../cluster/dao/KubernetesClusterVmMapDao.java | 2 ++
.../cluster/dao/KubernetesClusterVmMapDaoImpl.java | 12 ++++++++++
.../org/apache/cloudstack/service/NsxElement.java | 21 +++++++++-------
.../main/java/com/cloud/vm/UserVmManagerImpl.java | 28 ++++++++++++++++++----
.../core/spring-server-core-managers-context.xml | 5 ++--
7 files changed, 66 insertions(+), 16 deletions(-)
diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
index e445e50f82c..e160227749d 100644
--- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
+++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
@@ -22,4 +22,5 @@ import org.apache.cloudstack.acl.ControlledEntity;
public interface KubernetesClusterHelper extends Adapter {
ControlledEntity findByUuid(String uuid);
+ ControlledEntity findByVmId(long vmId);
}
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
index 0ef916ab959..60bd81c7c5a 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
@@ -17,6 +17,7 @@
package com.cloud.kubernetes.cluster;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
+import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.utils.component.AdapterBase;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.framework.config.ConfigKey;
@@ -24,18 +25,30 @@ import org.apache.cloudstack.framework.config.Configurable;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
+import java.util.Objects;
@Component
public class KubernetesClusterHelperImpl extends AdapterBase implements KubernetesClusterHelper, Configurable {
@Inject
private KubernetesClusterDao kubernetesClusterDao;
+ @Inject
+ private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
@Override
public ControlledEntity findByUuid(String uuid) {
return kubernetesClusterDao.findByUuid(uuid);
}
+ @Override
+ public ControlledEntity findByVmId(long vmId) {
+ KubernetesClusterVmMapVO clusterVmMapVO = kubernetesClusterVmMapDao.getClusterMapFromVmId(vmId);
+ if (Objects.isNull(clusterVmMapVO)) {
+ return null;
+ }
+ return kubernetesClusterDao.findById(clusterVmMapVO.getClusterId());
+ }
+
@Override
public String getConfigComponentName() {
return KubernetesClusterHelper.class.getSimpleName();
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
index 688a611ac99..45c0b79485c 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
@@ -23,6 +23,8 @@ import java.util.List;
public interface KubernetesClusterVmMapDao extends GenericDao<KubernetesClusterVmMapVO, Long> {
public List<KubernetesClusterVmMapVO> listByClusterId(long clusterId);
+
+ public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId);
public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);
int removeByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
index b9f2ec917b2..0d90a4cdaca 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
@@ -31,12 +31,17 @@ import com.cloud.utils.db.SearchCriteria;
public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase<KubernetesClusterVmMapVO, Long> implements KubernetesClusterVmMapDao {
private final SearchBuilder<KubernetesClusterVmMapVO> clusterIdSearch;
+ private final SearchBuilder<KubernetesClusterVmMapVO> vmIdSearch;
public KubernetesClusterVmMapDaoImpl() {
clusterIdSearch = createSearchBuilder();
clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
clusterIdSearch.and("vmIdsIN", clusterIdSearch.entity().getVmId(), SearchCriteria.Op.IN);
clusterIdSearch.done();
+
+ vmIdSearch = createSearchBuilder();
+ vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ);
+ vmIdSearch.done();
}
@Override
@@ -47,6 +52,13 @@ public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase<KubernetesClus
return listBy(sc, filter);
}
+ @Override
+ public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId) {
+ SearchCriteria<KubernetesClusterVmMapVO> sc = vmIdSearch.create();
+ sc.setParameters("vmId", vmId);
+ return findOneBy(sc);
+ }
+
@Override
public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds) {
SearchCriteria<KubernetesClusterVmMapVO> sc = clusterIdSearch.create();
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
index 0986c367cf9..571e68496d6 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
@@ -505,10 +505,12 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
if (!canHandle(network, Network.Service.PortForwarding)) {
return false;
}
+ boolean result = true;
for (PortForwardingRule rule : rules) {
IPAddressVO publicIp = ApiDBUtils.findIpAddressById(rule.getSourceIpAddressId());
UserVm vm = ApiDBUtils.findUserVmById(rule.getVirtualMachineId());
- if (vm == null || networkModel.getNicInNetwork(vm.getId(), network.getId()) == null) {
+ if ((vm == null && (rule.getState() != FirewallRule.State.Revoke)) ||
+ (vm != null && networkModel.getNicInNetwork(vm.getId(), network.getId()) == null)) {
continue;
}
NsxOpObject nsxObject = getNsxOpObject(network);
@@ -523,8 +525,8 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
.setNetworkResourceId(nsxObject.getNetworkResourceId())
.setNetworkResourceName(nsxObject.getNetworkResourceName())
.setVpcResource(nsxObject.isVpcResource())
- .setVmId(vm.getId())
- .setVmIp(vm.getPrivateIpAddress())
+ .setVmId(Objects.nonNull(vm) ? vm.getId() : 0)
+ .setVmIp(Objects.nonNull(vm) ? vm.getPrivateIpAddress() : null)
.setPublicIp(publicIp.getAddress().addr())
.setPrivatePort(privatePort)
.setPublicPort(publicPort)
@@ -532,12 +534,12 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
.setProtocol(rule.getProtocol().toUpperCase(Locale.ROOT))
.build();
if (rule.getState() == FirewallRule.State.Add) {
- return nsxService.createPortForwardRule(networkRule);
+ result &= nsxService.createPortForwardRule(networkRule);
} else if (rule.getState() == FirewallRule.State.Revoke) {
- return nsxService.deletePortForwardRule(networkRule);
+ result &= nsxService.deletePortForwardRule(networkRule);
}
}
- return true;
+ return result;
}
public Pair<VpcVO, NetworkVO> getVpcOrNetwork(Long vpcId, long networkId) {
@@ -613,6 +615,7 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
@Override
public boolean applyLBRules(Network network, List<LoadBalancingRule> rules) throws ResourceUnavailableException {
+ boolean result = true;
for (LoadBalancingRule loadBalancingRule : rules) {
if (loadBalancingRule.getState() == FirewallRule.State.Active) {
continue;
@@ -638,12 +641,12 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
.setAlgorithm(loadBalancingRule.getAlgorithm())
.build();
if (loadBalancingRule.getState() == FirewallRule.State.Add) {
- return nsxService.createLbRule(networkRule);
+ result &= nsxService.createLbRule(networkRule);
} else if (loadBalancingRule.getState() == FirewallRule.State.Revoke) {
- return nsxService.deleteLbRule(networkRule);
+ result &= nsxService.deleteLbRule(networkRule);
}
}
- return true;
+ return result;
}
@Override
diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
index 2927db638ab..844e0380757 100644
--- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
+++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
@@ -51,6 +51,9 @@ import javax.naming.ConfigurationException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
+import com.cloud.kubernetes.cluster.KubernetesClusterHelper;
+import com.cloud.network.dao.NsxProviderDao;
+import com.cloud.network.element.NsxProviderVO;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@@ -589,6 +592,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
@Inject
VMScheduleManager vmScheduleManager;
+ @Inject
+ NsxProviderDao nsxProviderDao;
private ScheduledExecutorService _executor = null;
private ScheduledExecutorService _vmIpFetchExecutor = null;
@@ -597,6 +602,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
private boolean _dailyOrHourly = false;
private int capacityReleaseInterval;
private ExecutorService _vmIpFetchThreadExecutor;
+ private List<KubernetesClusterHelper> kubernetesClusterHelpers;
private String _instance;
@@ -610,6 +616,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
private static final int NUM_OF_2K_BLOCKS = 512;
private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS * MAX_USER_DATA_LENGTH_BYTES;
+ public List<KubernetesClusterHelper> getKubernetesClusterHelpers() {
+ return kubernetesClusterHelpers;
+ }
+
+ public void setKubernetesClusterHelpers(final List<KubernetesClusterHelper> kubernetesClusterHelpers) {
+ this.kubernetesClusterHelpers = kubernetesClusterHelpers;
+ }
+
@Inject
private OrchestrationService _orchSrvc;
@@ -2528,11 +2542,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
}
// cleanup port forwarding rules
- if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
- s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
- } else {
- success = false;
- s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
+ VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId);
+ NsxProviderVO nsx = nsxProviderDao.findByZoneId(vmInstanceVO.getDataCenterId());
+ if (Objects.isNull(nsx) || Objects.isNull(kubernetesClusterHelpers.get(0).findByVmId(vmId))) {
+ if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
+ s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
+ } else {
+ success = false;
+ s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
+ }
}
// cleanup load balancer rules
diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
index 7227264e229..9656d544fbb 100644
--- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
+++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
@@ -106,8 +106,9 @@
<bean id="configurationServerImpl" class="com.cloud.server.ConfigurationServerImpl" />
-
- <bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl" />
+ <bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl">
+ <property name="kubernetesClusterHelpers" value="#{kubernetesClusterHelperRegistry.registered}" />
+ </bean>
<bean id="consoleProxyManagerImpl" class="com.cloud.consoleproxy.ConsoleProxyManagerImpl">
<property name="consoleProxyAllocators"