You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by pe...@apache.org on 2026/06/17 02:35:11 UTC
(cloudstack) 01/01: Pre-seed images on worker nodes before control plane upgrade for air-gapped envs
This is an automated email from the ASF dual-hosted git repository.
Pearl1594 pushed a commit to branch cks-offline-upgrade
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 220e4b2e59bba7d0e6ca6fb77cd52efb8ebf372e
Author: Pearl Dsilva <pe...@gmail.com>
AuthorDate: Tue Jun 16 22:34:50 2026 -0400
Pre-seed images on worker nodes before control plane upgrade for air-gapped envs
---
.../KubernetesClusterUpgradeWorker.java | 45 ++++++++++++++++++++++
.../main/resources/script/upgrade-kubernetes.sh | 10 +++++
2 files changed, 55 insertions(+)
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
index 4c2725fc2a2..9f59b04c44d 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
@@ -63,6 +63,20 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
upgradeScriptFile = retrieveScriptFile(upgradeScriptFilename);
}
+ private Pair<Boolean, String> preseedImagesOnVM(final UserVm vm, final int index) throws Exception {
+ int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index;
+ String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress;
+ SshHelper.scpTo(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null,
+ "~/", upgradeScriptFile.getAbsolutePath(), "0755");
+ String cmdStr = String.format("sudo ./%s %s false false %s %s true",
+ upgradeScriptFile.getName(),
+ upgradeVersion.getSemanticVersion(),
+ Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType()),
+ Objects.isNull(kubernetesCluster.getCniConfigId()));
+ return SshHelper.sshExecute(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null,
+ cmdStr, 10000, 10000, 5 * 60 * 1000);
+ }
+
private Pair<Boolean, String> runInstallScriptOnVM(final UserVm vm, final int index) throws Exception {
int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index;
String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress;
@@ -80,6 +94,37 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
}
private void upgradeKubernetesClusterNodes() {
+ // kubeadm upgrade apply schedules a post-upgrade health check pod on a non-cordoned node.
+ // In air-gapped clusters, workers lack the new pause image until their own upgrade runs,
+ // causing the pod pull to fail and the upgrade to stall. Pre-seed all workers first so
+ // the image is present before the control-plane upgrade starts.
+ if (clusterVMs.size() > 1) {
+ for (int i = 1; i < clusterVMs.size(); ++i) {
+ UserVm vm = clusterVMs.get(i);
+ String errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to pre-seed images on VM : %s",
+ kubernetesCluster.getName(), vm.getDisplayName());
+ for (int retry = KubernetesClusterService.KubernetesClusterUpgradeRetries.value(); retry >= 0; retry--) {
+ try {
+ Pair<Boolean, String> result = preseedImagesOnVM(vm, i);
+ if (result.first()) {
+ break;
+ }
+ if (retry > 0) {
+ logger.warn("{}, retries left: {}", errorMessage, retry);
+ } else {
+ logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
+ }
+ } catch (Exception e) {
+ if (retry > 0) {
+ logger.warn("{} due to {}, retries left: {}", errorMessage, e, retry);
+ } else {
+ logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e);
+ }
+ }
+ }
+ }
+ }
+
for (int i = 0; i < clusterVMs.size(); ++i) {
UserVm vm = clusterVMs.get(i);
String hostName = vm.getHostName();
diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
index a947d508436..494811a00de 100755
--- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
+++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
@@ -39,6 +39,10 @@ EXTERNAL_CNI=false
if [ $# -gt 4 ]; then
EXTERNAL_CNI="${5}"
fi
+PRESEED_ONLY=false
+if [ $# -gt 5 ]; then
+ PRESEED_ONLY="${6}"
+fi
export PATH=$PATH:/opt/bin
if [[ "$PATH" != *:/usr/sbin && "$PATH" != *:/usr/sbin:* ]]; then
@@ -120,6 +124,12 @@ if [ -d "$BINARIES_DIR" ]; then
sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml
fi
+ if [ "${PRESEED_ONLY}" == 'true' ]; then
+ systemctl restart containerd
+ umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
+ exit 0
+ fi
+
tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz
tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz