You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ro...@apache.org on 2022/07/06 07:04:55 UTC
[cloudstack] branch 4.17 updated: cks: fix k8s version upgrade (#6513)
This is an automated email from the ASF dual-hosted git repository.
rohit pushed a commit to branch 4.17
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.17 by this push:
new 67e941f690 cks: fix k8s version upgrade (#6513)
67e941f690 is described below
commit 67e941f690884cba63632b2c709e1dfa095e18a8
Author: Abhishek Kumar <ab...@gmail.com>
AuthorDate: Wed Jul 6 12:34:49 2022 +0530
cks: fix k8s version upgrade (#6513)
Fixes #6514
On latest systemvm template used for CKS /usr/sbin is not present in the $PATH for normal user used during upgrade. This leads to failure for blkid command. Due to this during k8s version upgrade ISO is not being able to mount on the k8s cluster VMs and upgrade process is not carried out.
This PR fixes mounting of k8s version ISO and also returns failure for script when ISO mounting is failed.
Same failure is not seen during deployment of the cluster because setup-kube-system workflow is executed as ROOT user and it has a different value for $PATH.
From /etc/login.defs:
ENV_SUPATH PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV_PATH PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
Signed-off-by: Abhishek Kumar <ab...@gmail.com>
---
.../KubernetesClusterUpgradeWorker.java | 5 +-
.../cluster/utils/KubernetesClusterUtil.java | 38 ++++++++-
.../main/resources/script/upgrade-kubernetes.sh | 6 ++
.../cluster/utils/KubernetesClusterUtilTest.java | 93 ++++++++++++++++++++++
4 files changed, 140 insertions(+), 2 deletions(-)
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
index e970cf6939..1cc525c49e 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
import com.cloud.hypervisor.Hypervisor;
@@ -36,7 +37,6 @@ import com.cloud.uservm.UserVm;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.ssh.SshHelper;
-import org.apache.commons.lang3.StringUtils;
public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorker {
@@ -122,6 +122,9 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
}
}
+ if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), i==0, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName)) {
+ logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node on VM : %s upgraded to version %s", kubernetesCluster.getName(), vm.getDisplayName(), upgradeVersion.getSemanticVersion()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
+ }
if (LOGGER.isInfoEnabled()) {
LOGGER.info(String.format("Successfully upgraded node on VM %s in Kubernetes cluster %s with Kubernetes version(%s) ID: %s",
vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid()));
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java
index 3ec49b1ded..99da4435c8 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java
@@ -32,6 +32,7 @@ import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import org.apache.cloudstack.utils.security.SSLUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import com.cloud.kubernetes.cluster.KubernetesCluster;
@@ -39,12 +40,13 @@ import com.cloud.uservm.UserVm;
import com.cloud.utils.Pair;
import com.cloud.utils.nio.TrustAllManager;
import com.cloud.utils.ssh.SshHelper;
-import org.apache.commons.lang3.StringUtils;
public class KubernetesClusterUtil {
protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterUtil.class);
+ public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl version --short";
+
public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port,
String user, File sshKeyFile, String nodeName) throws Exception {
Pair<Boolean, String> result = SshHelper.sshExecute(ipAddress, port,
@@ -324,4 +326,38 @@ public class KubernetesClusterUtil {
}
return token.toString().substring(0, 64);
}
+
+ public static boolean clusterNodeVersionMatches(final String version, boolean isControlNode,
+ final String ipAddress, final int port,
+ final String user, final File sshKeyFile,
+ final String hostName) {
+ Pair<Boolean, String> result = null;
+ try {
+ result = SshHelper.sshExecute(
+ ipAddress, port,
+ user, sshKeyFile, null,
+ CLUSTER_NODE_VERSION_COMMAND,
+ 10000, 10000, 20000);
+ } catch (Exception e) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(String.format("Failed to retrieve Kubernetes version from cluster node : %s due to exception", hostName), e);
+ }
+ return false;
+ }
+ if (Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) {
+ return false;
+ }
+ String response = result.second();
+ boolean clientVersionPresent = false;
+ boolean serverVersionPresent = false;
+ for (String line : response.split("\n")) {
+ if (line.contains("Client Version") && line.contains(String.format("v%s", version))) {
+ clientVersionPresent = true;
+ }
+ if (isControlNode && line.contains("Server Version") && line.contains(String.format("v%s", version))) {
+ serverVersionPresent = true;
+ }
+ }
+ return clientVersionPresent && (!isControlNode || serverVersionPresent);
+ }
}
diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
index 80ea10df1f..84f764de93 100755
--- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
+++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
@@ -37,6 +37,9 @@ if [ $# -gt 3 ]; then
fi
export PATH=$PATH:/opt/bin
+if [[ "$PATH" != *:/usr/sbin && "$PATH" != *:/usr/sbin:* ]]; then
+ export PATH=$PATH:/usr/sbin
+fi
ISO_MOUNT_DIR=/mnt/k8sdisk
BINARIES_DIR=${ISO_MOUNT_DIR}/
@@ -149,4 +152,7 @@ if [ -d "$BINARIES_DIR" ]; then
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
eject "${iso_drive_path}"
fi
+else
+ echo "ERROR: Unable to access Binaries directory for upgrade version ${UPGRADE_VERSION}"
+ exit 1
fi
diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java
new file mode 100644
index 0000000000..475eeba7fc
--- /dev/null
+++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.kubernetes.cluster.utils;
+
+import java.io.File;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import com.cloud.utils.Pair;
+import com.cloud.utils.ssh.SshHelper;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(SshHelper.class)
+public class KubernetesClusterUtilTest {
+ String ipAddress = "10.1.1.1";
+ int port = 2222;
+ String user = "user";
+ File sshKeyFile = Mockito.mock(File.class);
+ String hostName = "host";
+
+ private void mockSshHelperExecuteThrowAndTestVersionMatch() {
+ try {
+ Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenThrow(Exception.class);
+ } catch (Exception e) {
+ Assert.fail(String.format("Exception: %s", e.getMessage()));
+ }
+ Assert.assertFalse(KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", false, ipAddress, port, user, sshKeyFile, hostName));
+ }
+
+ private void mockSshHelperExecuteAndTestVersionMatch(boolean status, String response, boolean isControlNode, boolean expectedResult) {
+ try {
+ Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenReturn(new Pair<>(status, response));
+ } catch (Exception e) {
+ Assert.fail(String.format("Exception: %s", e.getMessage()));
+ }
+ boolean result = KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", isControlNode, ipAddress, port, user, sshKeyFile, hostName);
+ Assert.assertEquals(expectedResult, result);
+ }
+
+ @Test
+ public void testClusterNodeVersionMatches() {
+ PowerMockito.mockStatic(SshHelper.class);
+ String v1233WorkerNodeOutput = "Client Version: v1.23.3\n" +
+ "The connection to the server localhost:8080 was refused - did you specify the right host or port?";
+ String v1240WorkerNodeOutput = "Client Version: v1.24.0\n" +
+ "Kustomize Version: v4.5.4\n" +
+ "The connection to the server localhost:8080 was refused - did you specify the right host or port?";
+ String v1240ControlNodeOutput = "Client Version: v1.24.0\n" +
+ "Kustomize Version: v4.5.4\n" +
+ "Server Version: v1.24.0";
+ mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, false, true);
+
+ mockSshHelperExecuteAndTestVersionMatch(true, v1240ControlNodeOutput, true, true);
+
+ mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, true, false);
+
+ mockSshHelperExecuteAndTestVersionMatch(false, v1240WorkerNodeOutput, false, false);
+
+ mockSshHelperExecuteAndTestVersionMatch(true, v1233WorkerNodeOutput, false, false);
+
+ mockSshHelperExecuteAndTestVersionMatch(true, "Client Version: v1.24.0\n" +
+ "Kustomize Version: v4.5.4\n" +
+ "Server Version: v1.23.0", true, false);
+
+ mockSshHelperExecuteAndTestVersionMatch(true, null, false, false);
+
+ mockSshHelperExecuteAndTestVersionMatch(false, "-\n-", false, false);
+
+ mockSshHelperExecuteAndTestVersionMatch(false, "1.24.0", false, false);
+
+ mockSshHelperExecuteThrowAndTestVersionMatch();
+ }
+}