You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ro...@apache.org on 2021/08/09 09:15:40 UTC
[cloudstack] branch main updated: kubernetes: Deploy
kubernetes-provider when creating a cluster (#5254)
This is an automated email from the ASF dual-hosted git repository.
rohit pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/main by this push:
new 3c8c704 kubernetes: Deploy kubernetes-provider when creating a cluster (#5254)
3c8c704 is described below
commit 3c8c704df3e132472a2b42306b9f4814451c5854
Author: davidjumani <dj...@gmail.com>
AuthorDate: Mon Aug 9 14:45:20 2021 +0530
kubernetes: Deploy kubernetes-provider when creating a cluster (#5254)
* kubernetes: Deploy kubernetes-provider when creating a cluster
Co-authored-by: Abhishek Kumar <ab...@gmail.com>
---
.../main/java/com/cloud/user/AccountService.java | 2 +
.../cluster/KubernetesClusterManagerImpl.java | 51 ++++++++-
.../cluster/KubernetesClusterService.java | 1 +
.../KubernetesClusterActionWorker.java | 119 +++++++++++++++++++++
.../KubernetesClusterStartWorker.java | 1 +
.../KubernetesClusterUpgradeWorker.java | 6 +-
.../main/resources/conf/k8s-control-node-add.yml | 4 +
.../src/main/resources/conf/k8s-control-node.yml | 4 +
.../src/main/resources/conf/k8s-node.yml | 4 +
.../main/resources/script/deploy-cloudstack-secret | 68 ++++++++++++
.../src/main/resources/script/deploy-provider | 36 +++++++
.../main/resources/script/upgrade-kubernetes.sh | 4 +
.../contrail/management/MockAccountManager.java | 6 ++
scripts/util/create-kubernetes-binaries-iso.sh | 8 ++
.../java/com/cloud/user/AccountManagerImpl.java | 4 +
.../com/cloud/user/MockAccountManagerImpl.java | 5 +
test/integration/smoke/test_kubernetes_clusters.py | 8 +-
17 files changed, 327 insertions(+), 4 deletions(-)
diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java
index 4e3733b..98b1618 100644
--- a/api/src/main/java/com/cloud/user/AccountService.java
+++ b/api/src/main/java/com/cloud/user/AccountService.java
@@ -121,4 +121,6 @@ public interface AccountService {
UserAccount getUserAccountById(Long userId);
public Map<String, String> getKeys(GetUserKeysCmd cmd);
+
+ public Map<String, String> getKeys(Long userId);
}
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
index a384a07..7e52d98 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
@@ -32,6 +32,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.UUID;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
@@ -53,6 +54,7 @@ import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse;
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.api.response.UserVmResponse;
+import org.apache.cloudstack.config.ApiServiceConfiguration;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.ConfigKey;
@@ -134,7 +136,11 @@ import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountService;
import com.cloud.user.SSHKeyPairVO;
+import com.cloud.user.User;
+import com.cloud.user.UserAccount;
+import com.cloud.user.UserVO;
import com.cloud.user.dao.SSHKeyPairDao;
+import com.cloud.user.dao.UserDao;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
import com.cloud.utils.component.ComponentContext;
@@ -198,6 +204,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
@Inject
protected AccountManager accountManager;
@Inject
+ protected UserDao userDao;
+ @Inject
protected VMInstanceDao vmInstanceDao;
@Inject
protected UserVmJoinDao userVmJoinDao;
@@ -644,7 +652,17 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
return response;
}
+ private void validateEndpointUrl() {
+ String csUrl = ApiServiceConfiguration.ApiServletPath.value();
+ if (csUrl == null || csUrl.contains("localhost")) {
+ String error = String.format("Global setting %s has to be set to the Management Server's API end point",
+ ApiServiceConfiguration.ApiServletPath.key());
+ throw new InvalidParameterValueException(error);
+ }
+ }
+
private void validateKubernetesClusterCreateParameters(final CreateKubernetesClusterCmd cmd) throws CloudRuntimeException {
+ validateEndpointUrl();
final String name = cmd.getName();
final Long zoneId = cmd.getZoneId();
final Long kubernetesVersionId = cmd.getKubernetesVersionId();
@@ -927,6 +945,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) {
// Validate parameters
+ validateEndpointUrl();
+
final Long kubernetesClusterId = cmd.getId();
final Long upgradeVersionId = cmd.getKubernetesVersionId();
if (kubernetesClusterId == null || kubernetesClusterId < 1L) {
@@ -1093,6 +1113,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
startWorker = ComponentContext.inject(startWorker);
if (onCreate) {
// Start for Kubernetes cluster in 'Created' state
+ Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId());
+ String[] keys = getServiceUserKeys(owner);
+ startWorker.setKeys(keys);
return startWorker.startKubernetesClusterOnCreate();
} else {
// Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started
@@ -1100,6 +1123,29 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
}
+ private String[] getServiceUserKeys(Account owner) {
+ if (owner == null) {
+ owner = CallContext.current().getCallingAccount();
+ }
+ String username = owner.getAccountName() + "-" + KUBEADMIN_ACCOUNT_NAME;
+ UserAccount kubeadmin = accountService.getActiveUserAccount(username, owner.getDomainId());
+ String[] keys = null;
+ if (kubeadmin == null) {
+ User kube = userDao.persist(new UserVO(owner.getAccountId(), username, UUID.randomUUID().toString(), owner.getAccountName(),
+ KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN));
+ keys = accountService.createApiKeyAndSecretKey(kube.getId());
+ } else {
+ String apiKey = kubeadmin.getApiKey();
+ String secretKey = kubeadmin.getSecretKey();
+ if (Strings.isNullOrEmpty(apiKey) || Strings.isNullOrEmpty(secretKey)) {
+ keys = accountService.createApiKeyAndSecretKey(kubeadmin.getId());
+ } else {
+ keys = new String[]{apiKey, secretKey};
+ }
+ }
+ return keys;
+ }
+
@Override
public boolean stopKubernetesCluster(long kubernetesClusterId) throws CloudRuntimeException {
if (!KubernetesServiceEnabled.value()) {
@@ -1240,9 +1286,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
}
validateKubernetesClusterUpgradeParameters(cmd);
+ KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId());
+ Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId());
+ String[] keys = getServiceUserKeys(owner);
KubernetesClusterUpgradeWorker upgradeWorker =
new KubernetesClusterUpgradeWorker(kubernetesClusterDao.findById(cmd.getId()),
- kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this);
+ kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this, keys);
upgradeWorker = ComponentContext.inject(upgradeWorker);
return upgradeWorker.upgradeCluster();
}
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
index db5ab91..07939dd 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
@@ -34,6 +34,7 @@ public interface KubernetesClusterService extends PluggableService, Configurable
static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0";
static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2;
static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048;
+ static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin";
static final ConfigKey<Boolean> KubernetesServiceEnabled = new ConfigKey<Boolean>("Advanced", Boolean.class,
"cloud.kubernetes.service.enabled",
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
index 5f663df..5426e9c 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
@@ -17,7 +17,9 @@
package com.cloud.kubernetes.cluster.actionworkers;
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -28,6 +30,7 @@ import javax.inject.Inject;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.ca.CAManager;
+import org.apache.cloudstack.config.ApiServiceConfiguration;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.commons.collections.CollectionUtils;
@@ -50,6 +53,7 @@ import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao;
import com.cloud.network.IpAddress;
import com.cloud.network.IpAddressManager;
import com.cloud.network.Network;
+import com.cloud.network.Network.GuestType;
import com.cloud.network.NetworkModel;
import com.cloud.network.dao.NetworkDao;
import com.cloud.service.dao.ServiceOfferingDao;
@@ -70,6 +74,7 @@ import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.utils.fsm.StateMachine2;
+import com.cloud.utils.ssh.SshHelper;
import com.cloud.vm.UserVmService;
import com.cloud.vm.dao.UserVmDao;
import com.google.common.base.Strings;
@@ -127,12 +132,22 @@ public class KubernetesClusterActionWorker {
protected String publicIpAddress;
protected int sshPort;
+
+ protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret";
+ protected final String deployProviderScriptFilename = "deploy-provider";
+ protected final String scriptPath = "/opt/bin/";
+ protected File deploySecretsScriptFile;
+ protected File deployProviderScriptFile;
+ protected KubernetesClusterManagerImpl manager;
+ protected String[] keys;
+
protected KubernetesClusterActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) {
this.kubernetesCluster = kubernetesCluster;
this.kubernetesClusterDao = clusterManager.kubernetesClusterDao;
this.kubernetesClusterDetailsDao = clusterManager.kubernetesClusterDetailsDao;
this.kubernetesClusterVmMapDao = clusterManager.kubernetesClusterVmMapDao;
this.kubernetesSupportedVersionDao = clusterManager.kubernetesSupportedVersionDao;
+ this.manager = clusterManager;
}
protected void init() {
@@ -380,4 +395,108 @@ public class KubernetesClusterActionWorker {
return false;
}
}
+
+ protected boolean createCloudStackSecret(String[] keys) {
+ File pkFile = getManagementServerSshPublicKeyFile();
+ Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
+ publicIpAddress = publicIpSshPort.first();
+ sshPort = publicIpSshPort.second();
+
+ try {
+ final String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'",
+ scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]);
+ Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER,
+ pkFile, null, command, 10000, 10000, 60000);
+ return result.first();
+ } catch (Exception e) {
+ String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName());
+ LOGGER.warn(msg, e);
+ }
+ return false;
+ }
+
+ protected File retrieveScriptFile(String filename) {
+ File file = null;
+ try {
+ String data = readResourceFile("/script/" + filename);
+ file = File.createTempFile(filename, ".sh");
+ BufferedWriter writer = new BufferedWriter(new FileWriter(file));
+ writer.write(data);
+ writer.close();
+ } catch (IOException e) {
+ logAndThrow(Level.ERROR, String.format("Kubernetes Cluster %s : Failed to to fetch script %s",
+ kubernetesCluster.getName(), filename), e);
+ }
+ return file;
+ }
+
+ protected void retrieveScriptFiles() {
+ deploySecretsScriptFile = retrieveScriptFile(deploySecretsScriptFilename);
+ deployProviderScriptFile = retrieveScriptFile(deployProviderScriptFilename);
+ }
+
+ protected void copyScripts(String nodeAddress, final int sshPort) {
+ try {
+ SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
+ "~/", deploySecretsScriptFile.getAbsolutePath(), "0755");
+ SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
+ "~/", deployProviderScriptFile.getAbsolutePath(), "0755");
+ String cmdStr = String.format("sudo mv ~/%s %s/%s", deploySecretsScriptFile.getName(), scriptPath, deploySecretsScriptFilename);
+ SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
+ cmdStr, 10000, 10000, 10 * 60 * 1000);
+ cmdStr = String.format("sudo mv ~/%s %s/%s", deployProviderScriptFile.getName(), scriptPath, deployProviderScriptFilename);
+ SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
+ cmdStr, 10000, 10000, 10 * 60 * 1000);
+ } catch (Exception e) {
+ throw new CloudRuntimeException(e);
+ }
+ }
+
+ protected boolean deployProvider() {
+ Network network = networkDao.findById(kubernetesCluster.getNetworkId());
+ // Since the provider creates IP addresses, don't deploy it unless the underlying network supports it
+ if (network.getGuestType() != GuestType.Isolated) {
+ logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network",
+ kubernetesCluster.getName()), null);
+ return true;
+ }
+ File pkFile = getManagementServerSshPublicKeyFile();
+ Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
+ publicIpAddress = publicIpSshPort.first();
+ sshPort = publicIpSshPort.second();
+
+ try {
+ String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename);
+ Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER,
+ pkFile, null, command, 10000, 10000, 60000);
+
+ // Maybe the file isn't present. Try and copy it
+ if (!result.first()) {
+ logMessage(Level.INFO, "Provider files missing. Adding them now", null);
+ retrieveScriptFiles();
+ copyScripts(publicIpAddress, sshPort);
+
+ if (!createCloudStackSecret(keys)) {
+ logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s",
+ kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+ }
+
+ // If at first you don't succeed ...
+ result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER,
+ pkFile, null, command, 10000, 10000, 60000);
+ if (!result.first()) {
+ throw new CloudRuntimeException(result.second());
+ }
+ }
+ return true;
+ } catch (Exception e) {
+ String msg = String.format("Failed to deploy kubernetes provider: %s : %s", kubernetesCluster.getName(), e.getMessage());
+ logAndThrow(Level.ERROR, msg);
+ return false;
+ }
+ }
+
+ public void setKeys(String[] keys) {
+ this.keys = keys;
+ }
}
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
index 9a30fdd..072094e 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
@@ -571,6 +571,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
if (!isKubernetesClusterDashboardServiceRunning(true, startTimeoutTime)) {
logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed);
}
+ deployProvider();
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
return true;
}
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
index 86c5c8e..e8b61d4 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
@@ -50,9 +50,11 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
public KubernetesClusterUpgradeWorker(final KubernetesCluster kubernetesCluster,
final KubernetesSupportedVersion upgradeVersion,
- final KubernetesClusterManagerImpl clusterManager) {
+ final KubernetesClusterManagerImpl clusterManager,
+ final String[] keys) {
super(kubernetesCluster, clusterManager);
this.upgradeVersion = upgradeVersion;
+ this.keys = keys;
}
private void retrieveUpgradeScriptFile() {
@@ -110,6 +112,8 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
}
try {
+ int port = (sshPort == CLUSTER_NODES_DEFAULT_START_SSH_PORT) ? sshPort + i : sshPort;
+ deployProvider();
result = runInstallScriptOnVM(vm, i);
} catch (Exception e) {
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e);
diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
index a8650ac..601df21 100644
--- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
+++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
@@ -125,6 +125,10 @@ write-files:
done <<< "$output"
setup_complete=true
fi
+ if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
+ mkdir -p /opt/provider
+ cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
+ fi
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
eject "${iso_drive_path}"
diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
index c2cecc4..44a78a3 100644
--- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
+++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
@@ -147,6 +147,10 @@ write-files:
fi
mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
+ if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
+ mkdir -p /opt/provider
+ cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
+ fi
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
eject "${iso_drive_path}"
diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
index f65cc9c..03ed701 100644
--- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
+++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
@@ -125,6 +125,10 @@ write-files:
done <<< "$output"
setup_complete=true
fi
+ if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
+ mkdir -p /opt/provider
+ cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
+ fi
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
eject "${iso_drive_path}"
diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret
new file mode 100755
index 0000000..9356f8a
--- /dev/null
+++ b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret
@@ -0,0 +1,68 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function usage() {
+ cat << USAGE
+Usage: ./deploy-cloudstack-secret [OPTIONS]...
+To deploy the keys needed for the cloudstack kubernetes provider.
+Arguments:
+ -u, --url string ID of the cluster
+ -k, --key string API Key
+ -s, --secret string Secret Key
+Other arguments:
+ -h, --help Display this help message and exit
+Examples:
+ ./deploy-cloudstack-secret -u http://localhost:8080 -k abcd -s efgh
+USAGE
+ exit 0
+}
+API_URL=""
+API_KEY=""
+SECRET_KEY=""
+while [ -n "$1" ]; do
+ case "$1" in
+ -h | --help)
+ usage
+ ;;
+ -u | --url)
+ API_URL=$2
+ shift 2
+ ;;
+ -k | --key)
+ API_KEY=$2
+ shift 2
+ ;;
+ -s | --secret)
+ SECRET_KEY=$2
+ shift 2
+ ;;
+ -*|*)
+ echo "ERROR: no such option $1. -h or --help for help"
+ exit 1
+ ;;
+ esac
+done
+cat > /tmp/cloud-config <<EOF
+[Global]
+api-url = $API_URL
+api-key = $API_KEY
+secret-key = $SECRET_KEY
+EOF
+# Create secret if not exists
+/opt/bin/kubectl -n kube-system get secret cloudstack-secret || /opt/bin/kubectl -n kube-system create secret generic cloudstack-secret --from-file=/tmp/cloud-config
+rm /tmp/cloud-config
diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-provider b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-provider
new file mode 100755
index 0000000..e707b59
--- /dev/null
+++ b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-provider
@@ -0,0 +1,36 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+(kubectl get pods -A | grep cloud-controller-manager) && exit 0
+
+if [ -e /opt/provider/provider.yaml ]; then
+ /opt/bin/kubectl apply -f /opt/provider/provider.yaml
+ exit 0
+else
+ mkdir -p /opt/provider
+ PROVIDER_URL="https://raw.githubusercontent.com/apache/cloudstack-kubernetes-provider/main/deployment.yaml"
+ provider_conf_file="/opt/provider/provider.yaml"
+ curl -sSL ${PROVIDER_URL} -o ${provider_conf_file}
+ if [ $? -ne 0 ]; then
+ echo "Unable to connect to the internet to download the provider deployment and image"
+ exit 1
+ else
+ /opt/bin/kubectl apply -f /opt/provider/provider.yaml
+ exit 0
+ fi
+fi
diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
index d661760..99153c9 100644
--- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
+++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
@@ -96,6 +96,10 @@ if [ -d "$BINARIES_DIR" ]; then
docker load < "${BINARIES_DIR}/docker/$line"
done <<< "$output"
fi
+ if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
+ mkdir -p /opt/provider
+ cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
+ fi
tar -f "${BINARIES_DIR}/cni/cni-plugins-amd64.tgz" -C /opt/cni/bin -xz
tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-amd64.tar.gz" -C /opt/bin -xz
diff --git a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java
index 68ff2e8..fb1eaa1 100644
--- a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java
+++ b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java
@@ -457,6 +457,12 @@ public class MockAccountManager extends ManagerBase implements AccountManager {
return null;
}
+
+ @Override
+ public Map<String, String> getKeys(Long userId) {
+ return null;
+ }
+
@Override
public void checkAccess(User user, ControlledEntity entity)
throws PermissionDeniedException {
diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh
index 67062be..241b45e 100755
--- a/scripts/util/create-kubernetes-binaries-iso.sh
+++ b/scripts/util/create-kubernetes-binaries-iso.sh
@@ -77,6 +77,11 @@ echo "Downloading dashboard config ${DASHBORAD_CONFIG_URL}"
dashboard_conf_file="${working_dir}/dashboard.yaml"
curl -sSL ${DASHBORAD_CONFIG_URL} -o ${dashboard_conf_file}
+PROVIDER_URL="https://raw.githubusercontent.com/apache/cloudstack-kubernetes-provider/main/deployment.yaml"
+echo "Downloading kubernetes cluster provider ${PROVIDER_URL}"
+provider_conf_file="${working_dir}/provider.yaml"
+curl -sSL ${PROVIDER_URL} -o ${provider_conf_file}
+
echo "Fetching k8s docker images..."
docker -v
if [ $? -ne 0 ]; then
@@ -102,6 +107,9 @@ do
output=`printf "%s\n" ${output} ${images}`
done
+provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d ' '`
+output=`printf "%s\n" ${output} ${provider_image}`
+
while read -r line; do
echo "Downloading docker image $line ---"
sudo docker pull "$line"
diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java
index b6f4e5e..88cd217 100644
--- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java
+++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java
@@ -2431,7 +2431,11 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
@Override
public Map<String, String> getKeys(GetUserKeysCmd cmd) {
final long userId = cmd.getID();
+ return getKeys(userId);
+ }
+ @Override
+ public Map<String, String> getKeys(Long userId) {
User user = getActiveUser(userId);
if (user == null) {
throw new InvalidParameterValueException("Unable to find user by id");
diff --git a/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java b/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java
index ea6287d..7916007 100644
--- a/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java
+++ b/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java
@@ -426,6 +426,11 @@ public class MockAccountManagerImpl extends ManagerBase implements Manager, Acco
}
@Override
+ public Map<String, String> getKeys(Long userId) {
+ return null;
+ }
+
+ @Override
public void checkAccess(User user, ControlledEntity entity)
throws PermissionDeniedException {
diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py
index d78f3a0..d4be6b8 100644
--- a/test/integration/smoke/test_kubernetes_clusters.py
+++ b/test/integration/smoke/test_kubernetes_clusters.py
@@ -75,8 +75,12 @@ class TestKubernetesCluster(cloudstackTestCase):
cls.kubernetes_version_ids = []
if cls.hypervisorNotSupported == False:
- cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient,
- name="cloud.kubernetes.service.enabled")[0].value
+ cls.endpoint_url = Configurations.list(cls.apiclient, name="endpointe.url")[0].value
+ if "localhost" in cls.endpoint_url:
+ endpoint_url = "http://%s:%d/client/api " %(cls.mgtSvrDetails["mgtSvrIp"], cls.mgtSvrDetails["port"])
+ cls.debug("Setting endpointe.url to %s" %(endpoint_url))
+ Configurations.update(cls.apiclient, "endpointe.url", endpoint_url)
+ cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value
if cls.initial_configuration_cks_enabled not in ["true", True]:
cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server")
Configurations.update(cls.apiclient,