You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by GitBox <gi...@apache.org> on 2021/07/22 09:06:41 UTC
[GitHub] [cloudstack] rp- commented on a change in pull request #4994: Linstor volume plugin
rp- commented on a change in pull request #4994:
URL: https://github.com/apache/cloudstack/pull/4994#discussion_r674624082
##########
File path: plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
##########
@@ -0,0 +1,578 @@
+package com.cloud.hypervisor.kvm.storage;
+
+import com.linbit.linstor.api.ApiClient;
+import com.linbit.linstor.api.ApiException;
+import com.linbit.linstor.api.Configuration;
+import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Properties;
+import com.linbit.linstor.api.model.ProviderKind;
+import com.linbit.linstor.api.model.Resource;
+import com.linbit.linstor.api.model.ResourceCreate;
+import com.linbit.linstor.api.model.ResourceDefinition;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
+import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
+import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.VolumeDefinition;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.StringJoiner;
+
+import com.cloud.storage.Storage;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+@StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor)
+public class LinstorStorageAdaptor implements StorageAdaptor {
+ private static final Logger s_logger = Logger.getLogger(LinstorStorageAdaptor.class);
+ private static final Map<String, KVMStoragePool> MapStorageUuidToStoragePool = new HashMap<>();
+ private final String localNodeName;
+
+ private DevelopersApi getLinstorAPI(KVMStoragePool pool) {
+ ApiClient client = Configuration.getDefaultApiClient();
+ client.setBasePath(pool.getSourceHost());
+ return new DevelopersApi(client);
+ }
+
+ private String getLinstorRscName(String name) {
+ return "cs-" + name;
+ }
+
+ private String getHostname() {
+ // either there is already some function for that in the agent or a better way.
+ ProcessBuilder pb = new ProcessBuilder("hostname");
+ try
+ {
+ String result;
+ Process p = pb.start();
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
+
+ StringJoiner sj = new StringJoiner(System.getProperty("line.separator"));
+ reader.lines().iterator().forEachRemaining(sj::add);
+ result = sj.toString();
+
+ p.waitFor();
+ p.destroy();
+ return result.trim();
+ } catch (IOException | InterruptedException exc) {
+ throw new CloudRuntimeException("Unable to run 'hostname' command.");
+ }
+ }
+
+ private void handleLinstorApiAnswers(ApiCallRcList answers, String excMessage) {
+ if (answers.hasError()) {
+ for (ApiCallRc answer : answers) {
+ s_logger.error(answer.getMessage());
+ }
+ throw new CloudRuntimeException(excMessage);
+ }
+ }
+
+ public LinstorStorageAdaptor() {
+ localNodeName = getHostname();
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
+ s_logger.debug("Linstor getStoragePool: " + uuid + " -> " + refreshInfo);
+ return MapStorageUuidToStoragePool.get(uuid);
+ }
+
+ @Override
+ public KVMPhysicalDisk getPhysicalDisk(String name, KVMStoragePool pool)
+ {
+ s_logger.debug("Linstor: getPhysicalDisk for " + name);
+ if (name == null) {
+ return null;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try {
+ final String rscName = getLinstorRscName(name);
+
+ List<VolumeDefinition> volumeDefs = api.volumeDefinitionList(rscName, null, null);
+ final long size = volumeDefs.isEmpty() ? 0 : volumeDefs.get(0).getSizeKib() * 1024;
+
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+ if (!resources.isEmpty() && !resources.get(0).getVolumes().isEmpty()) {
+ final String devPath = resources.get(0).getVolumes().get(0).getDevicePath();
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath, name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ kvmDisk.setSize(size);
+ kvmDisk.setVirtualSize(size);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources or volumes for " + rscName);
+ throw new CloudRuntimeException("Linstor: viewResources didn't return resources or volumes.");
+ }
+ } catch (ApiException apiEx) {
+ s_logger.error(apiEx);
+ throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+ }
+ }
+
+ @Override
+ public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo,
+ Storage.StoragePoolType type)
+ {
+ s_logger.debug(String.format(
+ "Linstor createStoragePool: name: '%s', host: '%s', path: %s, userinfo: %s", name, host, path, userInfo));
+ LinstorStoragePool storagePool = new LinstorStoragePool(name, host, port, userInfo, type, this);
+
+ MapStorageUuidToStoragePool.put(name, storagePool);
+
+ return storagePool;
+ }
+
+ @Override
+ public boolean deleteStoragePool(String uuid) {
+ return MapStorageUuidToStoragePool.remove(uuid) != null;
+ }
+
+ @Override
+ public boolean deleteStoragePool(KVMStoragePool pool) {
+ return deleteStoragePool(pool.getUuid());
+ }
+
+ @Override
+ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format,
+ Storage.ProvisioningType provisioningType, long size)
+ {
+ final String rscName = getLinstorRscName(name);
+ LinstorStoragePool lpool = (LinstorStoragePool) pool;
+ final DevelopersApi api = getLinstorAPI(pool);
+
+ try {
+ List<ResourceDefinition> definitionList = api.resourceDefinitionList(
+ Collections.singletonList(rscName), null, null, null);
+
+ if (definitionList.isEmpty()) {
+ ResourceGroupSpawn rgSpawn = new ResourceGroupSpawn();
+ rgSpawn.setResourceDefinitionName(rscName);
+ rgSpawn.addVolumeSizesItem(size / 1024); // linstor uses KiB
+
+ s_logger.debug("Linstor: Spawn resource " + rscName);
+ ApiCallRcList answers = api.resourceGroupSpawn(lpool.getResourceGroup(), rgSpawn);
+ handleLinstorApiAnswers(answers, "Linstor: Unable to spawn resource.");
+ }
+
+ // query linstor for the device path
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.emptyList(),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+
+ // TODO make avialable on node
+
+ if (!resources.isEmpty() && !resources.get(0).getVolumes().isEmpty()) {
+ final String devPath = resources.get(0).getVolumes().get(0).getDevicePath();
+ s_logger.info("Linstor: Created drbd device: " + devPath);
+ final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath, name, pool);
+ kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
+ return kvmDisk;
+ } else {
+ s_logger.error("Linstor: viewResources didn't return resources or volumes.");
+ throw new CloudRuntimeException("Linstor: viewResources didn't return resources or volumes.");
+ }
+ } catch (ApiException apiEx) {
+ throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+ }
+ }
+
+ @Override
+ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map<String, String> details)
+ {
+ s_logger.debug(String.format("Linstor: connectPhysicalDisk %s:%s -> %s", pool.getUuid(), volumePath, details));
+ if (volumePath == null) {
+ s_logger.warn("volumePath is null, ignoring");
+ return false;
+ }
+
+ final DevelopersApi api = getLinstorAPI(pool);
+ try
+ {
+ final String rscName = getLinstorRscName(volumePath);
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(localNodeName),
+ Collections.singletonList(rscName),
+ Collections.emptyList(),
+ null,
+ null,
+ null);
+
+ if (resources.isEmpty()) {
+ s_logger.info("Linstor: resource not created yet: " + rscName);
+ ResourceCreate rscCreate = new ResourceCreate();
+ Resource rsc = new Resource();
+ rsc.setNodeName(localNodeName);
+ rscCreate.setResource(rsc);
+ ApiCallRcList answers = api.resourceCreate(rscName, Collections.singletonList(rscCreate));
+ if (answers.hasError()) {
+ s_logger.error("Could not create resource " + rscName);
+ throw new CloudRuntimeException(answers.get(0).getMessage());
+ }
+ // now diskful resource should be deployed
+ }
+ // keep diskless, maybe add an option, but I'm concerned about out of space situations
+// else {
+// final ResourceWithVolumes rsc = resources.get(0);
+// if (rsc.getFlags() != null && rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS)) {
+// ApiCallRcList answers = api.resourceToggleDiskful(rscName, localNode);
+// if (ApiCallChecker.hasError(answers)) {
+// s_logger.error("Unable to toggle disk on " + rscName);
+// throw new CloudRuntimeException(answers.get(0).getMessage());
+// }
+// }
+// }
+
+ // allow 2 primaries for live migration, should be removed by disconnect on the other end
+ ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+ Properties props = new Properties();
+ props.put("DrbdOptions/Net/allow-two-primaries", "yes");
+ rdm.setOverrideProps(props);
+ ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
+ if (answers.hasError()) {
+ s_logger.error("Unable to set 'allow-two-primaries' on " + rscName);
+ throw new CloudRuntimeException(answers.get(0).getMessage());
+ }
+ } catch (ApiException apiEx) {
+ s_logger.error(apiEx);
+ throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+ }
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool pool)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk " + pool.getUuid() + ":" + volumePath);
+ return true;
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(Map<String, String> volumeToDisconnect)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDisk map");
+ return true;
+ }
+
+ private Optional<ResourceWithVolumes> getResourceByPath(final List<ResourceWithVolumes> resources, String path) {
+ return resources.stream()
+ .filter(rsc -> rsc.getVolumes().stream()
+ .anyMatch(v -> v.getDevicePath().equals(path)))
+ .findFirst();
+ }
+
+ /**
+ * disconnectPhysicalDiskByPath is called after e.g. a live migration.
+ * The problem is we have no idea just from the path to which linstor-controller
+ * this resource would belong to. But as it should be highly unlikely that someone
+ * uses more than one linstor-controller to manage resource on the same kvm host.
+ * We will just take the first stored storagepool.
+ */
+ @Override
+ public boolean disconnectPhysicalDiskByPath(String localPath)
+ {
+ s_logger.debug("Linstor: disconnectPhysicalDiskByPath " + localPath);
+ // get first storage pool from the map, as we don't know any better:
+ if (!MapStorageUuidToStoragePool.isEmpty())
+ {
+ String firstKey = MapStorageUuidToStoragePool.keySet().stream().findFirst().get();
+ final KVMStoragePool pool = MapStorageUuidToStoragePool.get(firstKey);
+
+ s_logger.debug("Linstor: Using storpool: " + pool.getUuid());
+ final DevelopersApi api = getLinstorAPI(pool);
+
+ try
+ {
+ List<ResourceWithVolumes> resources = api.viewResources(
+ Collections.singletonList(localNodeName),
+ null,
+ null,
+ null,
+ null,
+ null);
+
+ Optional<ResourceWithVolumes> rsc = getResourceByPath(resources, localPath);
+
+ if (rsc.isPresent())
+ {
+ ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+ rdm.deleteProps(Collections.singletonList("DrbdOptions/Net/allow-two-primaries"));
+ ApiCallRcList answers = api.resourceDefinitionModify(rsc.get().getName(), rdm);
+ if (answers.hasError())
+ {
+ s_logger.error("Failed to remove 'allow-two-primaries' on " + rsc.get().getName());
+ throw new CloudRuntimeException(answers.get(0).getMessage());
+ }
+
+ return true;
+ }
+ s_logger.warn("Linstor: Couldn't find resource for this path: " + localPath);
+ } catch (ApiException apiEx) {
+ s_logger.error(apiEx);
+ throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean deletePhysicalDisk(String name, KVMStoragePool pool, Storage.ImageFormat format)
+ {
+ s_logger.debug("Linstor: deletePhysicalDisk " + name);
+ final DevelopersApi api = getLinstorAPI(pool);
+
+ try {
+ final String rscName = getLinstorRscName(name);
+ s_logger.debug("Linstor: delete resource definition " + rscName);
+ ApiCallRcList answers = api.resourceDefinitionDelete(rscName);
Review comment:
@slavkap No, you would need to delete the snapshots before you could delete the volume.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@cloudstack.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org