You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ha...@apache.org on 2023/05/31 11:58:38 UTC
[cloudstack] 07/23: working blockcopy api in libvirt
This is an automated email from the ASF dual-hosted git repository.
harikrishna pushed a commit to branch LiveStoragMigrationScaleIOMain
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 0572d51f057967fb575d87323108e7fc8a212e2d
Author: Harikrishna Patnala <ha...@gmail.com>
AuthorDate: Fri Mar 10 17:17:00 2023 +0530
working blockcopy api in libvirt
---
.../LibvirtMigrateVolumeCommandWrapper.java | 9 +-
.../driver/ScaleIOPrimaryDataStoreDriver.java | 238 +++++++++++++--------
2 files changed, 152 insertions(+), 95 deletions(-)
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java
index b6930683069..c78ea453250 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java
@@ -39,7 +39,6 @@ import java.util.Map;
import java.util.UUID;
import com.cloud.storage.Storage;
-import com.sun.jna.Pointer;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
@@ -54,9 +53,6 @@ import org.libvirt.LibvirtException;
import org.libvirt.event.BlockJobListener;
import org.libvirt.event.BlockJobStatus;
import org.libvirt.event.BlockJobType;
-import org.libvirt.jna.ConnectionPointer;
-import org.libvirt.jna.DomainPointer;
-import org.libvirt.jna.Libvirt;
@ResourceWrapper(handles = MigrateVolumeCommand.class)
public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVolumeCommand, Answer, LibvirtComputingResource> {
@@ -121,6 +117,11 @@ public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<Mig
return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null);
}
+ final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
+ PrimaryDataStoreTO spool = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore();
+ KVMStoragePool pool = storagePoolMgr.getStoragePool(spool.getPoolType(), spool.getUuid());
+ pool.connectPhysicalDisk(destVolumeObjectTO.getPath(), null);
+
LibvirtVMDef.DiskDef diskdef = generateDestinationDiskDefinition(dm, srcVolumeId, srcPath, diskFilePath);
TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0);
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
index 6c3dd0f1b08..b70ae125f80 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
@@ -707,6 +707,9 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
}
result = new CopyCommandResult(null, answer);
+ if (answer != null && !answer.getResult()) {
+ result.setResult(answer.getDetails());
+ }
callback.complete(result);
}
@@ -760,121 +763,176 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost) {
// Volume migration within same PowerFlex/ScaleIO cluster (with same System ID)
+ final long srcVolumeId = srcData.getId();
DataStore srcStore = srcData.getDataStore();
+ Map<String, String> srcDetails = getVolumeDetails((VolumeInfo) srcData, srcStore);
+
DataStore destStore = destData.getDataStore();
+ final long destPoolId = destStore.getId();
+ Map<String, String> destDetails = getVolumeDetails((VolumeInfo) destData, destStore);
+ VolumeObjectTO destVolTO = (VolumeObjectTO) destData.getTO();
+ String destVolumePath = null;
+
+ Host host = findEndpointForVolumeOperation(srcData);
+ EndPoint ep = RemoteHostEndPoint.getHypervisorHostEndPoint(host);
+
Answer answer = null;
try {
- long srcPoolId = srcStore.getId();
- long destPoolId = destStore.getId();
-
- final ScaleIOGatewayClient client = getScaleIOClient(srcPoolId);
- final String srcVolumePath = ((VolumeInfo) srcData).getPath();
- final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumePath);
- final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId);
- final String destStoragePoolId = destStoragePool.getPath();
CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId());
- String destVolumePath = createAnswer.getData().getPath();
- //final String destVolumeId = UUID.randomUUID().toString().replaceAll("-", "").substring(0, 16);
- //final String destScaleIOVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, srcData.getId(), srcData.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value());
- //String destVolumePath = String.format("%s:%s", destVolumeId, destScaleIOVolumeName);
-
- VolumeObjectTO destVolTO = (VolumeObjectTO) destData.getTO();
+ destVolumePath = createAnswer.getData().getPath();
destVolTO.setPath(destVolumePath);
- Map<String, String> srcDetails = getVolumeDetails((VolumeInfo) srcData, srcStore);
- Map<String, String> destDetails = getVolumeDetails((VolumeInfo) destData, destStore);
+ grantAccess(destData, host, destData.getDataStore());
- String value = configDao.getValue(Config.MigrateWait.key());
- int waitInterval = NumbersUtil.parseInt(value, Integer.parseInt(Config.MigrateWait.getDefaultValue()));
+ int waitInterval = NumbersUtil.parseInt(configDao.getValue(Config.MigrateWait.key()), Integer.parseInt(Config.MigrateWait.getDefaultValue()));
MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcData.getTO(), destVolTO,
srcDetails, destDetails, waitInterval);
-
- long hostId = 0;
- VMInstanceVO instance = vmInstanceDao.findVMByInstanceName(((VolumeInfo) srcData).getAttachedVmName());
- if (instance.getState().equals(VirtualMachine.State.Running)) {
- hostId = instance.getHostId();
- }
- if (hostId == 0) {
- hostId = selector.select(srcData, true).getId();
- }
- HostVO host = hostDao.findById(hostId);
- if (host == null) {
- throw new CloudRuntimeException("Found no hosts to run resize command on");
- }
- EndPoint ep = RemoteHostEndPoint.getHypervisorHostEndPoint(host);
- grantAccess(destData, ep, destData.getDataStore());
-
answer = ep.sendMessage(migrateVolumeCommand);
boolean migrateStatus = answer.getResult();
if (migrateStatus) {
- String newVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, destData.getId(),
- destStoragePool.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value());
- boolean renamed = client.renameVolume(srcVolumeId, newVolumeName);
+ updateVolumeAfterCopyVolume(srcData, destData);
+ updateSnapshotsAfterCopyVolume(srcData, destData);
- if (srcData.getId() != destData.getId()) {
- VolumeVO destVolume = volumeDao.findById(destData.getId());
- // Volume Id in the PowerFlex/ScaleIO pool remains the same after the migration
- // Update PowerFlex volume name only after it is renamed, to maintain the consistency
- if (renamed) {
- String newVolumePath = ScaleIOUtil.updatedPathWithVolumeName(srcVolumeId, newVolumeName);
- destVolume.set_iScsiName(newVolumePath);
- destVolume.setPath(newVolumePath);
- } else {
- destVolume.set_iScsiName(srcVolumePath);
- destVolume.setPath(srcVolumePath);
- }
- volumeDao.update(destData.getId(), destVolume);
+ LOGGER.debug(String.format("Successfully migrated migrate PowerFlex volume %d to storage pool %d", srcVolumeId, destPoolId));
+ answer = new Answer(null, true, null);
+ } else {
+ String errorMsg = "Failed to migrate PowerFlex volume: " + srcVolumeId + " to storage pool " + destPoolId;
+ LOGGER.debug(errorMsg);
+ answer = new Answer(null, false, errorMsg);
+ }
+ } catch (Exception e) {
+ LOGGER.error("Failed to migrate PowerFlex volume: " + srcVolumeId + " due to: " + e.getMessage());
+ answer = new Answer(null, false, e.getMessage());
+ }
- VolumeVO srcVolume = volumeDao.findById(srcData.getId());
- srcVolume.set_iScsiName(null);
- srcVolume.setPath(null);
- srcVolume.setFolder(null);
- volumeDao.update(srcData.getId(), srcVolume);
- } else {
- // Live migrate volume
- VolumeVO volume = volumeDao.findById(srcData.getId());
- Long oldPoolId = volume.getPoolId();
- volume.setPoolId(destPoolId);
- volume.setLastPoolId(oldPoolId);
- volumeDao.update(srcData.getId(), volume);
+ if (destVolumePath != null && !answer.getResult()) {
+ revertCopyVolumeOperations(srcData, destData, host, destVolumePath);
+ }
+
+ return answer;
+ }
+
+ private void checkForDestinationVolumeExistence(DataStore destStore, String destVolumePath) throws Exception {
+ int retryCount = 3;
+ while (retryCount > 0) {
+ try {
+ Thread.sleep(3000); // Try after few secs
+ String destScaleIOVolumeId = ScaleIOUtil.getVolumePath(destVolumePath);
+ final ScaleIOGatewayClient destClient = getScaleIOClient(destStore.getId());
+ org.apache.cloudstack.storage.datastore.api.Volume destScaleIOVolume = destClient.getVolume(destScaleIOVolumeId);
+ if (destScaleIOVolume != null) {
+ return;
}
+ } catch (Exception ex) {
+ LOGGER.error("Exception while checking for existence of the volume at " + destVolumePath + " - " + ex.getLocalizedMessage());
+ throw ex;
+ } finally {
+ retryCount--;
+ }
+ }
+ }
- List<SnapshotVO> snapshots = snapshotDao.listByVolumeId(srcData.getId());
- if (CollectionUtils.isNotEmpty(snapshots)) {
- for (SnapshotVO snapshot : snapshots) {
- SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary);
- if (snapshotStore == null) {
- continue;
- }
+ private void updateVolumeAfterCopyVolume(DataObject srcData, DataObject destData) {
+ // destination volume is already created and volume path is set in database by this time at "CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId());"
+ final long srcVolumeId = srcData.getId();
+ final long destVolumeId = destData.getId();
+
+ if (srcVolumeId != destVolumeId) {
+ VolumeVO srcVolume = volumeDao.findById(srcVolumeId);
+ srcVolume.set_iScsiName(null);
+ srcVolume.setPath(null);
+ srcVolume.setFolder(null);
+ volumeDao.update(srcVolumeId, srcVolume);
+ } else {
+ // Live migrate volume
+ VolumeVO volume = volumeDao.findById(srcVolumeId);
+ Long oldPoolId = volume.getPoolId();
+ volume.setLastPoolId(oldPoolId);
+ volumeDao.update(srcVolumeId, volume);
+ }
+ }
+ private Host findEndpointForVolumeOperation(DataObject srcData) {
+ long hostId = 0;
+ VMInstanceVO instance = vmInstanceDao.findVMByInstanceName(((VolumeInfo) srcData).getAttachedVmName());
+ if (instance.getState().equals(VirtualMachine.State.Running)) {
+ hostId = instance.getHostId();
+ }
+ if (hostId == 0) {
+ hostId = selector.select(srcData, true).getId();
+ }
+ HostVO host = hostDao.findById(hostId);
+ if (host == null) {
+ throw new CloudRuntimeException("Found no hosts to run migrate volume command on");
+ }
- String snapshotVolumeId = ScaleIOUtil.getVolumePath(snapshotStore.getInstallPath());
- String newSnapshotName = String.format("%s-%s-%s-%s", ScaleIOUtil.SNAPSHOT_PREFIX, snapshot.getId(),
- destStoragePool.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value());
- renamed = client.renameVolume(snapshotVolumeId, newSnapshotName);
+ return host;
+ }
+ private void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destData) throws Exception {
+ final long srcVolumeId = srcData.getId();
+ DataStore srcStore = srcData.getDataStore();
+ final long srcPoolId = srcStore.getId();
+ final ScaleIOGatewayClient client = getScaleIOClient(srcPoolId);
- snapshotStore.setDataStoreId(destPoolId);
- // Snapshot Id in the PowerFlex/ScaleIO pool remains the same after the migration
- // Update PowerFlex snapshot name only after it is renamed, to maintain the consistency
- if (renamed) {
- snapshotStore.setInstallPath(ScaleIOUtil.updatedPathWithVolumeName(snapshotVolumeId, newSnapshotName));
- }
- snapshotDataStoreDao.update(snapshotStore.getId(), snapshotStore);
- }
+ DataStore destStore = destData.getDataStore();
+ final long destPoolId = destStore.getId();
+ final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId);
+
+ List<SnapshotVO> snapshots = snapshotDao.listByVolumeId(srcVolumeId);
+ if (CollectionUtils.isNotEmpty(snapshots)) {
+ for (SnapshotVO snapshot : snapshots) {
+ SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary);
+ if (snapshotStore == null) {
+ continue;
}
- answer = new Answer(null, true, null);
- } else {
- String errorMsg = "Failed to migrate PowerFlex volume: " + srcData.getId() + " to storage pool " + destPoolId;
- LOGGER.debug(errorMsg);
- answer = new Answer(null, false, errorMsg);
+ String snapshotVolumeId = ScaleIOUtil.getVolumePath(snapshotStore.getInstallPath());
+ String newSnapshotName = String.format("%s-%s-%s-%s", ScaleIOUtil.SNAPSHOT_PREFIX, snapshot.getId(),
+ destStoragePool.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value());
+ boolean renamed = client.renameVolume(snapshotVolumeId, newSnapshotName);
+
+ snapshotStore.setDataStoreId(destPoolId);
+ // Snapshot Id in the PowerFlex/ScaleIO pool remains the same after the migration
+ // Update PowerFlex snapshot name only after it is renamed, to maintain the consistency
+ if (renamed) {
+ snapshotStore.setInstallPath(ScaleIOUtil.updatedPathWithVolumeName(snapshotVolumeId, newSnapshotName));
+ }
+ snapshotDataStoreDao.update(snapshotStore.getId(), snapshotStore);
}
+ }
+ }
+
+ private void revertCopyVolumeOperations(DataObject srcData, DataObject destData, Host host, String destVolumePath) {
+ final String srcVolumePath = ((VolumeInfo) srcData).getPath();
+ final String srcVolumeFolder = ((VolumeInfo) srcData).getFolder();
+ DataStore destStore = destData.getDataStore();
+
+ revokeAccess(destData, host, destData.getDataStore());
+ String errMsg;
+ try {
+ String scaleIOVolumeId = ScaleIOUtil.getVolumePath(destVolumePath);
+ final ScaleIOGatewayClient client = getScaleIOClient(destStore.getId());
+ Boolean deleteResult = client.deleteVolume(scaleIOVolumeId);
+ if (!deleteResult) {
+ errMsg = "Failed to delete PowerFlex volume with id: " + scaleIOVolumeId;
+ LOGGER.warn(errMsg);
+ }
+
} catch (Exception e) {
- LOGGER.error("Failed to migrate PowerFlex volume: " + srcData.getId() + " due to: " + e.getMessage());
- answer = new Answer(null, false, e.getMessage());
+ errMsg = "Unable to delete destination PowerFlex volume: " + destVolumePath + " due to " + e.getMessage();
+ LOGGER.warn(errMsg);
+ throw new CloudRuntimeException(errMsg, e);
}
- return answer;
+ final long srcVolumeId = srcData.getId();
+ if (srcVolumeId == destData.getId()) {
+ VolumeVO volume = volumeDao.findById(srcVolumeId);
+ volume.set_iScsiName(srcVolumePath);
+ volume.setPath(srcVolumePath);
+ volume.setFolder(srcVolumeFolder);
+ volume.setPoolId(((VolumeInfo) srcData).getPoolId());
+ volumeDao.update(srcVolumeId, volume);
+ }
}
private Map<String, String> getVolumeDetails(VolumeInfo volumeInfo, DataStore dataStore) {
@@ -938,8 +996,6 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId);
final String destStoragePoolId = destStoragePool.getPath();
int migrationTimeout = StorageManager.KvmStorageOfflineMigrationWait.value();
- LOGGER.info("HARI source volume " + srcVolumeId);
- LOGGER.info("HARI destination volume " + destStoragePoolId);
boolean migrateStatus = client.migrateVolume(srcVolumeId, destStoragePoolId, migrationTimeout);
if (migrateStatus) {
String newVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, destData.getId(),