You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by mc...@apache.org on 2013/08/13 20:42:42 UTC
git commit: updated refs/heads/4.2 to 0599a8b
Updated Branches:
refs/heads/4.2 86fcd647a -> 0599a8b15
CLOUDSTACK-3860: S3 and Swift-backed Secondary Storage Migration
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/0599a8b1
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/0599a8b1
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/0599a8b1
Branch: refs/heads/4.2
Commit: 0599a8b158641dbf1b4344dc48ba12fdc38cdd02
Parents: 86fcd64
Author: Min Chen <mi...@citrix.com>
Authored: Tue Aug 13 11:32:36 2013 -0700
Committer: Min Chen <mi...@citrix.com>
Committed: Tue Aug 13 11:41:31 2013 -0700
----------------------------------------------------------------------
.../com/cloud/upgrade/dao/Upgrade412to420.java | 620 +++++++++++++++----
1 file changed, 485 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/0599a8b1/engine/schema/src/com/cloud/upgrade/dao/Upgrade412to420.java
----------------------------------------------------------------------
diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade412to420.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade412to420.java
index 022485d..4e1e094 100644
--- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade412to420.java
+++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade412to420.java
@@ -27,11 +27,13 @@ import java.sql.Types;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
import org.apache.log4j.Logger;
@@ -70,8 +72,8 @@ public class Upgrade412to420 implements DbUpgrade {
return new File[] { new File(script) };
}
- @Override
- public void performDataMigration(Connection conn) {
+ @Override
+ public void performDataMigration(Connection conn) {
movePrivateZoneToDedicatedResource(conn);
upgradeVmwareLabels(conn);
persistLegacyZones(conn);
@@ -100,6 +102,8 @@ public class Upgrade412to420 implements DbUpgrade {
migrateVolumeHostRef(conn);
migrateTemplateHostRef(conn);
migrateSnapshotStoreRef(conn);
+ migrateS3ToImageStore(conn);
+ migrateSwiftToImageStore(conn);
fixNiciraKeys(conn);
fixRouterKeys(conn);
updateConcurrentConnectionsInNetworkOfferings(conn);
@@ -425,19 +429,19 @@ public class Upgrade412to420 implements DbUpgrade {
Map<HypervisorType, String> newTemplateUrl = new HashMap<HypervisorType, String>(){
{ put(HypervisorType.XenServer, "http://download.cloud.com/templates/4.2/systemvmtemplate-2013-06-12-master-xen.vhd.bz2");
- put(HypervisorType.VMware, "http://download.cloud.com/templates/4.2/systemvmtemplate-4.2-vh7.ova");
- put(HypervisorType.KVM, "http://download.cloud.com/templates/4.2/systemvmtemplate-2013-06-12-master-kvm.qcow2.bz2");
- put(HypervisorType.LXC, "http://download.cloud.com/templates/acton/acton-systemvm-02062012.qcow2.bz2");
- put(HypervisorType.Hyperv, "http://download.cloud.com/templates/4.2/systemvmtemplate-2013-06-12-master-xen.vhd.bz2");
+ put(HypervisorType.VMware, "http://download.cloud.com/templates/4.2/systemvmtemplate-4.2-vh7.ova");
+ put(HypervisorType.KVM, "http://download.cloud.com/templates/4.2/systemvmtemplate-2013-06-12-master-kvm.qcow2.bz2");
+ put(HypervisorType.LXC, "http://download.cloud.com/templates/acton/acton-systemvm-02062012.qcow2.bz2");
+ put(HypervisorType.Hyperv, "http://download.cloud.com/templates/4.2/systemvmtemplate-2013-06-12-master-xen.vhd.bz2");
}
};
Map<HypervisorType, String> newTemplateChecksum = new HashMap<HypervisorType, String>(){
{ put(HypervisorType.XenServer, "fb1b6e032a160d86f2c28feb5add6d83");
- put(HypervisorType.VMware, "8fde62b1089e5844a9cd3b9b953f9596");
- put(HypervisorType.KVM, "6cea42b2633841648040becb588bd8f0");
- put(HypervisorType.LXC, "2755de1f9ef2ce4d6f2bee2efbb4da92");
- put(HypervisorType.Hyperv, "fb1b6e032a160d86f2c28feb5add6d83");
+ put(HypervisorType.VMware, "8fde62b1089e5844a9cd3b9b953f9596");
+ put(HypervisorType.KVM, "6cea42b2633841648040becb588bd8f0");
+ put(HypervisorType.LXC, "2755de1f9ef2ce4d6f2bee2efbb4da92");
+ put(HypervisorType.Hyperv, "fb1b6e032a160d86f2c28feb5add6d83");
}
};
@@ -1826,26 +1830,238 @@ public class Upgrade412to420 implements DbUpgrade {
}
}
- // migrate secondary storages (NFS, S3, Swift) from host, s3, swift tables to image_store table
+ // migrate secondary storages NFS from host tables to image_store table
private void migrateSecondaryStorageToImageStore(Connection conn) {
PreparedStatement storeInsert = null;
PreparedStatement storeDetailInsert = null;
+ PreparedStatement nfsQuery = null;
+ PreparedStatement pstmt = null;
+ ResultSet rs = null;
+ ResultSet storeInfo = null;
+
+ s_logger.debug("Migrating secondary storage to image store");
+ boolean hasS3orSwift = false;
+ try {
+ s_logger.debug("Checking if we need to migrate NFS secondary storage to image store or staging store");
+ int numRows = 0;
+ pstmt = conn.prepareStatement("select count(*) from `cloud`.`s3`");
+ rs = pstmt.executeQuery();
+ if(rs.next()){
+ numRows = rs.getInt(1);
+ }
+ rs.close();
+ pstmt.close();
+ if (numRows > 0){
+ hasS3orSwift = true;
+ } else{
+ // check if there is swift storage
+ pstmt = conn.prepareStatement("select count(*) from `cloud`.`swift`");
+ rs = pstmt.executeQuery();
+ if(rs.next()){
+ numRows = rs.getInt(1);
+ }
+ rs.close();
+ pstmt.close();
+ if ( numRows > 0){
+ hasS3orSwift = true;
+ }
+ }
+
+ String store_role = "Image";
+ if ( hasS3orSwift){
+ store_role = "ImageCache";
+ }
+
+ s_logger.debug("Migrating NFS secondary storage to " + store_role + " store");
+
+ storeDetailInsert = conn
+ .prepareStatement("INSERT INTO `cloud`.`image_store_details` (store_id, name, value) values(?, ?, ?)");
+
+ // migrate NFS secondary storage, for nfs, keep previous host_id as the store_id
+ storeInsert = conn
+ .prepareStatement("INSERT INTO `cloud`.`image_store` (id, uuid, name, image_provider_name, protocol, url, data_center_id, scope, role, parent, total_size, created) values(?, ?, ?, 'NFS', 'nfs', ?, ?, 'ZONE', ?, ?, ?, ?)");
+ nfsQuery = conn
+ .prepareStatement("select id, uuid, url, data_center_id, parent, total_size, created from `cloud`.`host` where type = 'SecondaryStorage' and removed is null");
+ rs = nfsQuery.executeQuery();
+
+ while (rs.next()) {
+ Long nfs_id = rs.getLong("id");
+ String nfs_uuid = rs.getString("uuid");
+ String nfs_url = rs.getString("url");
+ String nfs_parent = rs.getString("parent");
+ int nfs_dcid = rs.getInt("data_center_id");
+ Long nfs_totalsize = rs.getObject("total_size") != null ? rs.getLong("total_size") : null;
+ Date nfs_created = rs.getDate("created");
+
+ // insert entry in image_store table and image_store_details
+ // table and store host_id and store_id mapping
+ storeInsert.setLong(1, nfs_id);
+ storeInsert.setString(2, nfs_uuid);
+ storeInsert.setString(3, nfs_uuid);
+ storeInsert.setString(4, nfs_url);
+ storeInsert.setInt(5, nfs_dcid);
+ storeInsert.setString(6, store_role);
+ storeInsert.setString(7, nfs_parent);
+ if (nfs_totalsize != null){
+ storeInsert.setLong(8, nfs_totalsize);
+ }
+ else{
+ storeInsert.setNull(8, Types.BIGINT);
+ }
+ storeInsert.setDate(9, nfs_created);
+ storeInsert.executeUpdate();
+ }
+ }
+ catch (SQLException e) {
+ String msg = "Unable to migrate secondary storages." + e.getMessage();
+ s_logger.error(msg);
+ throw new CloudRuntimeException(msg, e);
+ } finally {
+ try {
+ if (rs != null) {
+ rs.close();
+ }
+ if (storeInfo != null) {
+ storeInfo.close();
+ }
+
+ if (storeInsert != null) {
+ storeInsert.close();
+ }
+ if (storeDetailInsert != null) {
+ storeDetailInsert.close();
+ }
+ if (nfsQuery != null) {
+ nfsQuery.close();
+ }
+ if (pstmt != null) {
+ pstmt.close();
+ }
+ } catch (SQLException e) {
+ }
+ }
+ s_logger.debug("Completed migrating secondary storage to image store");
+ }
+
+ // migrate volume_host_ref to volume_store_ref
+ private void migrateVolumeHostRef(Connection conn) {
+ PreparedStatement volStoreInsert = null;
+ PreparedStatement volStoreUpdate = null;
+
+ s_logger.debug("Updating volume_store_ref table from volume_host_ref table");
+ try {
+ volStoreInsert = conn
+ .prepareStatement("INSERT INTO `cloud`.`volume_store_ref` (store_id, volume_id, zone_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, checksum, error_str, local_path, install_path, url, destroyed, update_count, ref_cnt, state) select host_id, volume_id, zone_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, checksum, error_str, local_path, install_path, url, destroyed, 0, 0, 'Allocated' from `cloud`.`volume_host_ref`");
+ int rowCount = volStoreInsert.executeUpdate();
+ s_logger.debug("Insert modified " + rowCount + " rows");
+
+ volStoreUpdate = conn.prepareStatement("update `cloud`.`volume_store_ref` set state = 'Ready' where download_state = 'DOWNLOADED'");
+ rowCount = volStoreUpdate.executeUpdate();
+ s_logger.debug("Update modified " + rowCount + " rows");
+ }
+ catch (SQLException e) {
+ String msg = "Unable to migrate volume_host_ref." + e.getMessage();
+ s_logger.error(msg);
+ throw new CloudRuntimeException(msg, e);
+ } finally {
+ try{
+ if (volStoreInsert != null) {
+ volStoreInsert.close();
+ }
+ if (volStoreUpdate != null) {
+ volStoreUpdate.close();
+ }
+ } catch (SQLException e) {
+ }
+ }
+ s_logger.debug("Completed updating volume_store_ref table from volume_host_ref table");
+ }
+
+ // migrate template_host_ref to template_store_ref
+ private void migrateTemplateHostRef(Connection conn) {
+ PreparedStatement tmplStoreInsert = null;
+ PreparedStatement tmplStoreUpdate = null;
+
+ s_logger.debug("Updating template_store_ref table from template_host_ref table");
+ try {
+ tmplStoreInsert = conn
+ .prepareStatement("INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, error_str, local_path, install_path, url, destroyed, is_copy, update_count, ref_cnt, store_role, state) select host_id, template_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, error_str, local_path, install_path, url, destroyed, is_copy, 0, 0, 'Image', 'Allocated' from `cloud`.`template_host_ref`");
+ int rowCount = tmplStoreInsert.executeUpdate();
+ s_logger.debug("Insert modified " + rowCount + " rows");
+
+ tmplStoreUpdate = conn.prepareStatement("update `cloud`.`template_store_ref` set state = 'Ready' where download_state = 'DOWNLOADED'");
+ rowCount = tmplStoreUpdate.executeUpdate();
+ s_logger.debug("Update modified " + rowCount + " rows");
+ }
+ catch (SQLException e) {
+ String msg = "Unable to migrate template_host_ref." + e.getMessage();
+ s_logger.error(msg);
+ throw new CloudRuntimeException(msg, e);
+ } finally {
+ try{
+ if (tmplStoreInsert != null) {
+ tmplStoreInsert.close();
+ }
+ if (tmplStoreUpdate != null) {
+ tmplStoreUpdate.close();
+ }
+ } catch (SQLException e) {
+ }
+ }
+ s_logger.debug("Completed updating template_store_ref table from template_host_ref table");
+ }
+
+ // migrate some entry contents of snapshots to snapshot_store_ref
+ private void migrateSnapshotStoreRef(Connection conn) {
+ PreparedStatement snapshotStoreInsert = null;
+
+ s_logger.debug("Updating snapshot_store_ref table from snapshots table");
+ try {
+ //Update all snapshots except KVM snapshots
+ snapshotStoreInsert = conn
+ .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, volume_id, update_count, ref_cnt, store_role, state) select sechost_id, id, created, size, prev_snap_id, CONCAT('snapshots', '/', account_id, '/', volume_id, '/', backup_snap_id), volume_id, 0, 0, 'Image', 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and hypervisor_type <> 'KVM' and sechost_id is not null and removed is null");
+ int rowCount = snapshotStoreInsert.executeUpdate();
+ s_logger.debug("Inserted " + rowCount + " snapshots into snapshot_store_ref");
+
+ //backsnap_id for KVM snapshots is complate path. CONCAT is not required
+ snapshotStoreInsert = conn
+ .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, volume_id, update_count, ref_cnt, store_role, state) select sechost_id, id, created, size, prev_snap_id, backup_snap_id, volume_id, 0, 0, 'Image', 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and hypervisor_type = 'KVM' and sechost_id is not null and removed is null");
+ rowCount = snapshotStoreInsert.executeUpdate();
+ s_logger.debug("Inserted " + rowCount + " KVM snapshots into snapshot_store_ref");
+ }
+ catch (SQLException e) {
+ String msg = "Unable to migrate snapshot_store_ref." + e.getMessage();
+ s_logger.error(msg);
+ throw new CloudRuntimeException(msg, e);
+ } finally {
+ try{
+ if (snapshotStoreInsert != null) {
+ snapshotStoreInsert.close();
+ }
+ } catch (SQLException e) {
+ }
+ }
+ s_logger.debug("Completed updating snapshot_store_ref table from snapshots table");
+ }
+
+ // migrate secondary storages S3 from s3 tables to image_store table
+ private void migrateS3ToImageStore(Connection conn) {
+ PreparedStatement storeInsert = null;
+ PreparedStatement storeDetailInsert = null;
PreparedStatement storeQuery = null;
PreparedStatement s3Query = null;
- PreparedStatement swiftQuery = null;
- PreparedStatement nfsQuery = null;
ResultSet rs = null;
ResultSet storeInfo = null;
Long storeId = null;
+ Map<Long, Long> s3_store_id_map = new HashMap<Long, Long>();
- s_logger.debug("Migrating secondary storage to image store");
+ s_logger.debug("Migrating S3 to image store");
try {
storeQuery = conn.prepareStatement("select id from `cloud`.`image_store` where uuid = ?");
storeDetailInsert = conn
.prepareStatement("INSERT INTO `cloud`.`image_store_details` (store_id, name, value) values(?, ?, ?)");
- /*
- // migrate S3 secondary storage
+ // migrate S3 to image_store
storeInsert = conn
.prepareStatement("INSERT INTO `cloud`.`image_store` (uuid, name, image_provider_name, protocol, scope, role, created) values(?, ?, 'S3', ?, 'REGION', 'Image', ?)");
s3Query = conn
@@ -1860,8 +2076,7 @@ public class Upgrade412to420 implements DbUpgrade {
String s3_endpoint = rs.getString("end_point");
String s3_bucket = rs.getString("bucket");
boolean s3_https = rs.getObject("https") != null ? (rs.getInt("https") == 0 ? false : true) : false;
- Integer s3_connectiontimeout = rs.getObject("connection_timeout") != null ? rs
- .getInt("connection_timeout") : null;
+ Integer s3_connectiontimeout = rs.getObject("connection_timeout") != null ? rs.getInt("connection_timeout") : null;
Integer s3_retry = rs.getObject("max_error_retry") != null ? rs.getInt("max_error_retry") : null;
Integer s3_sockettimeout = rs.getObject("socket_timeout") != null ? rs.getInt("socket_timeout") : null;
Date s3_created = rs.getDate("created");
@@ -1908,12 +2123,184 @@ public class Upgrade412to420 implements DbUpgrade {
}
s3_store_id_map.put(s3_id, storeId);
}
+ } catch (SQLException e) {
+ String msg = "Unable to migrate S3 secondary storages." + e.getMessage();
+ s_logger.error(msg);
+ throw new CloudRuntimeException(msg, e);
+ } finally {
+ try {
+ if (rs != null) {
+ rs.close();
+ }
+ if (storeInfo != null) {
+ storeInfo.close();
+ }
+
+ if (storeInsert != null) {
+ storeInsert.close();
+ }
+ if (storeDetailInsert != null) {
+ storeDetailInsert.close();
+ }
+ if (storeQuery != null) {
+ storeQuery.close();
+ }
+ if (s3Query != null) {
+ s3Query.close();
+ }
+ } catch (SQLException e) {
+ }
+ }
+
+ s_logger.debug("Migrating template_s3_ref to template_store_ref");
+ migrateTemplateS3Ref(conn, s3_store_id_map);
+
+ s_logger.debug("Migrating s3 backedup snapshots to snapshot_store_ref");
+ migrateSnapshotS3Ref(conn, s3_store_id_map);
+
+ s_logger.debug("Completed migrating S3 secondary storage to image store");
+ }
+
+ // migrate template_s3_ref to template_store_ref
+ private void migrateTemplateS3Ref(Connection conn, Map<Long, Long> s3StoreMap) {
+ PreparedStatement tmplStoreInsert = null;
+ PreparedStatement s3Query = null;
+ ResultSet rs = null;
+
+ s_logger.debug("Updating template_store_ref table from template_s3_ref table");
+ try{
+ tmplStoreInsert = conn
+ .prepareStatement("INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, download_pct, size, physical_size, download_state, local_path, install_path, update_count, ref_cnt, store_role, state) values(?, ?, ?, 100, ?, ?, 'DOWNLOADED', '?', '?', 0, 0, 'Image', 'Ready')");
+ s3Query = conn
+ .prepareStatement("select template_s3_ref.s3_id, template_s3_ref.template_id, template_s3_ref.created, template_s3_ref.size, template_s3_ref.physical_size, vm_template.account_id from `cloud`.`template_s3_ref`, `cloud`.`vm_template` where vm_template.id = template_s3_ref.template_id");
+ rs = s3Query.executeQuery();
+
+ while (rs.next()) {
+ Long s3_id = rs.getLong("s3_id");
+ Long s3_tmpl_id = rs.getLong("template_id");
+ Date s3_created = rs.getDate("created");
+ Long s3_size = rs.getObject("size") != null ? rs.getLong("size") : null;
+ Long s3_psize = rs.getObject("physical_size") != null ? rs.getLong("physical_size") : null;
+ Long account_id = rs.getLong("account_id");
+
+ tmplStoreInsert.setLong(1, s3StoreMap.get(s3_id));
+ tmplStoreInsert.setLong(2, s3_tmpl_id);
+ tmplStoreInsert.setDate(3, s3_created);
+ if (s3_size != null) {
+ tmplStoreInsert.setLong(4, s3_size);
+ } else {
+ tmplStoreInsert.setNull(4, Types.BIGINT);
+ }
+ if (s3_psize != null) {
+ tmplStoreInsert.setLong(5, s3_psize);
+ } else {
+ tmplStoreInsert.setNull(5, Types.BIGINT);
+ }
+ String path = "template/tmpl/" + account_id + "/" + s3_tmpl_id;
+ tmplStoreInsert.setString(6, path);
+ tmplStoreInsert.setString(7, path);
+ tmplStoreInsert.executeUpdate();
+ }
+ }
+ catch (SQLException e) {
+ String msg = "Unable to migrate template_s3_ref." + e.getMessage();
+ s_logger.error(msg);
+ throw new CloudRuntimeException(msg, e);
+ } finally {
+ try {
+ if (rs != null) {
+ rs.close();
+ }
+ if (tmplStoreInsert != null) {
+ tmplStoreInsert.close();
+ }
+ if (s3Query != null) {
+ s3Query.close();
+ }
+ } catch (SQLException e) {
+ }
+ }
+ s_logger.debug("Completed migrating template_s3_ref table.");
+ }
+
+ // migrate some entry contents of snapshots to snapshot_store_ref
+ private void migrateSnapshotS3Ref(Connection conn, Map<Long, Long> s3StoreMap) {
+ PreparedStatement snapshotStoreInsert = null;
+ PreparedStatement s3Query = null;
+ ResultSet rs = null;
+
+
+ s_logger.debug("Updating snapshot_store_ref table from snapshots table for s3");
+ try {
+ snapshotStoreInsert = conn
+ .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, volume_id, update_count, ref_cnt, store_role, state) values(?, ?, ?, ?, ?, ?, ?, 0, 0, 'Image', 'Ready')");
+ s3Query = conn
+ .prepareStatement("select s3_id, id, created, size, prev_snap_id, CONCAT('snapshots', '/', account_id, '/', volume_id, '/', backup_snap_id), volume_id, 0, 0, 'Image', 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and hypervisor_type <> 'KVM' and s3_id is not null and removed is null");
+ rs = s3Query.executeQuery();
+
+ while (rs.next()) {
+ Long s3_id = rs.getLong("s3_id");
+ Long snapshot_id = rs.getLong("id");
+ Date s3_created = rs.getDate("created");
+ Long s3_size = rs.getObject("size") != null ? rs.getLong("size") : null;
+ Long s3_prev_id = rs.getObject("prev_snap_id") != null ? rs.getLong("prev_snap_id") : null;
+ String install_path = rs.getString(6);
+ Long s3_vol_id = rs.getLong("volume_id");
+
+ snapshotStoreInsert.setLong(1, s3StoreMap.get(s3_id));
+ snapshotStoreInsert.setLong(2, snapshot_id);
+ snapshotStoreInsert.setDate(3, s3_created);
+ if (s3_size != null) {
+ snapshotStoreInsert.setLong(4, s3_size);
+ } else {
+ snapshotStoreInsert.setNull(4, Types.BIGINT);
+ }
+ if (s3_prev_id != null) {
+ snapshotStoreInsert.setLong(5, s3_prev_id);
+ } else {
+ snapshotStoreInsert.setNull(5, Types.BIGINT);
+ }
+ snapshotStoreInsert.setString(6, install_path);
+ snapshotStoreInsert.setLong(7, s3_vol_id);
+ snapshotStoreInsert.executeUpdate();
+ }
+ }
+ catch (SQLException e) {
+ String msg = "Unable to migrate s3 backedup snapshots to snapshot_store_ref." + e.getMessage();
+ s_logger.error(msg);
+ throw new CloudRuntimeException(msg, e);
+ } finally {
+ try{
+ if (snapshotStoreInsert != null) {
+ snapshotStoreInsert.close();
+ }
+ } catch (SQLException e) {
+ }
+ }
+ s_logger.debug("Completed updating snapshot_store_ref table from s3 snapshots entries");
+ }
+
+ // migrate secondary storages Swift from swift tables to image_store table
+ private void migrateSwiftToImageStore(Connection conn) {
+ PreparedStatement storeInsert = null;
+ PreparedStatement storeDetailInsert = null;
+ PreparedStatement storeQuery = null;
+ PreparedStatement swiftQuery = null;
+ ResultSet rs = null;
+ ResultSet storeInfo = null;
+ Long storeId = null;
+ Map<Long, Long> swift_store_id_map = new HashMap<Long, Long>();
+
+ s_logger.debug("Migrating Swift to image store");
+ try {
+ storeQuery = conn.prepareStatement("select id from `cloud`.`image_store` where uuid = ?");
+ storeDetailInsert = conn
+ .prepareStatement("INSERT INTO `cloud`.`image_store_details` (store_id, name, value) values(?, ?, ?)");
// migrate SWIFT secondary storage
storeInsert = conn
.prepareStatement("INSERT INTO `cloud`.`image_store` (uuid, name, image_provider_name, protocol, url, scope, role, created) values(?, ?, 'Swift', 'http', ?, 'REGION', 'Image', ?)");
- swiftQuery = conn
- .prepareStatement("select id, uuid, url, account, username, key, created from `cloud`.`swift`");
+ swiftQuery = conn.prepareStatement("select id, uuid, url, account, username, swift.key, created from `cloud`.`swift`");
rs = swiftQuery.executeQuery();
while (rs.next()) {
@@ -1955,52 +2342,9 @@ public class Upgrade412to420 implements DbUpgrade {
}
swift_store_id_map.put(swift_id, storeId);
}
- */
-
- // migrate NFS secondary storage, for nfs, keep previous host_id as the store_id
- storeInsert = conn
- .prepareStatement("INSERT INTO `cloud`.`image_store` (id, uuid, name, image_provider_name, protocol, url, data_center_id, scope, role, parent, total_size, created) values(?, ?, ?, 'NFS', 'nfs', ?, ?, 'ZONE', 'Image', ?, ?, ?)");
- nfsQuery = conn
- .prepareStatement("select id, uuid, url, data_center_id, parent, total_size, created from `cloud`.`host` where type = 'SecondaryStorage' and removed is null");
- rs = nfsQuery.executeQuery();
-
- while (rs.next()) {
- Long nfs_id = rs.getLong("id");
- String nfs_uuid = rs.getString("uuid");
- String nfs_url = rs.getString("url");
- String nfs_parent = rs.getString("parent");
- int nfs_dcid = rs.getInt("data_center_id");
- Long nfs_totalsize = rs.getObject("total_size") != null ? rs.getLong("total_size") : null;
- Date nfs_created = rs.getDate("created");
-
- // insert entry in image_store table and image_store_details
- // table and store host_id and store_id mapping
- storeInsert.setLong(1, nfs_id);
- storeInsert.setString(2, nfs_uuid);
- storeInsert.setString(3, nfs_uuid);
- storeInsert.setString(4, nfs_url);
- storeInsert.setInt(5, nfs_dcid);
- storeInsert.setString(6, nfs_parent);
- if (nfs_totalsize != null){
- storeInsert.setLong(7, nfs_totalsize);
- }
- else{
- storeInsert.setNull(7, Types.BIGINT);
- }
- storeInsert.setDate(8, nfs_created);
- storeInsert.executeUpdate();
-
- storeQuery.setString(1, nfs_uuid);
- storeInfo = storeQuery.executeQuery();
- if (storeInfo.next()) {
- storeId = storeInfo.getLong("id");
- }
-
- //host_store_id_map.put(nfs_id, storeId);
- }
}
catch (SQLException e) {
- String msg = "Unable to migrate secondary storages." + e.getMessage();
+ String msg = "Unable to migrate swift secondary storages." + e.getMessage();
s_logger.error(msg);
throw new CloudRuntimeException(msg, e);
} finally {
@@ -2024,117 +2368,123 @@ public class Upgrade412to420 implements DbUpgrade {
if (swiftQuery != null) {
swiftQuery.close();
}
- if (s3Query != null) {
- s3Query.close();
- }
- if (nfsQuery != null) {
- nfsQuery.close();
- }
} catch (SQLException e) {
}
}
- s_logger.debug("Completed migrating secondary storage to image store");
- }
- // migrate volume_host_ref to volume_store_ref
- private void migrateVolumeHostRef(Connection conn) {
- PreparedStatement volStoreInsert = null;
- PreparedStatement volStoreUpdate = null;
+ s_logger.debug("Migrating template_swift_ref to template_store_ref");
+ migrateTemplateSwiftRef(conn, swift_store_id_map);
- s_logger.debug("Updating volume_store_ref table from volume_host_ref table");
- try {
- volStoreInsert = conn
- .prepareStatement("INSERT INTO `cloud`.`volume_store_ref` (store_id, volume_id, zone_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, checksum, error_str, local_path, install_path, url, destroyed, update_count, ref_cnt, state) select host_id, volume_id, zone_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, checksum, error_str, local_path, install_path, url, destroyed, 0, 0, 'Allocated' from `cloud`.`volume_host_ref`");
- int rowCount = volStoreInsert.executeUpdate();
- s_logger.debug("Insert modified " + rowCount + " rows");
+ s_logger.debug("Migrating swift backedup snapshots to snapshot_store_ref");
+ migrateSnapshotSwiftRef(conn, swift_store_id_map);
- volStoreUpdate = conn.prepareStatement("update `cloud`.`volume_store_ref` set state = 'Ready' where download_state = 'DOWNLOADED'");
- rowCount = volStoreUpdate.executeUpdate();
- s_logger.debug("Update modified " + rowCount + " rows");
- }
- catch (SQLException e) {
- String msg = "Unable to migrate volume_host_ref." + e.getMessage();
- s_logger.error(msg);
- throw new CloudRuntimeException(msg, e);
- } finally {
- try{
- if (volStoreInsert != null) {
- volStoreInsert.close();
- }
- if (volStoreUpdate != null) {
- volStoreUpdate.close();
- }
- } catch (SQLException e) {
- }
- }
- s_logger.debug("Completed updating volume_store_ref table from volume_host_ref table");
+ s_logger.debug("Completed migrating Swift secondary storage to image store");
}
- // migrate template_host_ref to template_store_ref
- private void migrateTemplateHostRef(Connection conn) {
+ // migrate template_s3_ref to template_store_ref
+ private void migrateTemplateSwiftRef(Connection conn, Map<Long, Long> swiftStoreMap) {
PreparedStatement tmplStoreInsert = null;
- PreparedStatement tmplStoreUpdate = null;
+ PreparedStatement s3Query = null;
+ ResultSet rs = null;
- s_logger.debug("Updating template_store_ref table from template_host_ref table");
+ s_logger.debug("Updating template_store_ref table from template_swift_ref table");
try {
tmplStoreInsert = conn
- .prepareStatement("INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, error_str, local_path, install_path, url, destroyed, is_copy, update_count, ref_cnt, store_role, state) select host_id, template_id, created, last_updated, job_id, download_pct, size, physical_size, download_state, error_str, local_path, install_path, url, destroyed, is_copy, 0, 0, 'Image', 'Allocated' from `cloud`.`template_host_ref`");
- int rowCount = tmplStoreInsert.executeUpdate();
- s_logger.debug("Insert modified " + rowCount + " rows");
+ .prepareStatement("INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, download_pct, size, physical_size, download_state, local_path, install_path, update_count, ref_cnt, store_role, state) values(?, ?, ?, 100, ?, ?, 'DOWNLOADED', '?', '?', 0, 0, 'Image', 'Ready')");
+ s3Query = conn.prepareStatement("select swift_id, template_id, created, path, size, physical_size from `cloud`.`template_swift_ref`");
+ rs = s3Query.executeQuery();
- tmplStoreUpdate = conn.prepareStatement("update `cloud`.`template_store_ref` set state = 'Ready' where download_state = 'DOWNLOADED'");
- rowCount = tmplStoreUpdate.executeUpdate();
- s_logger.debug("Update modified " + rowCount + " rows");
- }
- catch (SQLException e) {
- String msg = "Unable to migrate template_host_ref." + e.getMessage();
+ while (rs.next()) {
+ Long swift_id = rs.getLong("swift_id");
+ Long tmpl_id = rs.getLong("template_id");
+ Date created = rs.getDate("created");
+ String path = rs.getString("path");
+ Long size = rs.getObject("size") != null ? rs.getLong("size") : null;
+ Long psize = rs.getObject("physical_size") != null ? rs.getLong("physical_size") : null;
+
+ tmplStoreInsert.setLong(1, swiftStoreMap.get(swift_id));
+ tmplStoreInsert.setLong(2, tmpl_id);
+ tmplStoreInsert.setDate(3, created);
+ if (size != null) {
+ tmplStoreInsert.setLong(4, size);
+ } else {
+ tmplStoreInsert.setNull(4, Types.BIGINT);
+ }
+ if (psize != null) {
+ tmplStoreInsert.setLong(5, psize);
+ } else {
+ tmplStoreInsert.setNull(5, Types.BIGINT);
+ }
+ tmplStoreInsert.setString(6, path);
+ tmplStoreInsert.setString(7, path);
+ tmplStoreInsert.executeUpdate();
+ }
+ } catch (SQLException e) {
+ String msg = "Unable to migrate template_swift_ref." + e.getMessage();
s_logger.error(msg);
throw new CloudRuntimeException(msg, e);
} finally {
- try{
+ try {
+ if (rs != null) {
+ rs.close();
+ }
if (tmplStoreInsert != null) {
tmplStoreInsert.close();
}
- if (tmplStoreUpdate != null) {
- tmplStoreUpdate.close();
+ if (s3Query != null) {
+ s3Query.close();
}
} catch (SQLException e) {
}
}
- s_logger.debug("Completed updating template_store_ref table from template_host_ref table");
+ s_logger.debug("Completed migrating template_swift_ref table.");
}
// migrate some entry contents of snapshots to snapshot_store_ref
- private void migrateSnapshotStoreRef(Connection conn) {
+ private void migrateSnapshotSwiftRef(Connection conn, Map<Long, Long> swiftStoreMap) {
PreparedStatement snapshotStoreInsert = null;
+ PreparedStatement s3Query = null;
+ ResultSet rs = null;
- s_logger.debug("Updating snapshot_store_ref table from snapshots table");
+ s_logger.debug("Updating snapshot_store_ref table from snapshots table for swift");
try {
- //Update all snapshots except KVM snapshots
snapshotStoreInsert = conn
- .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, volume_id, update_count, ref_cnt, store_role, state) select sechost_id, id, created, size, prev_snap_id, CONCAT('snapshots', '/', account_id, '/', volume_id, '/', backup_snap_id), volume_id, 0, 0, 'Image', 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and hypervisor_type <> 'KVM' and sechost_id is not null and removed is null");
- int rowCount = snapshotStoreInsert.executeUpdate();
- s_logger.debug("Inserted " + rowCount + " snapshots into snapshot_store_ref");
+ .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, volume_id, update_count, ref_cnt, store_role, state) values(?, ?, ?, ?, ?, ?, ?, 0, 0, 'Image', 'Ready')");
+ s3Query = conn
+ .prepareStatement("select swift_id, id, created, size, prev_snap_id, CONCAT('snapshots', '/', account_id, '/', volume_id, '/', backup_snap_id), volume_id, 0, 0, 'Image', 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and hypervisor_type <> 'KVM' and swift_id is not null and removed is null");
+ rs = s3Query.executeQuery();
- //backsnap_id for KVM snapshots is complate path. CONCAT is not required
- snapshotStoreInsert = conn
- .prepareStatement("INSERT INTO `cloud`.`snapshot_store_ref` (store_id, snapshot_id, created, size, parent_snapshot_id, install_path, volume_id, update_count, ref_cnt, store_role, state) select sechost_id, id, created, size, prev_snap_id, backup_snap_id, volume_id, 0, 0, 'Image', 'Ready' from `cloud`.`snapshots` where status = 'BackedUp' and hypervisor_type = 'KVM' and sechost_id is not null and removed is null");
- rowCount = snapshotStoreInsert.executeUpdate();
- s_logger.debug("Inserted " + rowCount + " KVM snapshots into snapshot_store_ref");
- }
- catch (SQLException e) {
- String msg = "Unable to migrate snapshot_store_ref." + e.getMessage();
+ while (rs.next()) {
+ Long swift_id = rs.getLong("swift_id");
+ Long snapshot_id = rs.getLong("id");
+ Date created = rs.getDate("created");
+ Long size = rs.getLong("size");
+ Long prev_id = rs.getLong("prev_snap_id");
+ String install_path = rs.getString(6);
+ Long vol_id = rs.getLong("volume_id");
+
+ snapshotStoreInsert.setLong(1, swiftStoreMap.get(swift_id));
+ snapshotStoreInsert.setLong(2, snapshot_id);
+ snapshotStoreInsert.setDate(3, created);
+ snapshotStoreInsert.setLong(4, size);
+ snapshotStoreInsert.setLong(5, prev_id);
+ snapshotStoreInsert.setString(6, install_path);
+ snapshotStoreInsert.setLong(7, vol_id);
+ snapshotStoreInsert.executeUpdate();
+ }
+ } catch (SQLException e) {
+ String msg = "Unable to migrate swift backedup snapshots to snapshot_store_ref." + e.getMessage();
s_logger.error(msg);
throw new CloudRuntimeException(msg, e);
} finally {
- try{
+ try {
if (snapshotStoreInsert != null) {
snapshotStoreInsert.close();
}
} catch (SQLException e) {
}
}
- s_logger.debug("Completed updating snapshot_store_ref table from snapshots table");
+ s_logger.debug("Completed updating snapshot_store_ref table from swift snapshots entries");
}
private void fixNiciraKeys(Connection conn) {