You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ds...@apache.org on 2016/02/18 00:10:37 UTC
[02/33] incubator-geode git commit: GEODE-657: Prevent two backups
from happening at the same time
GEODE-657: Prevent two backups from happening at the same time
Adding a dlock to prevent two backups from occuring at the same time.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/2b2e619e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/2b2e619e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/2b2e619e
Branch: refs/heads/feature/GEODE-831
Commit: 2b2e619e35a44c6e5f1f9881b5fba46d884a3934
Parents: 401280d
Author: Barry Oglesby <bo...@pivotal.io>
Authored: Thu Dec 10 16:41:14 2015 -0800
Committer: Dan Smith <up...@apache.org>
Committed: Thu Feb 11 16:42:27 2016 -0800
----------------------------------------------------------------------
.../internal/AdminDistributedSystemImpl.java | 25 +++++---
.../admin/internal/BackupDataStoreHelper.java | 60 ++++++++++++++++++++
.../admin/internal/BackupDataStoreResult.java | 41 +++++++++++++
.../gemfire/internal/i18n/LocalizedStrings.java | 1 +
.../internal/beans/DistributedSystemBridge.java | 35 ++++++------
5 files changed, 136 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2b2e619e/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/AdminDistributedSystemImpl.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/AdminDistributedSystemImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/AdminDistributedSystemImpl.java
index 5709fce..4e84936 100755
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/AdminDistributedSystemImpl.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/AdminDistributedSystemImpl.java
@@ -2421,31 +2421,38 @@ implements com.gemstone.gemfire.admin.AdminDistributedSystem,
public static BackupStatus backupAllMembers(DM dm, File targetDir, File baselineDir)
throws AdminException {
+ BackupStatus status = null;
+ if (BackupDataStoreHelper.obtainLock(dm)) {
+ try {
Set<PersistentID> missingMembers = getMissingPersistentMembers(dm);
Set recipients = dm.getOtherDistributionManagerIds();
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
targetDir = new File(targetDir, format.format(new Date()));
- FlushToDiskRequest.send(dm, recipients);
- Map<DistributedMember, Set<PersistentID>> existingDataStores
- = PrepareBackupRequest.send(dm, recipients);
- Map<DistributedMember, Set<PersistentID>> successfulMembers
- = FinishBackupRequest.send(dm, recipients, targetDir, baselineDir);
+ BackupDataStoreResult result = BackupDataStoreHelper.backupAllMembers(
+ dm, recipients, targetDir, baselineDir);
// It's possible that when calling getMissingPersistentMembers, some members are
// still creating/recovering regions, and at FinishBackupRequest.send, the
// regions at the members are ready. Logically, since the members in successfulMembers
// should override the previous missingMembers
- for(Set<PersistentID> onlineMembersIds : successfulMembers.values()) {
+ for(Set<PersistentID> onlineMembersIds : result.getSuccessfulMembers().values()) {
missingMembers.removeAll(onlineMembersIds);
}
- existingDataStores.keySet().removeAll(successfulMembers.keySet());
- for(Set<PersistentID> lostMembersIds : existingDataStores.values()) {
+ result.getExistingDataStores().keySet().removeAll(result.getSuccessfulMembers().keySet());
+ for(Set<PersistentID> lostMembersIds : result.getExistingDataStores().values()) {
missingMembers.addAll(lostMembersIds);
}
- return new BackupStatusImpl(successfulMembers, missingMembers);
+ status = new BackupStatusImpl(result.getSuccessfulMembers(), missingMembers);
+ } finally {
+ BackupDataStoreHelper.releaseLock(dm);
+ }
+ } else {
+ throw new AdminException(LocalizedStrings.DistributedSystem_BACKUP_ALREADY_IN_PROGRESS.toLocalizedString());
+ }
+ return status;
}
public Map<DistributedMember, Set<PersistentID>> compactAllDiskStores() throws AdminException {
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2b2e619e/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreHelper.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreHelper.java b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreHelper.java
new file mode 100644
index 0000000..26a70a8
--- /dev/null
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreHelper.java
@@ -0,0 +1,60 @@
+package com.gemstone.gemfire.admin.internal;
+
+import java.io.File;
+import java.util.Map;
+import java.util.Set;
+
+import com.gemstone.gemfire.cache.persistence.PersistentID;
+import com.gemstone.gemfire.distributed.DistributedLockService;
+import com.gemstone.gemfire.distributed.DistributedMember;
+import com.gemstone.gemfire.distributed.internal.DM;
+import com.gemstone.gemfire.internal.Assert;
+
+public class BackupDataStoreHelper {
+
+ private static String LOCK_SERVICE_NAME = BackupDataStoreHelper.class.getSimpleName();
+
+ private static String LOCK_NAME = LOCK_SERVICE_NAME + "_token";
+
+ private static Object LOCK_SYNC = new Object();
+
+ @SuppressWarnings("rawtypes")
+ public static BackupDataStoreResult backupAllMembers(
+ DM dm, Set recipients, File targetDir, File baselineDir) {
+ FlushToDiskRequest.send(dm, recipients);
+
+ boolean abort= true;
+ Map<DistributedMember, Set<PersistentID>> successfulMembers;
+ Map<DistributedMember, Set<PersistentID>> existingDataStores;
+ try {
+ existingDataStores = PrepareBackupRequest.send(dm, recipients);
+ abort = false;
+ } finally {
+ successfulMembers = FinishBackupRequest.send(dm, recipients, targetDir, baselineDir, abort);
+ }
+ return new BackupDataStoreResult(existingDataStores, successfulMembers);
+ }
+
+ private static DistributedLockService getLockService(DM dm) {
+ DistributedLockService dls = DistributedLockService.getServiceNamed(LOCK_SERVICE_NAME);
+ if (dls == null) {
+ synchronized (LOCK_SYNC) {
+ dls = DistributedLockService.getServiceNamed(LOCK_SERVICE_NAME);
+ if (dls == null) {
+ // Create the DistributedLockService
+ dls = DistributedLockService.create(LOCK_SERVICE_NAME, dm.getSystem());
+ }
+ }
+ }
+ Assert.assertTrue(dls != null);
+ return dls;
+ }
+
+ public static boolean obtainLock(DM dm) {
+ return getLockService(dm).lock(LOCK_NAME, 0, -1);
+ }
+
+ public static void releaseLock(DM dm) {
+ getLockService(dm).unlock(LOCK_NAME);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2b2e619e/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreResult.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreResult.java b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreResult.java
new file mode 100644
index 0000000..f8a8911
--- /dev/null
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreResult.java
@@ -0,0 +1,41 @@
+package com.gemstone.gemfire.admin.internal;
+
+import java.util.Map;
+import java.util.Set;
+
+import com.gemstone.gemfire.cache.persistence.PersistentID;
+import com.gemstone.gemfire.distributed.DistributedMember;
+
+public class BackupDataStoreResult {
+
+ private Map<DistributedMember, Set<PersistentID>> existingDataStores;
+
+ private Map<DistributedMember, Set<PersistentID>> successfulMembers;
+
+ public BackupDataStoreResult(
+ Map<DistributedMember, Set<PersistentID>> existingDataStores,
+ Map<DistributedMember, Set<PersistentID>> successfulMembers) {
+ this.existingDataStores = existingDataStores;
+ this.successfulMembers = successfulMembers;
+ }
+
+ public Map<DistributedMember, Set<PersistentID>> getExistingDataStores() {
+ return this.existingDataStores;
+ }
+
+ public Map<DistributedMember, Set<PersistentID>> getSuccessfulMembers() {
+ return this.successfulMembers;
+ }
+
+ public String toString() {
+ return new StringBuilder()
+ .append(getClass().getSimpleName())
+ .append("[")
+ .append("existingDataStores=")
+ .append(this.existingDataStores)
+ .append("; successfulMembers=")
+ .append(this.successfulMembers)
+ .append("]")
+ .toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2b2e619e/gemfire-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
index f6edb9b..8147718 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
@@ -2144,6 +2144,7 @@ public class LocalizedStrings extends ParentLocalizedStrings {
public static final StringId AUTH_PEER_AUTHENTICATION_MISSING_CREDENTIALS = new StringId(6610, "Failed to find credentials from [{0}] using Authenticator [{1}]");
public static final StringId AUTH_FAILED_TO_ACQUIRE_AUTHINITIALIZE_INSTANCE = new StringId(6611, "AuthInitialize instance could not be obtained");
public static final StringId AUTH_FAILED_TO_OBTAIN_CREDENTIALS_IN_0_USING_AUTHINITIALIZE_1_2 = new StringId(6612, "Failed to obtain credentials using AuthInitialize [{1}]. {2}");
+ public static final StringId DistributedSystem_BACKUP_ALREADY_IN_PROGRESS = new StringId(6613, "A backup is already in progress.");
/** Testing strings, messageId 90000-99999 **/
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2b2e619e/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/beans/DistributedSystemBridge.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/beans/DistributedSystemBridge.java b/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/beans/DistributedSystemBridge.java
index 5034f60..9427831 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/beans/DistributedSystemBridge.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/beans/DistributedSystemBridge.java
@@ -44,9 +44,8 @@ import javax.management.ObjectName;
import org.apache.logging.log4j.Logger;
-import com.gemstone.gemfire.admin.internal.FinishBackupRequest;
-import com.gemstone.gemfire.admin.internal.FlushToDiskRequest;
-import com.gemstone.gemfire.admin.internal.PrepareBackupRequest;
+import com.gemstone.gemfire.admin.internal.BackupDataStoreHelper;
+import com.gemstone.gemfire.admin.internal.BackupDataStoreResult;
import com.gemstone.gemfire.cache.persistence.PersistentID;
import com.gemstone.gemfire.distributed.DistributedMember;
import com.gemstone.gemfire.distributed.internal.DM;
@@ -59,6 +58,7 @@ import com.gemstone.gemfire.internal.admin.remote.RevokePersistentIDRequest;
import com.gemstone.gemfire.internal.admin.remote.ShutdownAllRequest;
import com.gemstone.gemfire.internal.cache.GemFireCacheImpl;
import com.gemstone.gemfire.internal.cache.persistence.PersistentMemberPattern;
+import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
import com.gemstone.gemfire.internal.logging.LogService;
import com.gemstone.gemfire.internal.logging.log4j.LocalizedMessage;
import com.gemstone.gemfire.management.CacheServerMXBean;
@@ -521,6 +521,7 @@ public class DistributedSystemBridge {
*/
public DiskBackupStatus backupAllMembers(String targetDirPath, String baselineDirPath)
throws Exception {
+ if (BackupDataStoreHelper.obtainLock(dm)) {
try {
if(targetDirPath == null || targetDirPath.isEmpty()){
@@ -540,26 +541,21 @@ public class DistributedSystemBridge {
Set<PersistentID> missingMembers = MissingPersistentIDsRequest.send(dm);
Set recipients = dm.getOtherDistributionManagerIds();
- FlushToDiskRequest.send(dm, recipients);
+ BackupDataStoreResult result = BackupDataStoreHelper.backupAllMembers(dm, recipients, targetDir, baselineDir);
- Map<DistributedMember, Set<PersistentID>> existingDataStores = PrepareBackupRequest
- .send(dm, recipients);
- Map<DistributedMember, Set<PersistentID>> successfulMembers = FinishBackupRequest
- .send(dm, recipients, targetDir, baselineDir);
-
- Iterator<DistributedMember> it = successfulMembers.keySet().iterator();
+ Iterator<DistributedMember> it = result.getSuccessfulMembers().keySet().iterator();
Map<String, String[]> backedUpDiskStores = new HashMap<String, String[]>();
while (it.hasNext()) {
DistributedMember member = it.next();
- Set<PersistentID> setOfDisk = successfulMembers.get(member);
+ Set<PersistentID> setOfDisk = result.getSuccessfulMembers().get(member);
String[] setOfDiskStr = new String[setOfDisk.size()];
int j = 0;
for (PersistentID id : setOfDisk) {
setOfDiskStr[j] = id.getDirectory();
j++;
}
- backedUpDiskStores.put(member.getName(), setOfDiskStr);
+ backedUpDiskStores.put(member.getId(), setOfDiskStr);
}
// It's possible that when calling getMissingPersistentMembers, some
@@ -569,17 +565,17 @@ public class DistributedSystemBridge {
// regions at the members are ready. Logically, since the members in
// successfulMembers
// should override the previous missingMembers
- for (Set<PersistentID> onlineMembersIds : successfulMembers.values()) {
+ for (Set<PersistentID> onlineMembersIds : result.getSuccessfulMembers().values()) {
missingMembers.removeAll(onlineMembersIds);
}
- existingDataStores.keySet().removeAll(successfulMembers.keySet());
+ result.getExistingDataStores().keySet().removeAll(result.getSuccessfulMembers().keySet());
String[] setOfMissingDiskStr = null;
- if (existingDataStores.size() > 0) {
- setOfMissingDiskStr = new String[existingDataStores.size()];
+ if (result.getExistingDataStores().size() > 0) {
+ setOfMissingDiskStr = new String[result.getExistingDataStores().size()];
int j = 0;
- for (Set<PersistentID> lostMembersIds : existingDataStores.values()) {
+ for (Set<PersistentID> lostMembersIds : result.getExistingDataStores().values()) {
for (PersistentID id : lostMembersIds) {
setOfMissingDiskStr[j] = id.getDirectory();
j++;
@@ -595,6 +591,11 @@ public class DistributedSystemBridge {
} catch (Exception e) {
throw new Exception(e.getLocalizedMessage());
+ } finally {
+ BackupDataStoreHelper.releaseLock(dm);
+ }
+ } else {
+ throw new Exception(LocalizedStrings.DistributedSystem_BACKUP_ALREADY_IN_PROGRESS.toLocalizedString());
}
}