You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by st...@apache.org on 2024/01/22 15:49:38 UTC
(jackrabbit-oak) 01/01: OAK-10622 : oak.documentMK.reuseDelayAfterRecoverSecs and recoverTime added
This is an automated email from the ASF dual-hosted git repository.
stefanegli pushed a commit to branch OAK-10622
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 4e81edea1fd857b2a73bd2bbded8060a79a66826
Author: Stefan Egli <st...@apache.org>
AuthorDate: Mon Jan 22 16:49:24 2024 +0100
OAK-10622 : oak.documentMK.reuseDelayAfterRecoverSecs and recoverTime added
---
.../oak/plugins/document/ClusterNodeInfo.java | 54 ++++++++++++++++++++--
.../oak/plugins/document/RecoveryLock.java | 2 +
.../oak/plugins/document/ClusterNodeInfoTest.java | 27 ++++++++++-
3 files changed, 79 insertions(+), 4 deletions(-)
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java
index d53e1822f9..6c1d8156a4 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java
@@ -87,6 +87,11 @@ public class ClusterNodeInfo {
*/
public static final String LEASE_END_KEY = "leaseEnd";
+ /**
+ * The time a recover was done, if the last shutdown required a recover - not set otherwise.
+ */
+ public static final String RECOVER_TIME_KEY = "recoverTime";
+
/**
* The start time.
*/
@@ -229,6 +234,12 @@ public class ClusterNodeInfo {
/** OAK-3399 : max number of times we're doing a 1sec retry loop just before declaring lease failure **/
private static final int MAX_RETRY_SLEEPS_BEFORE_LEASE_FAILURE = 5;
+ /** OAK-10622 : seconds to prevent reuse of clusterId if it crashed and thus required a recover */
+ private static final int DEFAULT_REUSE_AFTER_RECOVER_SECS = SystemPropertySupplier.create("oak.documentMK.reuseDelayAfterRecoverSecs", -1)
+ .loggingTo(LOG).validateWith(value -> value >= -1)
+ .formatSetMessage((name, value) -> String.format("Reuse delay after recover set to (secs): %ss (using system property %s)", name, value)).get();
+ public static final int DEFAULT_REUSE_AFTER_RECOVER_MILLIS = 1000 * DEFAULT_REUSE_AFTER_RECOVER_SECS;
+
/**
* The Oak version.
*/
@@ -457,6 +468,28 @@ public class ClusterNodeInfo {
String instanceId,
int configuredClusterId,
boolean invisible) {
+ return getInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId,
+ invisible, DEFAULT_REUSE_AFTER_RECOVER_MILLIS);
+ }
+
+ /**
+ * Get or create a cluster node info instance for the store.
+ *
+ * @param store the document store (for the lease)
+ * @param recoveryHandler the recovery handler to call for a clusterId with
+ * an expired lease.
+ * @param machineId the machine id (null for MAC address)
+ * @param instanceId the instance id (null for current working directory)
+ * @param configuredClusterId the configured cluster id (or 0 for dynamic assignment)
+ * @return the cluster node info
+ */
+ public static ClusterNodeInfo getInstance(DocumentStore store,
+ RecoveryHandler recoveryHandler,
+ String machineId,
+ String instanceId,
+ int configuredClusterId,
+ boolean invisible,
+ long reuseAfterRecoverMillis) {
// defaults for machineId and instanceID
if (machineId == null) {
machineId = MACHINE_ID;
@@ -469,7 +502,7 @@ public class ClusterNodeInfo {
for (int i = 0; i < retries; i++) {
Map.Entry<ClusterNodeInfo, Long> suggestedClusterNode =
createInstance(store, recoveryHandler, machineId,
- instanceId, configuredClusterId, i == 0, invisible);
+ instanceId, configuredClusterId, i == 0, invisible, reuseAfterRecoverMillis);
ClusterNodeInfo clusterNode = suggestedClusterNode.getKey();
Long currentStartTime = suggestedClusterNode.getValue();
String key = String.valueOf(clusterNode.id);
@@ -477,6 +510,7 @@ public class ClusterNodeInfo {
update.set(MACHINE_ID_KEY, clusterNode.machineId);
update.set(INSTANCE_ID_KEY, clusterNode.instanceId);
update.set(LEASE_END_KEY, clusterNode.leaseEndTime);
+ update.set(RECOVER_TIME_KEY, null);
update.set(START_TIME_KEY, clusterNode.startTime);
update.set(INFO_KEY, clusterNode.toString());
update.set(STATE, ACTIVE.name());
@@ -523,7 +557,8 @@ public class ClusterNodeInfo {
String instanceId,
int configuredClusterId,
boolean waitForLease,
- boolean invisible) {
+ boolean invisible,
+ long reuseAfterRecoverMillis) {
long now = getCurrentTime();
int maxId = 0;
@@ -582,7 +617,7 @@ public class ClusterNodeInfo {
&& iId.equals(instanceId)) {
boolean worthRetrying = waitForLeaseExpiry(store, doc, leaseEnd, machineId, instanceId);
if (worthRetrying) {
- return createInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId, false, invisible);
+ return createInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId, false, invisible, reuseAfterRecoverMillis);
}
}
@@ -615,6 +650,18 @@ public class ClusterNodeInfo {
// if we get here the cluster node entry is inactive. if recovery
// was needed, then it was successful
+ if (reuseAfterRecoverMillis > 0) {
+ Long lastRecoverTime = (Long) doc.get(RECOVER_TIME_KEY);
+ if (lastRecoverTime != null) {
+ long diff = now - lastRecoverTime;
+ if (diff < reuseAfterRecoverMillis) {
+ reuseFailureReason = reject(id,
+ "was recovered recently and is not configured for reuse until another " + diff + "ms");
+ continue;
+ }
+ }
+ }
+
// create a candidate. those with matching machine and instance id
// are preferred, then the one with the lowest clusterId.
candidates.add(new ClusterNodeInfo(id, store, mId, iId, false, invisible));
@@ -1139,6 +1186,7 @@ public class ClusterNodeInfo {
}
UpdateOp update = new UpdateOp("" + id, true);
update.set(LEASE_END_KEY, null);
+ update.set(RECOVER_TIME_KEY, null);
update.set(STATE, null);
update.set(INVISIBLE, false);
update.set(RUNTIME_ID_KEY, null);
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLock.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLock.java
index 39dcf75748..aa7a875d67 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLock.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLock.java
@@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory;
import static org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.ClusterNodeState.ACTIVE;
import static org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.DEFAULT_LEASE_DURATION_MILLIS;
import static org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.LEASE_END_KEY;
+import static org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.RECOVER_TIME_KEY;
import static org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.REV_RECOVERY_BY;
import static org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.REV_RECOVERY_LOCK;
import static org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.RUNTIME_ID_KEY;
@@ -104,6 +105,7 @@ class RecoveryLock {
update.set(STATE, null);
update.set(LEASE_END_KEY, null);
update.set(RUNTIME_ID_KEY, null);
+ update.set(RECOVER_TIME_KEY, clock.getTime());
} else {
// make sure lease is expired
update.set(LEASE_END_KEY, clock.getTime() - 1);
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java
index 5622a3b6ca..e84012b067 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java
@@ -57,6 +57,7 @@ public class ClusterNodeInfoTest {
private TestStore store;
private FailureHandler handler = new FailureHandler();
private boolean invisible;
+ private long reuseAfterRecoverMillis = ClusterNodeInfo.DEFAULT_REUSE_AFTER_RECOVER_MILLIS;
public ClusterNodeInfoTest(boolean invisible) {
this.invisible = invisible;
@@ -415,6 +416,7 @@ public class ClusterNodeInfoTest {
op.set(ClusterNodeInfo.REV_RECOVERY_LOCK, "NONE");
op.set(ClusterNodeInfo.STATE, null);
op.set(ClusterNodeInfo.LEASE_END_KEY, null);
+ op.set(ClusterNodeInfo.RECOVER_TIME_KEY, null);
assertNotNull(store.findAndUpdate(Collection.CLUSTER_NODES, op));
// should be able to acquire it
@@ -708,6 +710,28 @@ public class ClusterNodeInfoTest {
assertFalse(handler.isLeaseFailure());
}
+ @Test
+ public void reuseAfterRecover() throws Exception {
+ reuseAfterRecoverMillis = 60000;
+ ClusterNodeInfo info = newClusterNodeInfo(1);
+ assertEquals(1, info.getId());
+ // wait until after lease end
+ clock.waitUntil(info.getLeaseEndTime() + ClusterNodeInfo.DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS);
+ recoverClusterNode(1);
+ try {
+ info = newClusterNodeInfo(1);
+ fail("should fail");
+ } catch(Exception e) {
+ // should fail
+ }
+ assertEquals(42, newClusterNodeInfo(0).getId());
+ assertEquals(1, newClusterNodeInfo(0).getId());
+ assertEquals(3, newClusterNodeInfo(3).getId());
+ // wait until after recover + reuseAfterRecoverMillis
+ clock.waitUntil(info.getLeaseEndTime() + reuseAfterRecoverMillis);
+ assertEquals(1, newClusterNodeInfo(1).getId());
+ }
+
private void assertLeaseFailure() throws Exception {
for (int i = 0; i < 100; i++) {
if (handler.isLeaseFailure()) {
@@ -750,7 +774,8 @@ public class ClusterNodeInfoTest {
private ClusterNodeInfo newClusterNodeInfo(int clusterId,
String instanceId) {
ClusterNodeInfo info = ClusterNodeInfo.getInstance(store,
- new SimpleRecoveryHandler(store, clock), null, instanceId, clusterId, invisible);
+ new SimpleRecoveryHandler(store, clock), null, instanceId, clusterId, invisible,
+ reuseAfterRecoverMillis);
info.setLeaseFailureHandler(handler);
return info;
}