You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mr...@apache.org on 2018/07/03 15:25:39 UTC
svn commit: r1834986 [2/2] - in /jackrabbit/oak/trunk:
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/
oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/...
Propchange: jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLock.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java Tue Jul 3 15:25:38 2018
@@ -68,7 +68,10 @@ public class MongoMissingLastRevSeeker e
public boolean isRecoveryNeeded() {
Bson query = Filters.and(
Filters.eq(ClusterNodeInfo.STATE, ClusterNodeInfo.ClusterNodeState.ACTIVE.name()),
- Filters.lt(ClusterNodeInfo.LEASE_END_KEY, clock.getTime())
+ Filters.or(
+ Filters.lt(ClusterNodeInfo.LEASE_END_KEY, clock.getTime()),
+ Filters.eq(ClusterNodeInfo.REV_RECOVERY_LOCK, ClusterNodeInfo.RecoverLockState.ACQUIRED.name())
+ )
);
return getClusterNodeCollection().find(query).iterator().hasNext();
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java Tue Jul 3 15:25:38 2018
@@ -442,17 +442,6 @@ public abstract class BaseDocumentDiscov
writeSimulationThread.join();
}
}
-
- /** OAK-3292 : when on a machine without a mac address, the 'random:' prefix is used and instances
- * that have timed out are automagially removed by ClusterNodeInfo.createInstance - that poses
- * a problem to testing - so this method exposes whether the instance has such a 'random:' prefix
- * and thus allows to take appropriate action
- */
- public boolean hasRandomMachineId() {
- //TODO: this might not be the most stable way - but avoids having to change ClusterNodeInfo
- return ns.getClusterInfo().toString().contains("random:");
- }
-
}
interface Expectation {
@@ -760,19 +749,6 @@ public abstract class BaseDocumentDiscov
workingDir = reactivatedWorkingDir;
logger.info("Case 0: creating instance");
final SimplifiedInstance newInstance = createInstance(workingDir);
- if (newInstance.hasRandomMachineId()) {
- // OAK-3292 : on an instance which has no networkInterface with a mac address,
- // the machineId chosen by ClusterNodeInfo will be 'random:'.. and
- // ClusterNodeInfo.createInstance will feel free to remove it when the lease
- // has timed out
- // that really renders it very difficult to continue testing here,
- // since this test is all about keeping track who became inactive etc
- // and ClusterNodeInfo.createInstance removing it 'at a certain point' is difficult
- // and not very useful to test..
- //
- // so: stop testing at this point:
- return;
- }
newInstance.setLeastTimeout(5000, 1000);
newInstance.startSimulatingWrites(500);
logger.info("Case 0: created instance: " + newInstance.ns.getClusterId());
@@ -794,19 +770,6 @@ public abstract class BaseDocumentDiscov
if (instances.size() < MAX_NUM_INSTANCES) {
logger.info("Case 1: creating instance");
final SimplifiedInstance newInstance = createInstance(workingDir);
- if (newInstance.hasRandomMachineId()) {
- // OAK-3292 : on an instance which has no networkInterface with a mac address,
- // the machineId chosen by ClusterNodeInfo will be 'random:'.. and
- // ClusterNodeInfo.createInstance will feel free to remove it when the lease
- // has timed out
- // that really renders it very difficult to continue testing here,
- // since this test is all about keeping track who became inactive etc
- // and ClusterNodeInfo.createInstance removing it 'at a certain point' is difficult
- // and not very useful to test..
- //
- // so: stop testing at this point:
- return;
- }
newInstance.setLeastTimeout(5000, 1000);
newInstance.startSimulatingWrites(500);
logger.info("Case 1: created instance: " + newInstance.ns.getClusterId());
Added: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java?rev=1834986&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java (added)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java Tue Jul 3 15:25:38 2018
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.lang.reflect.Constructor;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
+import org.hamcrest.Matchers;
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.contains;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+public class ClusterNodeInfoComparatorTest {
+
+ private static final String MACHINE_ID = "machine";
+ private static final String INSTANCE_1 = "node-1";
+ private static final String INSTANCE_2 = "node-2";
+
+ private DocumentStore store = new MemoryDocumentStore();
+
+ private ClusterNodeInfoComparator comparator =
+ new ClusterNodeInfoComparator(MACHINE_ID, INSTANCE_1);
+
+ @Test
+ public void lowerClusterIdFirst() {
+ SortedSet<ClusterNodeInfo> infos = new TreeSet<>(comparator);
+ infos.add(newClusterNodeInfo(1, INSTANCE_1));
+ infos.add(newClusterNodeInfo(3, INSTANCE_1));
+ infos.add(newClusterNodeInfo(2, INSTANCE_1));
+ infos.add(newClusterNodeInfo(4, INSTANCE_1));
+
+ assertThat(idList(infos), contains(1, 2, 3, 4));
+ }
+
+ @Test
+ public void lowerClusterIdFirstNonMatchingEnvironment() {
+ SortedSet<ClusterNodeInfo> infos = new TreeSet<>(comparator);
+ infos.add(newClusterNodeInfo(1, INSTANCE_2));
+ infos.add(newClusterNodeInfo(3, INSTANCE_2));
+ infos.add(newClusterNodeInfo(2, INSTANCE_2));
+ infos.add(newClusterNodeInfo(4, INSTANCE_2));
+
+ assertThat(idList(infos), contains(1, 2, 3, 4));
+ }
+
+ @Test
+ public void matchingEnvironmentFirst() {
+ SortedSet<ClusterNodeInfo> infos = new TreeSet<>(comparator);
+ infos.add(newClusterNodeInfo(1, INSTANCE_2));
+ infos.add(newClusterNodeInfo(2, INSTANCE_2));
+ infos.add(newClusterNodeInfo(3, INSTANCE_1));
+ infos.add(newClusterNodeInfo(4, INSTANCE_1));
+ infos.add(newClusterNodeInfo(5, INSTANCE_2));
+
+ assertThat(idList(infos), contains(3, 4, 1, 2, 5));
+ }
+
+ private static List<Integer> idList(Set<ClusterNodeInfo> infos) {
+ return infos.stream().map(ClusterNodeInfo::getId).collect(Collectors.toList());
+ }
+
+ private ClusterNodeInfo newClusterNodeInfo(int id, String instanceId) {
+ try {
+ Constructor<ClusterNodeInfo> ctr = ClusterNodeInfo.class.getDeclaredConstructor(
+ int.class, DocumentStore.class, String.class, String.class, boolean.class);
+ ctr.setAccessible(true);
+ return ctr.newInstance(id, store, MACHINE_ID, instanceId, true);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ throw new IllegalStateException();
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java Tue Jul 3 15:25:38 2018
@@ -16,8 +16,17 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
import org.apache.jackrabbit.oak.stats.Clock;
@@ -25,8 +34,13 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -45,7 +59,7 @@ public class ClusterNodeInfoTest {
}
@After
- public void after() throws Exception {
+ public void after() {
ClusterNodeInfo.resetClockToDefault();
}
@@ -246,12 +260,250 @@ public class ClusterNodeInfoTest {
}
}
+ @Test
+ public void readOnlyClusterNodeInfo() {
+ ClusterNodeInfo info = ClusterNodeInfo.getReadOnlyInstance(store);
+ assertEquals(0, info.getId());
+ assertEquals(Long.MAX_VALUE, info.getLeaseEndTime());
+ assertFalse(info.renewLease());
+ }
+
+ @Test
+ public void ignoreEntryWithInvalidID() {
+ String instanceId1 = "node1";
+
+ ClusterNodeInfo info1 = newClusterNodeInfo(0, instanceId1);
+ assertEquals(1, info1.getId());
+ assertEquals(instanceId1, info1.getInstanceId());
+ // shut it down
+ info1.dispose();
+
+ // sneak in an invalid entry
+ UpdateOp op = new UpdateOp("invalid", true);
+ store.create(Collection.CLUSTER_NODES, Collections.singletonList(op));
+
+ // acquire again
+ info1 = newClusterNodeInfo(0, instanceId1);
+ assertEquals(1, info1.getId());
+ assertEquals(instanceId1, info1.getInstanceId());
+ info1.dispose();
+ }
+
+ @Test
+ public void acquireInactiveClusterId() {
+ // simulate multiple cluster nodes
+ String instanceId1 = "node1";
+ String instanceId2 = "node2";
+
+ ClusterNodeInfo info1 = newClusterNodeInfo(0, instanceId1);
+ assertEquals(1, info1.getId());
+ assertEquals(instanceId1, info1.getInstanceId());
+ // shut it down
+ info1.dispose();
+
+ // simulate start from different location
+ ClusterNodeInfo info2 = newClusterNodeInfo(0, instanceId2);
+ // must acquire inactive clusterId 1
+ assertEquals(1, info2.getId());
+ assertEquals(instanceId2, info2.getInstanceId());
+ info2.dispose();
+ }
+
+ @Test
+ public void acquireInactiveClusterIdWithMatchingEnvironment() {
+ // simulate multiple cluster nodes
+ String instanceId1 = "node1";
+ String instanceId2 = "node2";
+
+ ClusterNodeInfo info1 = newClusterNodeInfo(0, instanceId1);
+ assertEquals(1, info1.getId());
+ assertEquals(instanceId1, info1.getInstanceId());
+
+ // simulate start from different location
+ ClusterNodeInfo info2 = newClusterNodeInfo(0, instanceId2);
+ assertEquals(2, info2.getId());
+ assertEquals(instanceId2, info2.getInstanceId());
+
+ info1.dispose();
+ info2.dispose();
+
+ // restart node2
+ info2 = newClusterNodeInfo(0, instanceId2);
+ // must acquire clusterId 2 again
+ assertEquals(2, info2.getId());
+ assertEquals(instanceId2, info2.getInstanceId());
+ }
+
+ @Test
+ public void acquireInactiveClusterIdConcurrently() throws Exception {
+ ExecutorService executor = Executors.newCachedThreadPool();
+ String instanceId1 = "node1";
+ String instanceId2 = "node2";
+ String instanceId3 = "node3";
+ List<String> instanceIds = new ArrayList<>();
+ Collections.addAll(instanceIds, instanceId1, instanceId2, instanceId3);
+
+ // create a first clusterNode entry
+ ClusterNodeInfo info1 = newClusterNodeInfo(0, instanceId1);
+ assertEquals(1, info1.getId());
+ assertEquals(instanceId1, info1.getInstanceId());
+ // shut it down
+ info1.dispose();
+
+ // start multiple instances from different locations competing for
+ // the same inactive clusterId
+ List<Callable<ClusterNodeInfo>> tasks = new ArrayList<>();
+ for (String id : instanceIds) {
+ tasks.add(() -> newClusterNodeInfo(0, id));
+ }
+ Map<Integer, ClusterNodeInfo> clusterNodes = executor.invokeAll(tasks)
+ .stream().map(f -> {
+ try {
+ return f.get();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }).collect(Collectors.toMap(ClusterNodeInfo::getId, Function.identity()));
+
+ // must have different clusterIds
+ assertEquals(3, clusterNodes.size());
+ assertThat(clusterNodes.keySet(), containsInAnyOrder(1, 2, 3));
+
+ clusterNodes.values().forEach(ClusterNodeInfo::dispose);
+ executor.shutdown();
+ }
+
+ @Test
+ public void acquireExpiredClusterId() throws Exception {
+ String instanceId1 = "node1";
+
+ ClusterNodeInfo info1 = newClusterNodeInfo(0, instanceId1);
+ assertEquals(1, info1.getId());
+ assertEquals(instanceId1, info1.getInstanceId());
+ expireLease(info1);
+
+ // simulate a restart after a crash and expired lease
+ info1 = newClusterNodeInfo(0, instanceId1);
+ // must acquire expired clusterId 1
+ assertEquals(1, info1.getId());
+ assertEquals(instanceId1, info1.getInstanceId());
+ info1.dispose();
+ }
+
+ @Test
+ public void skipExpiredClusterIdWithDifferentInstanceId() throws Exception {
+ // simulate multiple cluster nodes
+ String instanceId1 = "node1";
+ String instanceId2 = "node2";
+
+ ClusterNodeInfo info1 = newClusterNodeInfo(0, instanceId1);
+ assertEquals(1, info1.getId());
+ assertEquals(instanceId1, info1.getInstanceId());
+ expireLease(info1);
+
+ // simulate start from different location
+ ClusterNodeInfo info2 = newClusterNodeInfo(0, instanceId2);
+ // must not acquire expired clusterId 1
+ assertEquals(2, info2.getId());
+ assertEquals(instanceId2, info2.getInstanceId());
+ info2.dispose();
+ }
+
+ @Test
+ public void acquireExpiredClusterIdStatic() throws Exception {
+ String instanceId1 = "node1";
+ String instanceId2 = "node2";
+
+ ClusterNodeInfo info1 = newClusterNodeInfo(0, instanceId1);
+ assertEquals(1, info1.getId());
+ assertEquals(instanceId1, info1.getInstanceId());
+ expireLease(info1);
+
+ // simulate start from different location and
+ // acquire with static clusterId
+ try {
+ newClusterNodeInfo(1, instanceId2);
+ fail("Must fail with DocumentStoreException");
+ } catch (DocumentStoreException e) {
+ assertThat(e.getMessage(), containsString("needs recovery"));
+ }
+ }
+
+ @Test
+ public void acquireExpiredClusterIdConcurrently() throws Exception {
+ ExecutorService executor = Executors.newCachedThreadPool();
+ String instanceId1 = "node1";
+ String instanceId2 = "node2";
+ String instanceId3 = "node3";
+ List<String> instanceIds = new ArrayList<>();
+ Collections.addAll(instanceIds, instanceId1, instanceId2, instanceId3);
+
+ ClusterNodeInfo info1 = newClusterNodeInfo(0, instanceId1);
+ assertEquals(1, info1.getId());
+ assertEquals(instanceId1, info1.getInstanceId());
+ expireLease(info1);
+
+ // start multiple instances from different locations competing for
+ // the same clusterId with expired lease
+ List<Callable<ClusterNodeInfo>> tasks = new ArrayList<>();
+ for (String id : instanceIds) {
+ tasks.add(() -> newClusterNodeInfo(0, id));
+ }
+ Map<Integer, ClusterNodeInfo> clusterNodes = executor.invokeAll(tasks)
+ .stream().map(f -> {
+ try {
+ return f.get();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }).collect(Collectors.toMap(ClusterNodeInfo::getId, Function.identity()));
+
+ // must have different clusterIds
+ assertEquals(3, clusterNodes.size());
+ assertThat(clusterNodes.keySet(), containsInAnyOrder(1, 2, 3));
+
+ clusterNodes.values().forEach(ClusterNodeInfo::dispose);
+ executor.shutdown();
+ }
+
+ @Test
+ public void skipClusterIdWithoutStartTime() {
+ ClusterNodeInfo info = newClusterNodeInfo(0);
+ int id = info.getId();
+ assertEquals(1, id);
+ // shut it down
+ info.dispose();
+
+ // remove startTime field
+ UpdateOp op = new UpdateOp(String.valueOf(id), false);
+ op.remove(ClusterNodeInfo.START_TIME_KEY);
+ assertNotNull(store.findAndUpdate(Collection.CLUSTER_NODES, op));
+
+ // acquire it again
+ info = newClusterNodeInfo(0);
+ // must not use clusterId 1
+ assertNotEquals(1, info.getId());
+ }
+
+ private void expireLease(ClusterNodeInfo info)
+ throws InterruptedException {
+ // let lease expire
+ clock.waitUntil(info.getLeaseEndTime() +
+ ClusterNodeInfo.DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS);
+ // check if expired -> recovery is needed
+ MissingLastRevSeeker util = new MissingLastRevSeeker(store, clock);
+ String key = String.valueOf(info.getId());
+ ClusterNodeInfoDocument infoDoc = store.find(Collection.CLUSTER_NODES, key);
+ assertNotNull(infoDoc);
+ assertTrue(infoDoc.isRecoveryNeeded(clock.getTime()));
+ }
+
private void recoverClusterNode(int clusterId) throws Exception {
DocumentNodeStore ns = new DocumentMK.Builder()
.setDocumentStore(store.getStore()) // use unwrapped store
.setAsyncDelay(0).setClusterId(42).clock(clock).getNodeStore();
try {
- LastRevRecoveryAgent recovery = new LastRevRecoveryAgent(ns);
+ LastRevRecoveryAgent recovery = new LastRevRecoveryAgent(ns.getDocumentStore(), ns);
recovery.recover(clusterId);
} finally {
ns.dispose();
@@ -262,13 +514,32 @@ public class ClusterNodeInfoTest {
clock.waitUntil(clock.getTime() + ClusterNodeInfo.DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS + 1);
}
- private ClusterNodeInfo newClusterNodeInfo(int clusterId) {
- ClusterNodeInfo info = ClusterNodeInfo.getInstance(store, clusterId);
+ private ClusterNodeInfo newClusterNodeInfo(int clusterId,
+ String instanceId) {
+ ClusterNodeInfo info = ClusterNodeInfo.getInstance(store,
+ new SimpleRecoveryHandler(), null, instanceId, clusterId);
info.setLeaseFailureHandler(handler);
- assertTrue(info.renewLease()); // perform initial lease renew
return info;
}
+ private class SimpleRecoveryHandler implements RecoveryHandler {
+
+ @Override
+ public boolean recover(int clusterId) {
+ // simulate recovery by acquiring recovery lock
+ RecoveryLock lock = new RecoveryLock(store, clock, clusterId);
+ if (lock.acquireRecoveryLock(clusterId)) {
+ lock.releaseRecoveryLock(true);
+ return true;
+ }
+ return false;
+ }
+ }
+
+ private ClusterNodeInfo newClusterNodeInfo(int clusterId) {
+ return newClusterNodeInfo(clusterId, null);
+ }
+
static final class FailureHandler implements LeaseFailureHandler {
private final AtomicBoolean leaseFailure = new AtomicBoolean();
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterTest.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterTest.java Tue Jul 3 15:25:38 2018
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
+import static org.apache.jackrabbit.oak.plugins.document.RecoveryHandler.NOOP;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -44,6 +45,7 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.Clock;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.junit.After;
@@ -66,6 +68,11 @@ public class ClusterTest {
private MemoryDocumentStore ds;
private MemoryBlobStore bs;
+ @After
+ public void resetClock() {
+ ClusterNodeInfo.resetClockToDefault();
+ }
+
@Test
public void threeNodes() throws Exception {
DocumentMK mk1 = createMK(1, 0);
@@ -121,17 +128,20 @@ public class ClusterTest {
@Test
public void clusterNodeInfoLease() throws InterruptedException {
+ Clock c = new Clock.Virtual();
+ c.waitUntil(System.currentTimeMillis());
+ ClusterNodeInfo.setClock(c);
+
MemoryDocumentStore store = new MemoryDocumentStore();
ClusterNodeInfo c1, c2;
- c1 = ClusterNodeInfo.getInstance(store, "m1", null);
+ c1 = ClusterNodeInfo.getInstance(store, NOOP, "m1", null, 0);
assertEquals(1, c1.getId());
- c1.setLeaseTime(1);
- c1.setLeaseUpdateInterval(0);
- // this will quickly expire
- c1.renewLease();
- Thread.sleep(10);
- c2 = ClusterNodeInfo.getInstance(store, "m1", null);
- assertEquals(1, c2.getId());
+ // expire lease
+ c.waitUntil(c1.getLeaseEndTime() + ClusterNodeInfo.DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS);
+
+ // using a NOOP RecoveryHandler must prevent use of expired clusterId 1 (OAK-7316)
+ c2 = ClusterNodeInfo.getInstance(store, NOOP, "m1", null, 0);
+ assertEquals(2, c2.getId());
}
@Test
@@ -230,36 +240,37 @@ public class ClusterTest {
@Test
public void clusterNodeInfo() {
MemoryDocumentStore store = new MemoryDocumentStore();
- ClusterNodeInfo c1, c2, c3, c4;
+ ClusterNodeInfo c1, c2;
- c1 = ClusterNodeInfo.getInstance(store, "m1", null);
+ c1 = ClusterNodeInfo.getInstance(store, NOOP, "m1", null, 0);
assertEquals(1, c1.getId());
c1.dispose();
// get the same id
- c1 = ClusterNodeInfo.getInstance(store, "m1", null);
+ c1 = ClusterNodeInfo.getInstance(store, NOOP, "m1", null, 0);
assertEquals(1, c1.getId());
c1.dispose();
- // now try to add another one:
- // must get a new id
- c2 = ClusterNodeInfo.getInstance(store, "m2", null);
- assertEquals(2, c2.getId());
-
// a different machine
- c3 = ClusterNodeInfo.getInstance(store, "m3", "/a");
- assertEquals(3, c3.getId());
+ // must get inactive id (OAK-7316)
+ c1 = ClusterNodeInfo.getInstance(store, NOOP, "m2", null, 0);
+ assertEquals(1, c1.getId());
+
+ // yet another machine
+ c2 = ClusterNodeInfo.getInstance(store, NOOP, "m3", "/a", 0);
+ assertEquals(2, c2.getId());
+ c1.dispose();
c2.dispose();
- c3.dispose();
- c3 = ClusterNodeInfo.getInstance(store, "m3", "/a");
- assertEquals(3, c3.getId());
+ // must acquire same id as before with matching machineId/instanceId
+ c1 = ClusterNodeInfo.getInstance(store, NOOP, "m3", "/a", 0);
+ assertEquals(2, c1.getId());
- c3.dispose();
+ c1.dispose();
- c4 = ClusterNodeInfo.getInstance(store, "m3", "/b");
- assertEquals(4, c4.getId());
+ c1 = ClusterNodeInfo.getInstance(store, NOOP, "m3", "/b", 0);
+ assertEquals(1, c1.getId());
c1.dispose();
}
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java Tue Jul 3 15:25:38 2018
@@ -2073,7 +2073,7 @@ public class DocumentNodeStoreTest {
.setDocumentStore(docStore).getNodeStore();
// perform recovery if needed
- LastRevRecoveryAgent agent = new LastRevRecoveryAgent(store2);
+ LastRevRecoveryAgent agent = new LastRevRecoveryAgent(docStore, store2);
if (agent.isRecoveryNeeded()) {
agent.recover(1);
}
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FormatVersionTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FormatVersionTest.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FormatVersionTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FormatVersionTest.java Tue Jul 3 15:25:38 2018
@@ -32,6 +32,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.document.FormatVersion.V1_6;
import static org.apache.jackrabbit.oak.plugins.document.FormatVersion.V1_8;
import static org.apache.jackrabbit.oak.plugins.document.FormatVersion.valueOf;
+import static org.apache.jackrabbit.oak.plugins.document.RecoveryHandler.NOOP;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertSame;
@@ -119,8 +120,7 @@ public class FormatVersionTest {
public void activeClusterNodes() throws Exception {
DocumentStore store = new MemoryDocumentStore();
V1_0.writeTo(store);
- ClusterNodeInfo info = ClusterNodeInfo.getInstance(store, 1);
- info.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
V1_2.writeTo(store);
}
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java Tue Jul 3 15:25:38 2018
@@ -376,7 +376,7 @@ public class JournalTest extends Abstrac
//lastRev should not be updated for C #2
assertNull(y1.getLastRev().get(c2Id));
- final LastRevRecoveryAgent recovery = new LastRevRecoveryAgent(ds1);
+ final LastRevRecoveryAgent recovery = new LastRevRecoveryAgent(ds, ds1);
// now 1 also has
final String change1 = "{\"x\":{\"y\":{}}}";
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgentTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgentTest.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgentTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgentTest.java Tue Jul 3 15:25:38 2018
@@ -22,6 +22,7 @@ package org.apache.jackrabbit.oak.plugin
import com.google.common.collect.Iterables;
import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -31,6 +32,8 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -192,6 +195,50 @@ public class LastRevRecoveryAgentTest ex
merge(ds1, b1);
}
+ @Test
+ public void dryRun() throws Exception {
+ //1. Create base structure /x/y
+ NodeBuilder b1 = ds1.getRoot().builder();
+ b1.child("x").child("y");
+ ds1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ ds1.runBackgroundOperations();
+
+ ds2.runBackgroundOperations();
+
+ //2. Add a new node /x/y/z in C2
+ NodeBuilder b2 = ds2.getRoot().builder();
+ b2.child("x").child("y").child("z").setProperty("foo", "bar");
+ ds2.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ Revision zlastRev2 = ds2.getHeadRevision().getRevision(ds2.getClusterId());
+
+ long leaseTime = ds1.getClusterInfo().getLeaseTime();
+ ds1.runBackgroundOperations();
+
+ clock.waitUntil(clock.getTime() + leaseTime + 10);
+
+ //Renew the lease for C1
+ ds1.getClusterInfo().renewLease();
+
+ assertTrue(ds1.getLastRevRecoveryAgent().isRecoveryNeeded());
+
+ Iterable<Integer> cids = ds1.getLastRevRecoveryAgent().getRecoveryCandidateNodes();
+ assertEquals(1, Iterables.size(cids));
+ assertEquals(c2Id, Iterables.get(cids, 0).intValue());
+
+ int updates = ds1.getLastRevRecoveryAgent().recover(
+ Utils.getAllDocuments(store1),
+ Iterables.get(cids, 0),
+ true // dryRun
+ );
+ assertEquals(3, updates);
+
+ // must not have been updated with dryRun set to true
+ assertNull(getDocument(ds1, "/x/y").getLastRev().get(c2Id));
+ assertNull(getDocument(ds1, "/x").getLastRev().get(c2Id));
+ assertNotEquals(zlastRev2, getDocument(ds1, "/").getLastRev().get(c2Id));
+ }
+
private static NodeDocument getDocument(DocumentNodeStore nodeStore,
String path) {
return nodeStore.getDocumentStore().find(NODES, getIdFromPath(path));
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java Tue Jul 3 15:25:38 2018
@@ -20,6 +20,7 @@
package org.apache.jackrabbit.oak.plugins.document;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
@@ -41,10 +42,13 @@ import static com.google.common.collect.
import static org.apache.jackrabbit.oak.plugins.document.Collection.CLUSTER_NODES;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getRootDocument;
+import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -134,7 +138,7 @@ public class LastRevRecoveryTest {
//lastRev should not be updated for C #2
assertNull(y1.getLastRev().get(c2Id));
- LastRevRecoveryAgent recovery = new LastRevRecoveryAgent(ds1);
+ LastRevRecoveryAgent recovery = new LastRevRecoveryAgent(sharedStore, ds1);
//Do not pass y1 but still y1 should be updated
recovery.recover(Lists.newArrayList(x1, z1), c2Id);
@@ -164,7 +168,7 @@ public class LastRevRecoveryTest {
clock.waitUntil(doc.getLeaseEndTime() + 1);
// run recovery on ds2
- LastRevRecoveryAgent agent = new LastRevRecoveryAgent(ds2);
+ LastRevRecoveryAgent agent = new LastRevRecoveryAgent(sharedStore, ds2);
Iterable<Integer> clusterIds = agent.getRecoveryCandidateNodes();
assertTrue(Iterables.contains(clusterIds, c1Id));
assertEquals("must not recover any documents",
@@ -194,7 +198,7 @@ public class LastRevRecoveryTest {
seeker.acquireRecoveryLock(c1Id, c2Id);
// run recovery from ds1
- LastRevRecoveryAgent a1 = new LastRevRecoveryAgent(ds1);
+ LastRevRecoveryAgent a1 = new LastRevRecoveryAgent(sharedStore, ds1);
// use current time -> do not wait for recovery of other agent
assertEquals(-1, a1.recover(c1Id, clock.getTime()));
@@ -208,6 +212,7 @@ public class LastRevRecoveryTest {
public void failStartupOnRecoveryTimeout() throws Exception {
String clusterId = String.valueOf(c1Id);
ClusterNodeInfoDocument doc = sharedStore.find(CLUSTER_NODES, clusterId);
+ assertNotNull(doc);
NodeBuilder builder = ds1.getRoot().builder();
builder.child("x").child("y").child("z");
@@ -221,7 +226,7 @@ public class LastRevRecoveryTest {
// 'wait' until lease expires
clock.waitUntil(doc.getLeaseEndTime() + 1);
// make sure ds2 lease is still fine
- ds2.getClusterInfo().renewLease();
+ assertTrue(ds2.getClusterInfo().renewLease());
// simulate ongoing recovery by cluster node 2
MissingLastRevSeeker seeker = new MissingLastRevSeeker(sharedStore, clock);
@@ -237,6 +242,7 @@ public class LastRevRecoveryTest {
fail("DocumentStoreException expected");
} catch (DocumentStoreException e) {
// expected
+ assertThat(e.getMessage(), containsString("needs recovery"));
}
seeker.releaseRecoveryLock(c1Id, true);
}
@@ -246,6 +252,7 @@ public class LastRevRecoveryTest {
public void breakRecoveryLockWithExpiredLease() throws Exception {
String clusterId = String.valueOf(c1Id);
ClusterNodeInfoDocument info1 = sharedStore.find(CLUSTER_NODES, clusterId);
+ assertNotNull(info1);
NodeBuilder builder = ds1.getRoot().builder();
builder.child("x").child("y").child("z");
@@ -266,6 +273,7 @@ public class LastRevRecoveryTest {
assertTrue(seeker.acquireRecoveryLock(c1Id, c2Id));
// simulate crash of ds2
ClusterNodeInfoDocument info2 = sharedStore.find(CLUSTER_NODES, String.valueOf(c2Id));
+ assertNotNull(info2);
ds2.dispose();
// reset clusterNodes entry
sharedStore.remove(CLUSTER_NODES, String.valueOf(c2Id));
@@ -274,7 +282,8 @@ public class LastRevRecoveryTest {
clock.waitUntil(info2.getLeaseEndTime() + 1);
info1 = sharedStore.find(CLUSTER_NODES, clusterId);
- assertTrue(seeker.isRecoveryNeeded(info1));
+ assertNotNull(info1);
+ assertTrue(info1.isRecoveryNeeded(clock.getTime()));
assertTrue(info1.isBeingRecovered());
// restart ds1
@@ -286,7 +295,8 @@ public class LastRevRecoveryTest {
.setClusterId(1)
.getNodeStore();
info1 = sharedStore.find(CLUSTER_NODES, clusterId);
- assertFalse(seeker.isRecoveryNeeded(info1));
+ assertNotNull(info1);
+ assertFalse(info1.isRecoveryNeeded(clock.getTime()));
assertFalse(info1.isBeingRecovered());
}
@@ -317,7 +327,7 @@ public class LastRevRecoveryTest {
ds2.getClusterInfo().renewLease();
// run recovery on ds2 for ds1
- LastRevRecoveryAgent agent = new LastRevRecoveryAgent(ds2);
+ LastRevRecoveryAgent agent = new LastRevRecoveryAgent(sharedStore, ds2);
Iterable<Integer> clusterIds = agent.getRecoveryCandidateNodes();
assertTrue(Iterables.contains(clusterIds, c1Id));
// nothing to recover
@@ -328,6 +338,71 @@ public class LastRevRecoveryTest {
assertNull(doc.getSweepRevisions().getRevision(c1Id));
}
+ @Test
+ public void selfRecoveryPassedDeadline() throws Exception {
+ String clusterId = String.valueOf(c1Id);
+ ClusterNodeInfoDocument info1 = sharedStore.find(CLUSTER_NODES, clusterId);
+ assertNotNull(info1);
+
+ NodeBuilder builder = ds1.getRoot().builder();
+ builder.child("x").child("y").child("z");
+ merge(ds1, builder);
+ ds1.dispose();
+
+ // reset clusterNodes entry to simulate a crash of ds1
+ sharedStore.remove(CLUSTER_NODES, clusterId);
+ sharedStore.create(CLUSTER_NODES, newArrayList(updateOpFromDocument(info1)));
+
+ // 'wait' until lease expires
+ clock.waitUntil(info1.getLeaseEndTime() + 1);
+
+ AtomicBoolean delay = new AtomicBoolean(true);
+ // simulate a startup with self-recovery by acquiring the clusterId
+ // this will call the recovery handler because the lease is expired
+ // use a seeker that takes longer than the lease duration
+ MissingLastRevSeeker seeker = new MissingLastRevSeeker(sharedStore, clock) {
+ @Override
+ public boolean acquireRecoveryLock(int clusterId, int recoveredBy) {
+ assertTrue(super.acquireRecoveryLock(clusterId, recoveredBy));
+ if (delay.get()) {
+ try {
+ clock.waitUntil(clock.getTime() + ClusterNodeInfo.DEFAULT_LEASE_DURATION_MILLIS + 1);
+ } catch (InterruptedException e) {
+ fail();
+ }
+ }
+ return true;
+ }
+ };
+ RecoveryHandler recoveryHandler = new RecoveryHandlerImpl(
+ sharedStore, clock, seeker);
+ try {
+ // Explicitly acquiring the clusterId must fail
+ // when it takes too long to recover
+ ClusterNodeInfo.getInstance(sharedStore, recoveryHandler,
+ null, null, c1Id);
+ fail("must fail with DocumentStoreException");
+ } catch (DocumentStoreException e) {
+ assertThat(e.getMessage(), containsString("needs recovery"));
+ }
+ // But must succeed with auto-assignment of clusterId
+ // even if machineId and instanceId match
+ ClusterNodeInfo cni = ClusterNodeInfo.getInstance(sharedStore,
+ recoveryHandler,null, null, 0);
+ // though clusterId must not be the one that took too long to recover
+ assertNotEquals(c1Id, cni.getId());
+ // hence recovery is still needed for c1Id
+ assertTrue(seeker.isRecoveryNeeded());
+ cni.dispose();
+ // now run again without delay with the explicit clusterId
+ delay.set(false);
+ // must succeed now
+ cni = ClusterNodeInfo.getInstance(sharedStore, recoveryHandler,
+ null, null, c1Id);
+ assertEquals(c1Id, cni.getId());
+ cni.dispose();
+ }
+
private NodeDocument getDocument(DocumentNodeStore nodeStore, String path) {
return nodeStore.getDocumentStore().find(Collection.NODES, Utils.getIdFromPath(path));
}
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeekerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeekerTest.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeekerTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeekerTest.java Tue Jul 3 15:25:38 2018
@@ -28,6 +28,7 @@ import org.junit.Before;
import org.junit.Test;
import static org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.DEFAULT_LEASE_DURATION_MILLIS;
+import static org.apache.jackrabbit.oak.plugins.document.RecoveryHandler.NOOP;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -68,16 +69,14 @@ public class MissingLastRevSeekerTest ex
@Test
public void acquireRecoveryLockOnActiveClusterNode() {
- ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, 1);
- nodeInfo1.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
assertFalse(seeker.acquireRecoveryLock(1, 2));
}
@Test
public void acquireRecoveryLockOnInactiveClusterNode() {
- ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, 1);
- nodeInfo1.renewLease();
+ ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
nodeInfo1.dispose();
assertFalse(seeker.acquireRecoveryLock(1, 2));
@@ -85,8 +84,7 @@ public class MissingLastRevSeekerTest ex
@Test
public void acquireRecoveryLockOnExpiredLease() throws Exception {
- ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, 1);
- nodeInfo1.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
// expire the lease
clock.waitUntil(clock.getTime() + DEFAULT_LEASE_DURATION_MILLIS + 1);
@@ -95,13 +93,11 @@ public class MissingLastRevSeekerTest ex
@Test
public void acquireRecoveryLockOnAlreadyLocked() throws Exception {
- ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, 1);
- nodeInfo1.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
// expire the lease
clock.waitUntil(clock.getTime() + DEFAULT_LEASE_DURATION_MILLIS + 1);
- ClusterNodeInfo nodeInfo2 = ClusterNodeInfo.getInstance(store, 2);
- nodeInfo2.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 2);
assertTrue(seeker.acquireRecoveryLock(1, 2));
assertFalse(seeker.acquireRecoveryLock(1, 3));
@@ -109,8 +105,7 @@ public class MissingLastRevSeekerTest ex
@Test
public void acquireRecoveryLockAgain() throws Exception {
- ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, 1);
- nodeInfo1.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
// expire the lease
clock.waitUntil(clock.getTime() + DEFAULT_LEASE_DURATION_MILLIS + 1);
@@ -120,8 +115,7 @@ public class MissingLastRevSeekerTest ex
@Test
public void releaseRecoveryLockSuccessTrue() throws Exception {
- ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, 1);
- nodeInfo1.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
// expire the lease
clock.waitUntil(clock.getTime() + DEFAULT_LEASE_DURATION_MILLIS + 1);
@@ -132,14 +126,13 @@ public class MissingLastRevSeekerTest ex
assertFalse(getClusterNodeInfo(1).isBeingRecovered());
assertFalse(getClusterNodeInfo(1).isActive());
// recovery not needed anymore
- assertFalse(seeker.isRecoveryNeeded(getClusterNodeInfo(1)));
+ assertFalse(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime()));
assertFalse(seeker.acquireRecoveryLock(1, 2));
}
@Test
public void releaseRecoveryLockSuccessFalse() throws Exception {
- ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, 1);
- nodeInfo1.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
// expire the lease
clock.waitUntil(clock.getTime() + DEFAULT_LEASE_DURATION_MILLIS + 1);
@@ -150,43 +143,59 @@ public class MissingLastRevSeekerTest ex
assertFalse(getClusterNodeInfo(1).isBeingRecovered());
assertTrue(getClusterNodeInfo(1).isActive());
// recovery still needed
- assertTrue(seeker.isRecoveryNeeded(getClusterNodeInfo(1)));
+ assertTrue(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime()));
assertTrue(seeker.acquireRecoveryLock(1, 2));
}
@Test
public void isRecoveryNeeded() throws Exception {
- ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, 1);
- nodeInfo1.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
// expire the lease
clock.waitUntil(clock.getTime() + DEFAULT_LEASE_DURATION_MILLIS + 1);
- ClusterNodeInfo nodeInfo2 = ClusterNodeInfo.getInstance(store, 2);
- nodeInfo2.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 2);
assertTrue(seeker.isRecoveryNeeded());
- assertTrue(seeker.isRecoveryNeeded(getClusterNodeInfo(1)));
- assertFalse(seeker.isRecoveryNeeded(getClusterNodeInfo(2)));
+ assertTrue(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime()));
+ assertFalse(getClusterNodeInfo(2).isRecoveryNeeded(clock.getTime()));
assertTrue(seeker.acquireRecoveryLock(1, 2));
seeker.releaseRecoveryLock(1, true);
assertFalse(seeker.isRecoveryNeeded());
- assertFalse(seeker.isRecoveryNeeded(getClusterNodeInfo(1)));
- assertFalse(seeker.isRecoveryNeeded(getClusterNodeInfo(2)));
+ assertFalse(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime()));
+ assertFalse(getClusterNodeInfo(2).isRecoveryNeeded(clock.getTime()));
}
@Test
+ public void isRecoveryNeededWithRecoveryLock() throws Exception {
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
+ // expire the lease
+ clock.waitUntil(clock.getTime() + DEFAULT_LEASE_DURATION_MILLIS + 1);
+
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 2);
+
+ assertTrue(seeker.acquireRecoveryLock(1, 2));
+
+ assertTrue(seeker.isRecoveryNeeded());
+ assertTrue(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime()));
+
+ seeker.releaseRecoveryLock(1, true);
+
+ assertFalse(seeker.isRecoveryNeeded());
+ assertFalse(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime()));
+ }
+
+
+ @Test
public void getAllClusterNodes() {
assertEquals(0, Iterables.size(seeker.getAllClusters()));
- ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, 1);
- nodeInfo1.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
assertEquals(1, Iterables.size(seeker.getAllClusters()));
- ClusterNodeInfo nodeInfo2 = ClusterNodeInfo.getInstance(store, 2);
- nodeInfo2.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 2);
assertEquals(2, Iterables.size(seeker.getAllClusters()));
}
@@ -195,8 +204,7 @@ public class MissingLastRevSeekerTest ex
public void getClusterNodeInfo() {
assertNull(getClusterNodeInfo(1));
- ClusterNodeInfo nodeInfo1 = ClusterNodeInfo.getInstance(store, 1);
- nodeInfo1.renewLease();
+ ClusterNodeInfo.getInstance(store, NOOP, null, null, 1);
assertNotNull(getClusterNodeInfo(1));
}
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RandomDocumentNodeStoreSweepTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RandomDocumentNodeStoreSweepTest.java?rev=1834986&r1=1834985&r2=1834986&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RandomDocumentNodeStoreSweepTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RandomDocumentNodeStoreSweepTest.java Tue Jul 3 15:25:38 2018
@@ -176,8 +176,7 @@ public class RandomDocumentNodeStoreSwee
}
}
- private void guardedFail(Operation op, String message)
- throws CommitFailedException {
+ private void guardedFail(Operation op, String message) {
store.fail().after(1).eternally();
try {
op.call();
Added: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerTest.java?rev=1834986&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerTest.java (added)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerTest.java Tue Jul 3 15:25:38 2018
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.apache.jackrabbit.oak.plugins.document.TestUtils.merge;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class RecoveryHandlerTest {
+
+ @Rule
+ public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider();
+
+ private Clock clock = new Clock.Virtual();
+ private FailingDocumentStore store = new FailingDocumentStore(new MemoryDocumentStore());
+ private MissingLastRevSeeker seeker = new MissingLastRevSeeker(store, clock);
+ private RecoveryHandler handler = new RecoveryHandlerImpl(store, clock, seeker);
+
+ @Before
+ public void before() throws Exception {
+ clock.waitUntil(System.currentTimeMillis());
+ Revision.setClock(clock);
+ ClusterNodeInfo.setClock(clock);
+ }
+
+ @AfterClass
+ public static void after() {
+ Revision.resetClockToDefault();
+ ClusterNodeInfo.resetClockToDefault();
+ }
+
+ @Test
+ public void failWithException() throws Exception {
+ DocumentNodeStore ns = builderProvider.newBuilder().clock(clock)
+ .setDocumentStore(store).build();
+ int clusterId = ns.getClusterId();
+
+ NodeBuilder builder = ns.getRoot().builder();
+ builder.child("foo");
+ merge(ns, builder);
+
+ // crash the node store
+ store.fail().after(0).eternally();
+ try {
+ ns.dispose();
+ fail("dispose must fail with exception");
+ } catch (DocumentStoreException e) {
+ // expected
+ }
+
+ // let lease time out
+ clock.waitUntil(clock.getTime() + ClusterNodeInfo.DEFAULT_LEASE_DURATION_MILLIS + 1);
+
+ // must not be able to recover with a failing document store
+ assertFalse(handler.recover(clusterId));
+
+ store.fail().never();
+ // must succeed after store is accessible again
+ assertTrue(handler.recover(clusterId));
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLockTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLockTest.java?rev=1834986&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLockTest.java (added)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLockTest.java Tue Jul 3 15:25:38 2018
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.Semaphore;
+
+import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS;
+import static org.apache.jackrabbit.oak.plugins.document.Collection.CLUSTER_NODES;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class RecoveryLockTest {
+
+ private DocumentStore store = new MemoryDocumentStore();
+
+ private Clock clock = new Clock.Virtual();
+
+ private ExecutorService executor = Executors.newCachedThreadPool();
+
+ private RecoveryLock lock1 = new RecoveryLock(store, clock, 1);
+ private RecoveryLock lock2 = new RecoveryLock(store, clock, 2);
+
+ private ClusterNodeInfo info1;
+ private ClusterNodeInfo info2;
+
+ @Before
+ public void before() throws Exception {
+ clock.waitUntil(System.currentTimeMillis());
+ ClusterNodeInfo.setClock(clock);
+ info1 = ClusterNodeInfo.getInstance(store, RecoveryHandler.NOOP,
+ null, "node1", 1);
+ }
+
+ @After
+ public void after() {
+ ClusterNodeInfo.resetClockToDefault();
+ }
+
+ @Test
+ public void recoveryNotNeeded() {
+ assertFalse(lock1.acquireRecoveryLock(2));
+ }
+
+ @Test
+ public void acquireUnknown() {
+ assertFalse(lock2.acquireRecoveryLock(1));
+ }
+
+ @Test
+ public void releaseRemovedClusterNodeInfo() throws Exception {
+ clock.waitUntil(info1.getLeaseEndTime() + DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS);
+ assertTrue(lock1.acquireRecoveryLock(2));
+ store.remove(CLUSTER_NODES, String.valueOf(info1.getId()));
+ try {
+ lock1.releaseRecoveryLock(false);
+ fail("Must fail with DocumentStoreException");
+ } catch (DocumentStoreException e) {
+ assertThat(e.getMessage(), containsString("does not exist"));
+ }
+ }
+
+ @Test
+ public void acquireAfterLeaseEnd() throws Exception {
+ clock.waitUntil(info1.getLeaseEndTime() + DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS);
+ assertTrue(lock1.acquireRecoveryLock(2));
+ ClusterNodeInfoDocument c = infoDocument(1);
+ assertTrue(c.isActive());
+ assertTrue(c.isBeingRecovered());
+ assertEquals(Long.valueOf(2), c.getRecoveryBy());
+ assertNotNull(c.get(ClusterNodeInfo.LEASE_END_KEY));
+ }
+
+ @Test
+ public void successfulRecovery() throws Exception {
+ clock.waitUntil(info1.getLeaseEndTime() + DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS);
+ assertTrue(lock1.acquireRecoveryLock(2));
+ lock1.releaseRecoveryLock(true);
+ ClusterNodeInfoDocument c = infoDocument(1);
+ assertFalse(c.isActive());
+ assertFalse(c.isBeingRecovered());
+ assertFalse(c.isBeingRecoveredBy(2));
+ assertNull(c.get(ClusterNodeInfo.LEASE_END_KEY));
+ }
+
+ @Test
+ public void unsuccessfulRecovery() throws Exception {
+ clock.waitUntil(info1.getLeaseEndTime() + DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS);
+ assertTrue(lock1.acquireRecoveryLock(2));
+ lock1.releaseRecoveryLock(false);
+ ClusterNodeInfoDocument c = infoDocument(1);
+ assertTrue(c.isActive());
+ assertFalse(c.isBeingRecovered());
+ assertFalse(c.isBeingRecoveredBy(2));
+ assertNotNull(c.get(ClusterNodeInfo.LEASE_END_KEY));
+ assertThat(c.getLeaseEndTime(), lessThan(clock.getTime()));
+ }
+
+ @Test
+ public void inactive() {
+ info1.dispose();
+ assertFalse(lock1.acquireRecoveryLock(1));
+ assertFalse(lock1.acquireRecoveryLock(2));
+ }
+
+ @Test
+ public void selfRecoveryWithinDeadline() throws Exception {
+ // expire clusterId 1
+ clock.waitUntil(info1.getLeaseEndTime() + DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS);
+ ClusterNodeInfoDocument c = infoDocument(1);
+ MissingLastRevSeeker seeker = new MissingLastRevSeeker(store, clock);
+ assertTrue(c.isRecoveryNeeded(clock.getTime()));
+ assertFalse(c.isBeingRecovered());
+
+ Semaphore recovering = new Semaphore(0);
+ Semaphore recovered = new Semaphore(0);
+ // simulate new startup and get info again
+ Future<ClusterNodeInfo> infoFuture = executor.submit(() ->
+ ClusterNodeInfo.getInstance(store, clusterId -> {
+ assertTrue(lock1.acquireRecoveryLock(1));
+ recovering.release();
+ recovered.acquireUninterruptibly();
+ lock1.releaseRecoveryLock(true);
+ return true;
+ }, null, "node1", 1));
+ // wait until submitted task is in recovery
+ recovering.acquireUninterruptibly();
+
+ // check state again
+ c = infoDocument(1);
+ assertTrue(c.isRecoveryNeeded(clock.getTime()));
+ assertTrue(c.isBeingRecovered());
+ assertTrue(c.isBeingRecoveredBy(1));
+ // clusterId 2 must not be able to acquire (break) the recovery lock
+ assertFalse(lock1.acquireRecoveryLock(2));
+
+ // signal recovery to continue
+ recovered.release();
+ ClusterNodeInfo info1 = infoFuture.get();
+ assertEquals(1, info1.getId());
+
+ // check state again
+ c = infoDocument(1);
+ assertFalse(c.isRecoveryNeeded(clock.getTime()));
+ assertFalse(c.isBeingRecovered());
+ assertFalse(c.isBeingRecoveredBy(1));
+
+ // neither must be able to acquire a recovery lock on
+ // an active entry with a valid lease
+ assertFalse(lock1.acquireRecoveryLock(1));
+ assertFalse(lock1.acquireRecoveryLock(2));
+ }
+
+ private ClusterNodeInfoDocument infoDocument(int clusterId) {
+ ClusterNodeInfoDocument doc = store.find(CLUSTER_NODES, String.valueOf(clusterId));
+ assertNotNull(doc);
+ return doc;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLockTest.java
------------------------------------------------------------------------------
svn:eol-style = native