You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mr...@apache.org on 2015/09/02 08:32:47 UTC
svn commit: r1700709 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/document/
test/java/org/apache/jackrabbit/oak/plugins/document/
Author: mreutegg
Date: Wed Sep 2 06:32:46 2015
New Revision: 1700709
URL: http://svn.apache.org/r1700709
Log:
OAK-3305: Self recovering instance may not see all changes
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryRandomizedIT.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRootBuilder.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1700709&r1=1700708&r2=1700709&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java Wed Sep 2 06:32:46 2015
@@ -503,6 +503,10 @@ public final class DocumentNodeStore
backgroundWrite();
}
}
+ checkLastRevRecovery();
+ // Renew the lease because it may have been stale
+ renewClusterIdLease();
+
getRevisionComparator().add(headRevision, Revision.newRevision(0));
dispatcher = new ChangeDispatcher(getRoot());
@@ -522,9 +526,6 @@ public final class DocumentNodeStore
new BackgroundOperation(this, isDisposed),
"DocumentNodeStore background update thread " + threadNamePostfix);
backgroundUpdateThread.setDaemon(true);
- checkLastRevRecovery();
- // Renew the lease because it may have been stale
- renewClusterIdLease();
backgroundReadThread.start();
backgroundUpdateThread.start();
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRootBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRootBuilder.java?rev=1700709&r1=1700708&r2=1700709&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRootBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRootBuilder.java Wed Sep 2 06:32:46 2015
@@ -169,7 +169,7 @@ class DocumentRootBuilder extends Abstra
return reset();
}
- private void purge() {
+ void purge() {
branch.setRoot(super.getNodeState());
super.reset(branch.getHead());
updates = 0;
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java?rev=1700709&r1=1700708&r2=1700709&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java Wed Sep 2 06:32:46 2015
@@ -22,6 +22,7 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.collect.Maps.filterKeys;
import static java.util.Collections.singletonList;
import static org.apache.jackrabbit.oak.plugins.document.Collection.JOURNAL;
+import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.PROPERTY_OR_DELETED;
import java.util.Iterator;
@@ -138,7 +139,7 @@ public class LastRevRecoveryAgent {
UnsavedModifications unsavedParents = new UnsavedModifications();
//Map of known last rev of checked paths
- Map<String, Revision> knownLastRevs = MapFactory.getInstance().create();
+ Map<String, Revision> knownLastRevOrModification = MapFactory.getInstance().create();
final DocumentStore docStore = nodeStore.getDocumentStore();
final JournalEntry changes = JOURNAL.newDocument(docStore);
@@ -160,7 +161,7 @@ public class LastRevRecoveryAgent {
// most recent revision currently obtained from either a
// _lastRev entry or an explicit modification on the document
if (lastRevForParents != null) {
- knownLastRevs.put(doc.getPath(), lastRevForParents);
+ knownLastRevOrModification.put(doc.getPath(), lastRevForParents);
}
//If both currentLastRev and lostLastRev are null it means
@@ -186,7 +187,21 @@ public class LastRevRecoveryAgent {
for (String parentPath : unsavedParents.getPaths()) {
Revision calcLastRev = unsavedParents.get(parentPath);
- Revision knownLastRev = knownLastRevs.get(parentPath);
+ Revision knownLastRev = knownLastRevOrModification.get(parentPath);
+ if (knownLastRev == null) {
+ // we don't know when the document was last modified with
+ // the given clusterId. need to read from store
+ String id = Utils.getIdFromPath(parentPath);
+ NodeDocument doc = docStore.find(NODES, id);
+ if (doc != null) {
+ Revision lastRev = doc.getLastRev().get(clusterId);
+ Revision lastMod = determineLastModification(doc, clusterId);
+ knownLastRev = Utils.max(lastRev, lastMod);
+ } else {
+ log.warn("Unable to find document: {}", id);
+ continue;
+ }
+ }
//Copy the calcLastRev of parent only if they have changed
//In many case it might happen that parent have consistent lastRev
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryRandomizedIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryRandomizedIT.java?rev=1700709&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryRandomizedIT.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryRandomizedIT.java Wed Sep 2 06:32:46 2015
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.DefaultNodeStateDiff;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.jackrabbit.oak.spi.state.AbstractNodeState.comparePropertiesAgainstBaseState;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * A randomized test for _lastRev recovery.
+ */
+public class LastRevRecoveryRandomizedIT {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LastRevRecoveryRandomizedIT.class);
+ private static final int SEED = Integer.getInteger(
+ LastRevRecoveryRandomizedIT.class.getSimpleName() + "-seed",
+ new Random().nextInt());
+
+ private Random random = new Random();
+
+ private MemoryDocumentStore store;
+
+ private DocumentNodeStore ns;
+
+ private Map<String, NodeState> currentState = Maps.newHashMap();
+
+ private DocumentRootBuilder builder;
+
+ private Map<String, NodeState> pending = Maps.newHashMap();
+
+ private int counter = 0;
+
+ private List<String> ops = Lists.newArrayList();
+
+ private Clock clock;
+
+ @Before
+ public void setUp() throws Exception {
+ LOG.info("Running " + getClass().getSimpleName() + " with seed " + SEED);
+ clock = new Clock.Virtual();
+ Revision.setClock(clock);
+ ClusterNodeInfo.setClock(clock);
+ random.setSeed(SEED);
+ store = new MemoryDocumentStore();
+ ns = new DocumentMK.Builder().setDocumentStore(store)
+ .setLeaseCheck(false).clock(clock)
+ .setAsyncDelay(0).getNodeStore();
+ builder = newBuilder(ns);
+ builder.child("root");
+ merge(ns, builder);
+ currentState.put("/root", ns.getRoot().getChildNode("root"));
+ builder = newBuilder(ns);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ Revision.resetClockToDefault();
+ ClusterNodeInfo.resetClockToDefault();
+ ns.dispose();
+ }
+
+ @Test
+ public void randomized() throws Exception {
+ for (int i = 0; i < 1000; i++) {
+ boolean success = false;
+ try {
+ switch (random.nextInt(10)) {
+ case 0:
+ case 1:
+ case 2:
+ addNode();
+ break;
+ case 3:
+ addLeafNode();
+ break;
+ case 4:
+ removeNode();
+ break;
+ case 5:
+ case 6:
+ setProperty();
+ break;
+ case 7:
+ merge();
+ break;
+ case 8:
+ purge();
+ break;
+ case 9:
+ bgOp();
+ break;
+ }
+ checkStore();
+ success = true;
+ } finally {
+ if (!success) {
+ int num = 0;
+ for (String line : ops) {
+ System.out.println(num++ + ": " + line);
+ }
+ }
+ }
+ }
+ }
+
+ private void bgOp() {
+ ops.add("runBackgroundOperations()");
+ ns.runBackgroundOperations();
+ }
+
+ private void purge() {
+ ops.add("purge()");
+ builder.purge();
+ }
+
+ private void merge() throws CommitFailedException {
+ ops.add("merge()");
+ merge(ns, builder);
+ for (Map.Entry<String, NodeState> entry : pending.entrySet()) {
+ if (entry.getValue() == null) {
+ currentState.remove(entry.getKey());
+ } else {
+ currentState.put(entry.getKey(), entry.getValue());
+ }
+ }
+ pending.clear();
+ builder = newBuilder(ns);
+ }
+
+ private void setProperty() {
+ String p = choosePath();
+ String name = "p-" + counter++;
+ ops.add("setProperty() " + PathUtils.concat(p, name));
+ NodeBuilder ns = getNode(p);
+ ns.setProperty(name, "v");
+ pending.put(p, ns.getNodeState());
+ }
+
+ private void removeNode() {
+ String p = choosePath();
+ if (p.equals("/root")) {
+ return;
+ }
+ ops.add("removeNode() " + p);
+ getNode(p).remove();
+ pending.put(p, null);
+ }
+
+ private void addNode() {
+ String p = choosePath();
+ List<String> elements = Lists.newArrayList(PathUtils.elements(p));
+ if (elements.size() > 2) {
+ elements = elements.subList(1, elements.size() - 1);
+ elements = elements.subList(0, random.nextInt(elements.size() + 1));
+ p = PathUtils.concat("/root", elements.toArray(new String[elements.size()]));
+ }
+ String name = "n-" + counter++;
+ ops.add("addNode() " + PathUtils.concat(p, name));
+ NodeBuilder ns = getNode(p);
+ pending.put(PathUtils.concat(p, name), ns.child(name).getNodeState());
+ }
+
+ private void addLeafNode() {
+ String p = choosePath();
+ String name = "n-" + counter++;
+ ops.add("addLeafNode() " + PathUtils.concat(p, name));
+ NodeBuilder ns = getNode(p);
+ pending.put(PathUtils.concat(p, name), ns.child(name).getNodeState());
+ }
+
+ private NodeBuilder getNode(String path) {
+ NodeBuilder node = builder;
+ for (String name : PathUtils.elements(path)) {
+ node = node.getChildNode(name);
+ }
+ if (!node.exists()) {
+ throw new IllegalStateException("node does not exist: " + path);
+ }
+ return node;
+ }
+
+ private String choosePath() {
+ String path = "/root";
+ String next;
+ while ((next = chooseNode(path)) != null) {
+ path = next;
+ }
+ return path;
+ }
+
+ private String chooseNode(String parentPath) {
+ NodeBuilder node = getNode(parentPath);
+
+ int numChildren = (int) node.getChildNodeCount(Long.MAX_VALUE);
+ if (numChildren == 0) {
+ return null;
+ }
+ int k = random.nextInt(numChildren);
+ int c = 0;
+ for (String name : node.getChildNodeNames()) {
+ if (c++ == k) {
+ return PathUtils.concat(parentPath, name);
+ }
+ }
+
+ return null;
+ }
+
+ private void checkStore() {
+ MemoryDocumentStore s = store.copy();
+ // force lease expire
+ UpdateOp op = new UpdateOp(String.valueOf(ns.getClusterId()), false);
+ op.set(ClusterNodeInfo.LEASE_END_KEY, clock.getTime() - 1000);
+ if (s.findAndUpdate(Collection.CLUSTER_NODES, op) == null) {
+ fail("failed to set lease end");
+ }
+ // will trigger recovery on startup
+ DocumentNodeStore dns = new DocumentMK.Builder()
+ .setClusterId(ns.getClusterId())
+ .clock(clock).setLeaseCheck(false)
+ .setDocumentStore(s).setAsyncDelay(0).getNodeStore();
+ Map<String, NodeState> states = Maps.newHashMap(currentState);
+ NodeState root = dns.getRoot().getChildNode("root");
+ compareAndTraverse(root, "/root", states);
+ assertTrue("missing nodes: " + states.keySet() + " (seed=" + SEED + ")",
+ states.isEmpty());
+ dns.dispose();
+ }
+
+ private void compareAndTraverse(NodeState state,
+ final String path,
+ Map<String, NodeState> states) {
+ NodeState expected = states.remove(path);
+ if (expected == null) {
+ fail("unexpected node at " + path + " (seed=" + SEED + ")");
+ return;
+ }
+ comparePropertiesAgainstBaseState(state, expected, new DefaultNodeStateDiff() {
+ @Override
+ public boolean propertyAdded(PropertyState after) {
+ fail("unexpected property: " + path + "/" + after + " (seed=" + SEED + ")");
+ return super.propertyAdded(after);
+ }
+
+ @Override
+ public boolean propertyChanged(PropertyState before,
+ PropertyState after) {
+ assertEquals("property mismatch on node " + path + " (seed=" + SEED + ")",
+ before, after);
+ return super.propertyChanged(before, after);
+ }
+
+ @Override
+ public boolean propertyDeleted(PropertyState before) {
+ fail("missing property: " + path + "/" + before + " (seed=" + SEED + ")");
+ return super.propertyDeleted(before);
+ }
+ });
+ for (ChildNodeEntry entry : state.getChildNodeEntries()) {
+ String p = PathUtils.concat(path, entry.getName());
+ compareAndTraverse(entry.getNodeState(), p, states);
+ }
+ }
+
+ private static DocumentRootBuilder newBuilder(DocumentNodeStore store) {
+ return (DocumentRootBuilder) store.getRoot().builder();
+ }
+
+ private static void merge(NodeStore ns, NodeBuilder builder)
+ throws CommitFailedException {
+ ns.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryRandomizedIT.java
------------------------------------------------------------------------------
svn:eol-style = native