You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2019/09/26 10:48:14 UTC
svn commit: r1867571 - in /jackrabbit/oak/trunk:
oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/
oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/
oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/docume...
Author: amitj
Date: Thu Sep 26 10:48:14 2019
New Revision: 1867571
URL: http://svn.apache.org/viewvc?rev=1867571&view=rev
Log:
OAK-8593: Enable a transient cluster-node to connect as invisible to oak discovery
Added an 'Invisible' flag in ClusterInfo
DocumentFixtureProvider in oak-run-commons now adds 'invisible' by default
Added:
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java (with props)
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java
jackrabbit/oak/trunk/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBroadcastConfig.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteService.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceIT.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java
Modified: jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java (original)
+++ jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java Thu Sep 26 10:48:14 2019
@@ -79,6 +79,7 @@ class DocumentFixtureProvider {
if (readOnly) {
builder.setReadOnlyMode();
}
+ builder.setClusterInvisible(true);
int cacheSize = docStoreOpts.getCacheSize();
if (cacheSize != 0) {
Modified: jackrabbit/oak/trunk/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java (original)
+++ jackrabbit/oak/trunk/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java Thu Sep 26 10:48:14 2019
@@ -20,8 +20,11 @@
package org.apache.jackrabbit.oak.run.cli;
import java.io.IOException;
+import java.util.List;
import joptsimple.OptionParser;
+import org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfoDocument;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder;
import org.apache.jackrabbit.oak.plugins.document.MongoUtils;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -34,6 +37,7 @@ import org.junit.Test;
import static java.util.Collections.emptyMap;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -54,6 +58,7 @@ public class DocumentFixtureTest {
builder.setChildNode("foo");
store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
assertNotNull(fixture.getBlobStore());
+ assertClusterInvisible(store);
}
}
@@ -75,4 +80,11 @@ public class DocumentFixtureTest {
opts.parseAndConfigure(parser, new String[] {MongoUtils.URL});
return opts;
}
+
+ private void assertClusterInvisible(NodeStore store) {
+ List<ClusterNodeInfoDocument> clusterInfos =
+ ClusterNodeInfoDocument.all(((DocumentNodeStore) store).getDocumentStore());
+ assertNotNull(clusterInfos);
+ assertTrue(clusterInfos.get(0).isInvisible());
+ }
}
Modified: jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java Thu Sep 26 10:48:14 2019
@@ -166,6 +166,11 @@ public class ClusterNodeInfo {
private static final String READ_WRITE_MODE_KEY = "readWriteMode";
/**
+ * Key for invisible flag
+ */
+ public static final String INVISIBLE = "invisible";
+
+ /**
* The unique machine id (the MAC address if available).
*/
private static final String MACHINE_ID = getHardwareMachineId();
@@ -339,8 +344,14 @@ public class ClusterNodeInfo {
*/
private LeaseFailureHandler leaseFailureHandler;
+ /**
+ * Flag to indicate this node is invisible to cluster view and thus recovery.
+ */
+ private boolean invisible;
+
+
private ClusterNodeInfo(int id, DocumentStore store, String machineId,
- String instanceId, boolean newEntry) {
+ String instanceId, boolean newEntry, boolean invisible) {
this.id = id;
this.startTime = getCurrentTime();
this.leaseEndTime = this.startTime +leaseTime;
@@ -349,6 +360,7 @@ public class ClusterNodeInfo {
this.machineId = machineId;
this.instanceId = instanceId;
this.newEntry = newEntry;
+ this.invisible = invisible;
}
void setLeaseCheckMode(@NotNull LeaseCheckMode mode) {
@@ -371,6 +383,10 @@ public class ClusterNodeInfo {
return instanceId;
}
+ boolean isInvisible() {
+ return invisible;
+ }
+
/**
* Create a cluster node info instance to be utilized for read only access
* to underlying store.
@@ -379,7 +395,7 @@ public class ClusterNodeInfo {
* @return the cluster node info
*/
public static ClusterNodeInfo getReadOnlyInstance(DocumentStore store) {
- return new ClusterNodeInfo(0, store, MACHINE_ID, WORKING_DIR, true) {
+ return new ClusterNodeInfo(0, store, MACHINE_ID, WORKING_DIR, true, true) {
@Override
public void dispose() {
}
@@ -418,10 +434,31 @@ public class ClusterNodeInfo {
* @return the cluster node info
*/
public static ClusterNodeInfo getInstance(DocumentStore store,
+ RecoveryHandler recoveryHandler,
+ String machineId,
+ String instanceId,
+ int configuredClusterId) {
+
+ return getInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId, false);
+ }
+
+ /**
+ * Get or create a cluster node info instance for the store.
+ *
+ * @param store the document store (for the lease)
+ * @param recoveryHandler the recovery handler to call for a clusterId with
+ * an expired lease.
+ * @param machineId the machine id (null for MAC address)
+ * @param instanceId the instance id (null for current working directory)
+ * @param configuredClusterId the configured cluster id (or 0 for dynamic assignment)
+ * @return the cluster node info
+ */
+ public static ClusterNodeInfo getInstance(DocumentStore store,
RecoveryHandler recoveryHandler,
String machineId,
String instanceId,
- int configuredClusterId) {
+ int configuredClusterId,
+ boolean invisible) {
// defaults for machineId and instanceID
if (machineId == null) {
machineId = MACHINE_ID;
@@ -434,7 +471,7 @@ public class ClusterNodeInfo {
for (int i = 0; i < retries; i++) {
Map.Entry<ClusterNodeInfo, Long> suggestedClusterNode =
createInstance(store, recoveryHandler, machineId,
- instanceId, configuredClusterId, i == 0);
+ instanceId, configuredClusterId, i == 0, invisible);
ClusterNodeInfo clusterNode = suggestedClusterNode.getKey();
Long currentStartTime = suggestedClusterNode.getValue();
String key = String.valueOf(clusterNode.id);
@@ -446,6 +483,7 @@ public class ClusterNodeInfo {
update.set(INFO_KEY, clusterNode.toString());
update.set(STATE, ACTIVE.name());
update.set(OAK_VERSION_KEY, OAK_VERSION);
+ update.set(INVISIBLE, invisible);
ClusterNodeInfoDocument before = null;
final boolean success;
@@ -485,7 +523,8 @@ public class ClusterNodeInfo {
String machineId,
String instanceId,
int configuredClusterId,
- boolean waitForLease) {
+ boolean waitForLease,
+ boolean invisible) {
long now = getCurrentTime();
int maxId = 0;
@@ -544,7 +583,7 @@ public class ClusterNodeInfo {
&& iId.equals(instanceId)) {
boolean worthRetrying = waitForLeaseExpiry(store, doc, leaseEnd, machineId, instanceId);
if (worthRetrying) {
- return createInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId, false);
+ return createInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId, false, invisible);
}
}
@@ -579,7 +618,7 @@ public class ClusterNodeInfo {
// create a candidate. those with matching machine and instance id
// are preferred, then the one with the lowest clusterId.
- candidates.add(new ClusterNodeInfo(id, store, mId, iId, false));
+ candidates.add(new ClusterNodeInfo(id, store, mId, iId, false, invisible));
startTimes.put(id, doc.getStartTime());
}
@@ -596,19 +635,21 @@ public class ClusterNodeInfo {
clusterNodeId = maxId + 1;
}
// No usable existing entry found so create a new entry
- candidates.add(new ClusterNodeInfo(clusterNodeId, store, machineId, instanceId, true));
+ candidates.add(new ClusterNodeInfo(clusterNodeId, store, machineId, instanceId, true, invisible));
}
// use the best candidate
ClusterNodeInfo info = candidates.first();
// and replace with an info matching the current machine and instance id
- info = new ClusterNodeInfo(info.id, store, machineId, instanceId, info.newEntry);
+ info = new ClusterNodeInfo(info.id, store, machineId, instanceId, info.newEntry, invisible);
return new AbstractMap.SimpleImmutableEntry<>(info, startTimes.get(info.getId()));
}
private static void logClusterIdAcquired(ClusterNodeInfo clusterNode,
ClusterNodeInfoDocument before) {
String type = clusterNode.newEntry ? "new" : "existing";
+ type = clusterNode.invisible ? (type + " (invisible)") : type;
+
String machineInfo = clusterNode.machineId;
String instanceInfo = clusterNode.instanceId;
if (before != null) {
@@ -654,7 +695,7 @@ public class ClusterNodeInfo {
// check state of cluster node info
ClusterNodeInfoDocument reread = store.find(Collection.CLUSTER_NODES, key);
if (reread == null) {
- LOG.info("Cluster node info " + key + ": gone; continueing.");
+ LOG.info("Cluster node info " + key + ": gone; continuing.");
return true;
} else {
Long newLeaseEnd = (Long) reread.get(LEASE_END_KEY);
@@ -1098,6 +1139,7 @@ public class ClusterNodeInfo {
UpdateOp update = new UpdateOp("" + id, true);
update.set(LEASE_END_KEY, null);
update.set(STATE, null);
+ update.set(INVISIBLE, false);
store.createOrUpdate(Collection.CLUSTER_NODES, update);
state = NONE;
}
@@ -1114,7 +1156,8 @@ public class ClusterNodeInfo {
"leaseCheckMode: " + leaseCheckMode.name() + ",\n" +
"state: " + state + ",\n" +
"oakVersion: " + OAK_VERSION + ",\n" +
- "formatVersion: " + DocumentNodeStore.VERSION;
+ "formatVersion: " + DocumentNodeStore.VERSION + ",\n" +
+ "invisible: " + invisible;
}
/**
Modified: jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java Thu Sep 26 10:48:14 2019
@@ -157,4 +157,14 @@ public class ClusterNodeInfoDocument ext
public String getLastWrittenRootRev() {
return (String) get(ClusterNodeInfo.LAST_WRITTEN_ROOT_REV_KEY);
}
+
+ /**
+ * Is the cluster node marked as invisible
+ * @return {@code true} if invisible; {@code false}
+ * otherwise.
+ */
+ public boolean isInvisible() {
+ Boolean invisible = (Boolean) get(ClusterNodeInfo.INVISIBLE);
+ return invisible != null ? invisible : false;
+ }
}
Modified: jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBroadcastConfig.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBroadcastConfig.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBroadcastConfig.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBroadcastConfig.java Thu Sep 26 10:48:14 2019
@@ -41,7 +41,7 @@ public class DocumentBroadcastConfig imp
public List<Map<String, String>> getClientInfo() {
ArrayList<Map<String, String>> list = new ArrayList<Map<String, String>>();
for (ClusterNodeInfoDocument doc : ClusterNodeInfoDocument.all(documentNodeStore.getDocumentStore())) {
- if (!doc.isActive()) {
+ if (!doc.isActive() || doc.isInvisible()) {
continue;
}
Object broadcastId = doc.get(DynamicBroadcastConfig.ID);
Modified: jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteService.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteService.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteService.java Thu Sep 26 10:48:14 2019
@@ -372,19 +372,21 @@ public class DocumentDiscoveryLiteServic
for (Iterator<ClusterNodeInfoDocument> it = allClusterNodes.iterator(); it.hasNext();) {
ClusterNodeInfoDocument clusterNode = it.next();
- allNodeIds.put(clusterNode.getClusterId(), clusterNode);
- if (clusterNode.isBeingRecovered()) {
- recoveringNodes.put(clusterNode.getClusterId(), clusterNode);
- } else if (!clusterNode.isActive()) {
- if (hasBacklog(clusterNode)) {
- backlogNodes.put(clusterNode.getClusterId(), clusterNode);
+ if (!clusterNode.isInvisible()) {
+ allNodeIds.put(clusterNode.getClusterId(), clusterNode);
+ if (clusterNode.isBeingRecovered()) {
+ recoveringNodes.put(clusterNode.getClusterId(), clusterNode);
+ } else if (!clusterNode.isActive()) {
+ if (hasBacklog(clusterNode)) {
+ backlogNodes.put(clusterNode.getClusterId(), clusterNode);
+ } else {
+ inactiveNoBacklogNodes.put(clusterNode.getClusterId(), clusterNode);
+ }
+ } else if (clusterNode.getLeaseEndTime() < System.currentTimeMillis()) {
+ activeButTimedOutNodes.put(clusterNode.getClusterId(), clusterNode);
} else {
- inactiveNoBacklogNodes.put(clusterNode.getClusterId(), clusterNode);
+ activeNotTimedOutNodes.put(clusterNode.getClusterId(), clusterNode);
}
- } else if (clusterNode.getLeaseEndTime() < System.currentTimeMillis()) {
- activeButTimedOutNodes.put(clusterNode.getClusterId(), clusterNode);
- } else {
- activeNotTimedOutNodes.put(clusterNode.getClusterId(), clusterNode);
}
}
@@ -471,7 +473,8 @@ public class DocumentDiscoveryLiteServic
return null;
}
- private boolean hasBacklog(ClusterNodeInfoDocument clusterNode) {
+ /** package access only for testing **/
+ boolean hasBacklog(ClusterNodeInfoDocument clusterNode) {
if (logger.isTraceEnabled()) {
logger.trace("hasBacklog: start. clusterNodeId: {}", clusterNode.getClusterId());
}
@@ -653,7 +656,7 @@ public class DocumentDiscoveryLiteServic
* background-read has finished - as it could be waiting for a crashed
* node's recovery to finish - which it can only do by checking the
* lastKnownRevision of the crashed instance - and that check is best done
- * after the background read is just finished (it could optinoally do that
+ * after the background read is just finished (it could optionally do that
* just purely time based as well, but going via a listener is more timely,
* that's why this approach has been chosen).
*/
Modified: jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java Thu Sep 26 10:48:14 2019
@@ -562,7 +562,7 @@ public final class DocumentNodeStore
} else {
clusterNodeInfo = ClusterNodeInfo.getInstance(nonLeaseCheckingStore,
new RecoveryHandlerImpl(nonLeaseCheckingStore, clock, lastRevSeeker),
- null, null, cid);
+ null, null, cid, builder.isClusterInvisible());
checkRevisionAge(nonLeaseCheckingStore, clusterNodeInfo, clock);
}
this.clusterId = clusterNodeInfo.getId();
Modified: jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java Thu Sep 26 10:48:14 2019
@@ -51,7 +51,6 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.document.util.StringValue;
import org.apache.jackrabbit.oak.spi.blob.AbstractBlobStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.apache.jackrabbit.oak.spi.gc.GCMonitor;
import org.apache.jackrabbit.oak.spi.gc.LoggingGCMonitor;
@@ -151,6 +150,7 @@ public class DocumentNodeStoreBuilder<T
private GCMonitor gcMonitor = new LoggingGCMonitor(
LoggerFactory.getLogger(VersionGarbageCollector.class));
private Predicate<Path> nodeCachePredicate = Predicates.alwaysTrue();
+ private boolean clusterInvisible;
/**
* @return a new {@link DocumentNodeStoreBuilder}.
@@ -336,6 +336,18 @@ public class DocumentNodeStoreBuilder<T
return thisBuilder();
}
+ /**
+ * Set the cluster as invisible to the discovery lite service. By default
+ * it is visible.
+ *
+ * @return this
+ * @see DocumentDiscoveryLiteService
+ */
+ public T setClusterInvisible(boolean invisible) {
+ this.clusterInvisible = invisible;
+ return thisBuilder();
+ }
+
public T setCacheSegmentCount(int cacheSegmentCount) {
this.cacheSegmentCount = cacheSegmentCount;
return thisBuilder();
@@ -350,6 +362,10 @@ public class DocumentNodeStoreBuilder<T
return clusterId;
}
+ public boolean isClusterInvisible() {
+ return clusterInvisible;
+ }
+
/**
* Set the maximum delay to write the last revision to the root node. By
* default 1000 (meaning 1 second) is used.
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java Thu Sep 26 10:48:14 2019
@@ -85,7 +85,7 @@ public abstract class BaseDocumentDiscov
*/
class SimplifiedInstance {
- private DocumentDiscoveryLiteService service;
+ DocumentDiscoveryLiteService service;
DocumentNodeStore ns;
private final Descriptors descriptors;
private Map<String, Object> registeredServices;
@@ -152,6 +152,10 @@ public abstract class BaseDocumentDiscov
return Boolean.valueOf(finalStr);
}
+ boolean isInvisible() {
+ return ns.getClusterInfo().isInvisible();
+ }
+
boolean hasActiveIds(String clusterViewStr, int... expected) throws Exception {
return hasIds(clusterViewStr, "active", expected);
}
@@ -464,9 +468,9 @@ public abstract class BaseDocumentDiscov
class ViewExpectation implements Expectation {
- private int[] activeIds;
- private int[] deactivatingIds;
- private int[] inactiveIds;
+ private int[] activeIds = new int[0];
+ private int[] deactivatingIds = new int[0];
+ private int[] inactiveIds = new int[0];
private final SimplifiedInstance discoveryLiteCombo;
private boolean isFinal = true;
@@ -526,7 +530,7 @@ public abstract class BaseDocumentDiscov
if (!discoveryLiteCombo.hasInactiveIds(clusterViewStr, inactiveIds)) {
return "inactiveIds dont match, expected: " + beautify(inactiveIds) + ", got clusterView: " + clusterViewStr;
}
- if (discoveryLiteCombo.isFinal() != isFinal) {
+ if (!discoveryLiteCombo.isInvisible() && discoveryLiteCombo.isFinal() != isFinal) {
return "final flag does not match. expected: " + isFinal + ", but is: " + discoveryLiteCombo.isFinal();
}
return null;
@@ -579,6 +583,12 @@ public abstract class BaseDocumentDiscov
// subsequent tests should get a DocumentDiscoveryLiteService setup from the
// start
DocumentNodeStore createNodeStore(String workingDir) throws SecurityException, Exception {
+ return createNodeStore(workingDir, false);
+ }
+
+ // subsequent tests should get a DocumentDiscoveryLiteService setup from the
+ // start
+ DocumentNodeStore createNodeStore(String workingDir, boolean invisible) throws SecurityException, Exception {
String prevWorkingDir = ClusterNodeInfo.WORKING_DIR;
try {
// ensure that we always get a fresh cluster[node]id
@@ -587,7 +597,8 @@ public abstract class BaseDocumentDiscov
// then create the DocumentNodeStore
DocumentMK mk1 = createMK(
0 /* to make sure the clusterNodes collection is used **/,
- 500 /* asyncDelay: background interval */);
+ 500, /* asyncDelay: background interval */
+ invisible /* cluster node invisibility */);
logger.info("createNodeStore: created DocumentNodeStore with cid=" + mk1.nodeStore.getClusterId() + ", workingDir="
+ workingDir);
@@ -599,12 +610,20 @@ public abstract class BaseDocumentDiscov
}
SimplifiedInstance createInstance() throws Exception {
+ return createInstance(false);
+ }
+
+ SimplifiedInstance createInstance(boolean invisible) throws Exception {
final String workingDir = UUID.randomUUID().toString();
- return createInstance(workingDir);
+ return createInstance(workingDir, invisible);
}
SimplifiedInstance createInstance(String workingDir) throws SecurityException, Exception {
- DocumentNodeStore ns = createNodeStore(workingDir);
+ return createInstance(workingDir, false);
+ }
+
+ SimplifiedInstance createInstance(String workingDir, boolean invisible) throws SecurityException, Exception {
+ DocumentNodeStore ns = createNodeStore(workingDir, invisible);
return createInstance(ns, workingDir);
}
@@ -658,7 +677,9 @@ public abstract class BaseDocumentDiscov
final List<Integer> activeIds = new LinkedList<Integer>();
for (Iterator<SimplifiedInstance> it = instances.iterator(); it.hasNext();) {
SimplifiedInstance anInstance = it.next();
- activeIds.add(anInstance.ns.getClusterId());
+ if (!anInstance.isInvisible()) {
+ activeIds.add(anInstance.ns.getClusterId());
+ }
}
logger.info("checkFiestaState: checking state. expected active: "+activeIds+", inactive: "+inactiveIds);
for (Iterator<SimplifiedInstance> it = instances.iterator(); it.hasNext();) {
@@ -695,6 +716,11 @@ public abstract class BaseDocumentDiscov
}
DocumentMK createMK(int clusterId, int asyncDelay) {
+ return createMK(clusterId, asyncDelay, false);
+ }
+
+
+ DocumentMK createMK(int clusterId, int asyncDelay, boolean invisible) {
if (MONGO_DB) {
MongoConnection connection = connectionFactory.getConnection();
return register(new DocumentMK.Builder()
@@ -708,13 +734,14 @@ public abstract class BaseDocumentDiscov
if (bs == null) {
bs = new MemoryBlobStore();
}
- return createMK(clusterId, asyncDelay, ds, bs);
+ return createMK(clusterId, asyncDelay, ds, bs, invisible);
}
}
- DocumentMK createMK(int clusterId, int asyncDelay, DocumentStore ds, BlobStore bs) {
- return register(new DocumentMK.Builder().setDocumentStore(ds).setBlobStore(bs).setClusterId(clusterId).setLeaseCheckMode(LeaseCheckMode.DISABLED)
- .setAsyncDelay(asyncDelay).open());
+ DocumentMK createMK(int clusterId, int asyncDelay, DocumentStore ds, BlobStore bs, boolean invisible) {
+ return register(new DocumentMK.Builder().setDocumentStore(ds).setBlobStore(bs).setClusterId(clusterId).setClusterInvisible(invisible)
+ .setLeaseCheckMode(LeaseCheckMode.DISABLED)
+ .setAsyncDelay(asyncDelay).open());
}
DocumentMK register(DocumentMK mk) {
@@ -723,6 +750,20 @@ public abstract class BaseDocumentDiscov
}
/**
+ * Probability of invisible instance at 20%
+ * @param random
+ * @return
+ */
+ boolean isInvisibleInstance(Random random) {
+ boolean invisible = false;
+ double invisibleProb = random.nextDouble();
+ if (invisibleProb <= 0.2) {
+ invisible = true;
+ }
+ return invisible;
+ }
+
+ /**
* This test creates a large number of documentnodestores which it starts,
* runs, stops in a random fashion, always testing to make sure the
* clusterView is correct
@@ -758,7 +799,8 @@ public abstract class BaseDocumentDiscov
logger.info("Case 0 - reactivated instance " + cid + ", workingDir=" + reactivatedWorkingDir);
workingDir = reactivatedWorkingDir;
logger.info("Case 0: creating instance");
- final SimplifiedInstance newInstance = createInstance(workingDir);
+
+ final SimplifiedInstance newInstance = createInstance(workingDir, isInvisibleInstance(random));
newInstance.setLeastTimeout(5000, 1000);
newInstance.startSimulatingWrites(500);
logger.info("Case 0: created instance: " + newInstance.ns.getClusterId());
@@ -779,7 +821,7 @@ public abstract class BaseDocumentDiscov
// creates a new instance
if (instances.size() < MAX_NUM_INSTANCES) {
logger.info("Case 1: creating instance");
- final SimplifiedInstance newInstance = createInstance(workingDir);
+ final SimplifiedInstance newInstance = createInstance(workingDir, isInvisibleInstance(random));
newInstance.setLeastTimeout(5000, 1000);
newInstance.startSimulatingWrites(500);
logger.info("Case 1: created instance: " + newInstance.ns.getClusterId());
@@ -803,7 +845,9 @@ public abstract class BaseDocumentDiscov
final SimplifiedInstance instance = instances.remove(random.nextInt(instances.size()));
assertNotNull(instance.workingDir);
logger.info("Case 3: Shutdown instance: " + instance.ns.getClusterId());
- inactiveIds.put(instance.ns.getClusterId(), instance.workingDir);
+ if (!instance.isInvisible()) {
+ inactiveIds.put(instance.ns.getClusterId(), instance.workingDir);
+ }
instance.shutdown();
}
break;
@@ -817,7 +861,9 @@ public abstract class BaseDocumentDiscov
final SimplifiedInstance instance = instances.remove(random.nextInt(instances.size()));
assertNotNull(instance.workingDir);
logger.info("Case 4: Crashing instance: " + instance.ns.getClusterId());
- inactiveIds.put(instance.ns.getClusterId(), instance.workingDir);
+ if (!instance.isInvisible()) {
+ inactiveIds.put(instance.ns.getClusterId(), instance.workingDir);
+ }
instance.addNode("/" + instance.ns.getClusterId() + "/stuffForRecovery/" + random.nextInt(10000));
instance.crash();
}
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java Thu Sep 26 10:48:14 2019
@@ -144,7 +144,26 @@ public class ClusterInfoTest {
}
@Test
- public void useAbandoned() throws InterruptedException {
+ public void useAbandonedStdToStd() throws InterruptedException {
+ useAbandoned(false, false);
+ }
+
+ @Test
+ public void useAbandonedInvisibleToStd() throws InterruptedException {
+ useAbandoned(true, false);
+ }
+
+ @Test
+ public void useAbandonedStdToInvisible() throws InterruptedException {
+ useAbandoned(false, true);
+ }
+
+ @Test
+ public void useAbandonedInvisibleToInvisible() throws InterruptedException {
+ useAbandoned(true, true);
+ }
+
+ public void useAbandoned(boolean firstInvisible, boolean secondInvisible) throws InterruptedException {
Clock clock = new Clock.Virtual();
clock.waitUntil(System.currentTimeMillis());
ClusterNodeInfo.setClock(clock);
@@ -155,6 +174,7 @@ public class ClusterInfoTest {
clock(clock).
setAsyncDelay(0).
setLeaseCheckMode(LeaseCheckMode.DISABLED).
+ setClusterInvisible(firstInvisible).
getNodeStore();
DocumentStore ds = ns1.getDocumentStore();
@@ -163,6 +183,7 @@ public class ClusterInfoTest {
ClusterNodeInfoDocument cnid = ds.find(Collection.CLUSTER_NODES, "" + cid);
assertNotNull(cnid);
assertEquals(ClusterNodeState.ACTIVE.toString(), cnid.get(ClusterNodeInfo.STATE));
+ assertEquals("Cluster should have been " + firstInvisible, firstInvisible, cnid.isInvisible());
ns1.dispose();
long waitFor = 2000;
@@ -179,9 +200,16 @@ public class ClusterInfoTest {
clock(clock).
setAsyncDelay(0).
setLeaseCheckMode(LeaseCheckMode.DISABLED).
+ setClusterInvisible(secondInvisible).
getNodeStore();
assertEquals("should have re-used existing cluster id", cid, ns1.getClusterId());
+
+ cnid = ds.find(Collection.CLUSTER_NODES, "" + cid);
+ assertNotNull(cnid);
+ assertEquals(ClusterNodeState.ACTIVE.toString(), cnid.get(ClusterNodeInfo.STATE));
+ assertEquals("Cluster should have been " + secondInvisible, secondInvisible, cnid.isInvisible());
+
ns1.dispose();
}
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java Thu Sep 26 10:48:14 2019
@@ -82,9 +82,9 @@ public class ClusterNodeInfoComparatorTe
private ClusterNodeInfo newClusterNodeInfo(int id, String instanceId) {
try {
Constructor<ClusterNodeInfo> ctr = ClusterNodeInfo.class.getDeclaredConstructor(
- int.class, DocumentStore.class, String.class, String.class, boolean.class);
+ int.class, DocumentStore.class, String.class, String.class, boolean.class, boolean.class);
ctr.setAccessible(true);
- return ctr.newInstance(id, store, MACHINE_ID, instanceId, true);
+ return ctr.newInstance(id, store, MACHINE_ID, instanceId, true, false);
} catch (Exception e) {
fail(e.getMessage());
}
Added: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java?rev=1867571&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java (added)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java Thu Sep 26 10:48:14 2019
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.util.List;
+
+import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
+import org.junit.Test;
+
+import static org.apache.jackrabbit.oak.plugins.document.RecoveryHandler.NOOP;
+import static org.hamcrest.Matchers.hasSize;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class ClusterNodeInfoDocumentTest {
+
+ private DocumentStore store = new MemoryDocumentStore();
+
+ @Test
+ public void invisibleTrue() {
+ assertFalse(createInactive(true).isInvisible());
+ }
+
+ @Test
+ public void invisibleFalse() {
+ assertFalse(createInactive(false).isInvisible());
+ }
+
+ @Test
+ public void compatibility1_18() {
+ ClusterNodeInfoDocument doc = createInactive(false);
+ // remove invisible field introduced after 1.18
+ UpdateOp op = new UpdateOp(String.valueOf(doc.getClusterId()), false);
+ op.remove(ClusterNodeInfo.INVISIBLE);
+ assertNotNull(store.findAndUpdate(Collection.CLUSTER_NODES, op));
+ List<ClusterNodeInfoDocument> docs = ClusterNodeInfoDocument.all(store);
+ assertThat(docs, hasSize(1));
+ assertFalse(docs.get(0).isInvisible());
+ }
+
+ private ClusterNodeInfoDocument createInactive(boolean invisible) {
+ int clusterId = 1;
+ ClusterNodeInfo.getInstance(store, NOOP, "machineId", "instanceId", clusterId, invisible).dispose();
+ return store.find(Collection.CLUSTER_NODES, String.valueOf(clusterId));
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java Thu Sep 26 10:48:14 2019
@@ -28,11 +28,14 @@ import java.util.concurrent.atomic.Atomi
import java.util.function.Function;
import java.util.stream.Collectors;
+import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
import org.apache.jackrabbit.oak.stats.Clock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
@@ -44,11 +47,22 @@ import static org.junit.Assert.assertTha
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+@RunWith(Parameterized.class)
public class ClusterNodeInfoTest {
private Clock clock;
private TestStore store;
private FailureHandler handler = new FailureHandler();
+ private boolean invisible;
+
+ public ClusterNodeInfoTest(boolean invisible) {
+ this.invisible = invisible;
+ }
+
+ @Parameterized.Parameters(name="{index}: ({0})")
+ public static List<Boolean> fixtures() {
+ return Lists.newArrayList(false, true);
+ }
@Before
public void before() throws Exception {
@@ -551,7 +565,7 @@ public class ClusterNodeInfoTest {
private ClusterNodeInfo newClusterNodeInfo(int clusterId,
String instanceId) {
ClusterNodeInfo info = ClusterNodeInfo.getInstance(store,
- new SimpleRecoveryHandler(), null, instanceId, clusterId);
+ new SimpleRecoveryHandler(), null, instanceId, clusterId, invisible);
info.setLeaseFailureHandler(handler);
return info;
}
Added: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java?rev=1867571&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java (added)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java Thu Sep 26 10:48:14 2019
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.util.UUID;
+import java.util.concurrent.Semaphore;
+
+import junitx.util.PrivateAccessor;
+import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.AdditionalAnswers.delegatesTo;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+
+public class DocumentDiscoveryLiteInvisibleServiceCrashTest
+ extends BaseDocumentDiscoveryLiteServiceTest {
+
+ private static final int TEST_WAIT_TIMEOUT = 10000;
+
+ private Clock clock;
+ private DocumentStore store;
+ private String wd1;
+ private DocumentNodeStore ns1;
+ private String wd2;
+ private DocumentNodeStore ns2;
+
+ @Before
+ public void setup() throws Exception {
+ clock = new Clock.Virtual();
+ clock.waitUntil(System.currentTimeMillis());
+ ClusterNodeInfo.setClock(clock);
+ store = new MemoryDocumentStore();
+ wd1 = UUID.randomUUID().toString();
+ wd2 = UUID.randomUUID().toString();
+ }
+
+ @After
+ public void reset() {
+ ns1.dispose();
+ ns2.dispose();
+ ClusterNodeInfo.resetClockToDefault();
+ }
+
+ @Test
+ public void testTwoNodesWithCrashAndLongduringRecovery() throws Throwable {
+ doTestTwoNodesWithCrashAndLongduringDeactivation(false);
+ }
+
+ @Test
+ public void testTwoNodesWithCrashAndLongduringRecoveryAndBacklog() throws Throwable {
+ doTestTwoNodesWithCrashAndLongduringDeactivation(true);
+ }
+
+ private void doTestTwoNodesWithCrashAndLongduringDeactivation(boolean withBacklog) throws Throwable {
+ ns1 = newDocumentNodeStore(store, wd1);
+ SimplifiedInstance s1 = createInstance(ns1, wd1);
+ ViewExpectation e1 = new ViewExpectation(s1);
+ e1.setActiveIds(ns1.getClusterId());
+ waitFor(e1, TEST_WAIT_TIMEOUT, "first should see itself active");
+ ns1.runBackgroundOperations();
+
+ ns2 = newDocumentNodeStore(store, wd2, true);
+ SimplifiedInstance s2 = createInstance(ns2, wd2);
+ ViewExpectation e2 = new ViewExpectation(s2);
+ e2.setActiveIds(ns1.getClusterId());
+ waitFor(e2, TEST_WAIT_TIMEOUT, "second should see only first active");
+ ns2.runBackgroundOperations();
+
+ ns1.runBackgroundReadOperations();
+ // now ns1 should also see both active
+ ViewExpectation e3 = new ViewExpectation(s1);
+ e3.setActiveIds(ns1.getClusterId());
+ waitFor(e3, TEST_WAIT_TIMEOUT, "first should see only itself as active");
+
+ // before crashing s2, make sure that s1's lastRevRecovery thread
+ // doesn't run
+ s1.stopLastRevThread();
+ if (withBacklog) {
+ // if we want to do backlog testing, then s2 should write
+ // something
+ // before it crashes, so here it comes:
+ s2.addNode("/foo/bar");
+ s2.setProperty("/foo/bar", "prop", "value");
+ }
+
+ // then wait 2 sec
+ clock.waitUntil(clock.getTime() + 2000);
+
+ s2.crash();
+
+ // then wait 2 sec
+ clock.waitUntil(clock.getTime() + 2000);
+
+ // at this stage, while s2 has crashed, we have stopped s1's
+ // lastRevRecoveryThread, so we should still see both as active
+ logger.info(s1.getClusterViewStr());
+ final ViewExpectation expectation1AfterCrashBeforeLastRevRecovery = new ViewExpectation(s1);
+ expectation1AfterCrashBeforeLastRevRecovery.setActiveIds(ns1.getClusterId());
+ waitFor(expectation1AfterCrashBeforeLastRevRecovery, TEST_WAIT_TIMEOUT, "first should still see only itself as active");
+
+ // the next part is a bit tricky: we want to fine-control the
+ // lastRevRecoveryThread's acquire/release locking.
+ // the chosen way to do this is to make heavy use of mockito and two
+ // semaphores:
+ // when acquireRecoveryLock is called, that thread should wait for the
+ // waitBeforeLocking semaphore to be released
+ final MissingLastRevSeeker missingLastRevUtil = (MissingLastRevSeeker) PrivateAccessor
+ .getField(s1.ns.getLastRevRecoveryAgent(), "missingLastRevUtil");
+ assertNotNull(missingLastRevUtil);
+ MissingLastRevSeeker mockedLongduringMissingLastRevUtil = mock(MissingLastRevSeeker.class, delegatesTo(missingLastRevUtil));
+ final Semaphore waitBeforeLocking = new Semaphore(0);
+ doAnswer(new Answer<Boolean>() {
+ @Override
+ public Boolean answer(InvocationOnMock invocation) throws Throwable {
+ logger.info("going to waitBeforeLocking");
+ waitBeforeLocking.acquire();
+ logger.info("done with waitBeforeLocking");
+ return missingLastRevUtil.acquireRecoveryLock((Integer) invocation.getArguments()[0],
+ (Integer) invocation.getArguments()[1]);
+ }
+ }).when(mockedLongduringMissingLastRevUtil).acquireRecoveryLock(anyInt(), anyInt());
+ PrivateAccessor.setField(s1.ns.getLastRevRecoveryAgent(), "missingLastRevUtil", mockedLongduringMissingLastRevUtil);
+
+ // so let's start the lastRevThread again and wait for that
+ // waitBeforeLocking semaphore to be hit
+ s1.startLastRevThread();
+ waitFor(new Expectation() {
+
+ @Override
+ public String fulfilled() throws Exception {
+ if (!waitBeforeLocking.hasQueuedThreads()) {
+ return "no thread queued";
+ }
+ return null;
+ }
+
+ }, TEST_WAIT_TIMEOUT, "lastRevRecoveryThread should acquire a lock");
+
+ // at this stage the crashed s2 is still not in recovery mode, so let's
+ // check:
+ logger.info(s1.getClusterViewStr());
+ final ViewExpectation expectation1AfterCrashBeforeLastRevRecoveryLocking = new ViewExpectation(s1);
+ expectation1AfterCrashBeforeLastRevRecoveryLocking.setActiveIds(ns1.getClusterId());
+ waitFor(expectation1AfterCrashBeforeLastRevRecoveryLocking, TEST_WAIT_TIMEOUT, "first should still see itself as active");
+
+ // one thing, before we let the waitBeforeLocking go, setup the release
+ // semaphore/mock:
+ final Semaphore waitBeforeUnlocking = new Semaphore(0);
+ Mockito.doAnswer(new Answer<Void>() {
+ public Void answer(InvocationOnMock invocation) throws InterruptedException {
+ logger.info("Going to waitBeforeUnlocking");
+ waitBeforeUnlocking.acquire();
+ logger.info("Done with waitBeforeUnlocking");
+ missingLastRevUtil.releaseRecoveryLock(
+ (Integer) invocation.getArguments()[0],
+ (Boolean) invocation.getArguments()[1]);
+ return null;
+ }
+ }).when(mockedLongduringMissingLastRevUtil).releaseRecoveryLock(anyInt(), anyBoolean());
+
+ // let go (or tschaedere loh)
+ waitBeforeLocking.release();
+
+ // then, right after we let the waitBeforeLocking semaphore go, we
+ // should see s2 in recovery mode
+ final ViewExpectation expectation1AfterCrashWhileLastRevRecoveryLocking = new ViewExpectation(s1);
+ expectation1AfterCrashWhileLastRevRecoveryLocking.setActiveIds(ns1.getClusterId());
+ waitFor(expectation1AfterCrashWhileLastRevRecoveryLocking, TEST_WAIT_TIMEOUT, "first should still see itself as active");
+
+ // ok, meanwhile, the lastRevRecoveryAgent should have hit the ot
+ waitFor(new Expectation() {
+
+ @Override
+ public String fulfilled() throws Exception {
+ if (!waitBeforeUnlocking.hasQueuedThreads()) {
+ return "no thread queued";
+ }
+ return null;
+ }
+
+ }, TEST_WAIT_TIMEOUT, "lastRevRecoveryThread should want to release a lock");
+
+ // so then, we should still see the same state
+ waitFor(expectation1AfterCrashWhileLastRevRecoveryLocking, TEST_WAIT_TIMEOUT, "first should still see itself as active");
+
+ logger.info("Waiting 2 sec");
+ clock.waitUntil(clock.getTime() + 2000);
+ logger.info("Waiting done");
+
+ // first, lets check to see what the view looks like - should be
+ // unchanged:
+ waitFor(expectation1AfterCrashWhileLastRevRecoveryLocking, TEST_WAIT_TIMEOUT, "first should still see itself as active");
+
+ // let waitBeforeUnlocking go
+ logger.info("releasing waitBeforeUnlocking, state: " + s1.getClusterViewStr());
+ waitBeforeUnlocking.release();
+ logger.info("released waitBeforeUnlocking");
+
+ if (!withBacklog) {
+ final ViewExpectation expectationWithoutBacklog = new ViewExpectation(s1);
+ expectationWithoutBacklog.setActiveIds(ns1.getClusterId());
+ waitFor(expectationWithoutBacklog, TEST_WAIT_TIMEOUT, "only first as active");
+ waitFor(() -> {
+ if (!getLatestClusterInfo(ns2.getClusterId(), ns2).isActive() &&
+ !getLatestClusterInfo(ns2.getClusterId(), ns2).isBeingRecovered()) {
+ return null;
+ } else {
+ return "Still not inactive";
+ }
+ }, TEST_WAIT_TIMEOUT, "Second cluster should be inactive");
+ } else {
+ // wait just 2 sec to see if the bgReadThread is really stopped
+ logger.info("sleeping 2 sec");
+ clock.waitUntil(clock.getTime() + 2000);
+ logger.info("sleeping 2 sec done, state: " + s1.getClusterViewStr());
+
+ // when that's the case, check the view - it should now be in a
+ // special 'final=false' mode
+ final ViewExpectation expectationBeforeBgRead = new ViewExpectation(s1);
+ expectationBeforeBgRead.setActiveIds(ns1.getClusterId());
+ waitFor(() -> {
+ ClusterNodeInfoDocument latestClusterInfo = getLatestClusterInfo(ns2.getClusterId(), ns2);
+ boolean hasBacklog = s1.service.hasBacklog(latestClusterInfo);
+ if (hasBacklog) {
+ return null;
+ } else {
+ return "No Backlog";
+ }
+ }, TEST_WAIT_TIMEOUT, "Second cluster should have backlogs");
+ waitFor(expectationBeforeBgRead, TEST_WAIT_TIMEOUT, "first should only see itself after shutdown");
+
+ // ook, now we explicitly do a background read to get out of the
+ // backlog situation
+ ns1.runBackgroundReadOperations();
+
+ final ViewExpectation expectationAfterBgRead = new ViewExpectation(s1);
+ expectationAfterBgRead.setActiveIds(ns1.getClusterId());
+ waitFor(expectationAfterBgRead, TEST_WAIT_TIMEOUT, "we should see s1 as only active");
+ waitFor(() -> {
+ ClusterNodeInfoDocument latestClusterInfo = getLatestClusterInfo(ns2.getClusterId(), ns2);
+ boolean hasBacklog = s1.service.hasBacklog(latestClusterInfo);
+ if (!hasBacklog) {
+ return null;
+ } else {
+ return "Still has Backlog";
+ }
+ }, TEST_WAIT_TIMEOUT, "Second cluster should not have backlog any longer");
+ }
+ }
+
+ private static ClusterViewDocument read(DocumentNodeStore documentNodeStore) {
+ DocumentStore documentStore = documentNodeStore.getDocumentStore();
+ Document doc = documentStore.find(Collection.SETTINGS, "clusterView",
+ -1 /* -1; avoid caching */);
+ if (doc == null) {
+ return null;
+ } else {
+ ClusterViewDocument clusterView = new ClusterViewDocument(doc);
+ if (clusterView.isValid()) {
+ return clusterView;
+ } else {
+ return null;
+ }
+ }
+ }
+
+ private ClusterNodeInfoDocument getLatestClusterInfo(int id, DocumentNodeStore nodeStore) {
+ for (ClusterNodeInfoDocument doc : ClusterNodeInfoDocument.all(nodeStore.getDocumentStore())) {
+ int cId = doc.getClusterId();
+ if (cId == id) {
+ return doc;
+ }
+ }
+ return null;
+ }
+
+ private DocumentNodeStore newDocumentNodeStore(DocumentStore store,
+ String workingDir) {
+ return newDocumentNodeStore(store, workingDir, false);
+ }
+
+ private DocumentNodeStore newDocumentNodeStore(DocumentStore store,
+ String workingDir, boolean invisible) {
+ String prevWorkingDir = ClusterNodeInfo.WORKING_DIR;
+ try {
+ // ensure that we always get a fresh cluster[node]id
+ ClusterNodeInfo.WORKING_DIR = workingDir;
+
+ return new DocumentMK.Builder()
+ .clock(clock)
+ .setAsyncDelay(0)
+ .setDocumentStore(store)
+ .setLeaseCheckMode(LeaseCheckMode.DISABLED)
+ .setClusterInvisible(invisible)
+ .getNodeStore();
+ } finally {
+ ClusterNodeInfo.WORKING_DIR = prevWorkingDir;
+ }
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceIT.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceIT.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceIT.java Thu Sep 26 10:48:14 2019
@@ -31,7 +31,7 @@ public class DocumentDiscoveryLiteServic
@Test
public void testLargeStartStopFiesta() throws Throwable {
logger.info("testLargeStartStopFiesta: start, seed="+SEED);
- final int LOOP_CNT = 50; // with too many loops have also seen mongo
+ final int LOOP_CNT = 40; // with too many loops have also seen mongo
// connections becoming starved thus test
// failed
doStartStopFiesta(LOOP_CNT);
Modified: jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java (original)
+++ jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java Thu Sep 26 10:48:14 2019
@@ -57,6 +57,14 @@ public class DocumentDiscoveryLiteServic
}
@Test
+ public void testOneInvisibleNode() throws Exception {
+ final SimplifiedInstance s1 = createInstance(true);
+ final ViewExpectation expectation = new ViewExpectation(s1);
+ expectation.setActiveIds(new int[0]);
+ waitFor(expectation, 2000, "no one is active");
+ }
+
+ @Test
public void testTwoNodesWithCleanShutdown() throws Exception {
final SimplifiedInstance s1 = createInstance();
final SimplifiedInstance s2 = createInstance();
@@ -75,6 +83,23 @@ public class DocumentDiscoveryLiteServic
}
@Test
+ public void testTwoNodesWithInvisibleCleanShutdown() throws Exception {
+ final SimplifiedInstance s1 = createInstance(true);
+ final SimplifiedInstance s2 = createInstance();
+ final ViewExpectation expectation1 = new ViewExpectation(s1);
+ final ViewExpectation expectation2 = new ViewExpectation(s2);
+ expectation1.setActiveIds(s2.ns.getClusterId());
+ expectation2.setActiveIds(s2.ns.getClusterId());
+ waitFor(expectation1, 2000, "Only second is active");
+ waitFor(expectation2, 2000, "Second should not see first as active");
+
+ s1.shutdown();
+ final ViewExpectation expectation1AfterShutdown = new ViewExpectation(s2);
+ expectation1AfterShutdown.setActiveIds(s2.ns.getClusterId());
+ waitFor(expectation1AfterShutdown, 2000, "no one is active after shutdown");
+ }
+
+ @Test
public void testTwoNodesWithCrash() throws Throwable {
final SimplifiedInstance s1 = createInstance();
final SimplifiedInstance s2 = createInstance();
@@ -93,6 +118,24 @@ public class DocumentDiscoveryLiteServic
waitFor(expectation1AfterShutdown, 4000, "first should only see itself after shutdown");
}
+ @Test
+ public void testTwoNodesInvisibleWithCrash() throws Throwable {
+ final SimplifiedInstance s1 = createInstance(true);
+ final SimplifiedInstance s2 = createInstance();
+ final ViewExpectation expectation1 = new ViewExpectation(s1);
+ final ViewExpectation expectation2 = new ViewExpectation(s2);
+ expectation1.setActiveIds(s2.ns.getClusterId());
+ expectation2.setActiveIds(s2.ns.getClusterId());
+ waitFor(expectation1, 2000, "first should see only second as active");
+ waitFor(expectation2, 2000, "second should not see first as active");
+
+ s1.crash();
+
+ final ViewExpectation expectation1AfterShutdown = new ViewExpectation(s1);
+ expectation1AfterShutdown.setActiveIds(s2.ns.getClusterId());
+ waitFor(expectation1AfterShutdown, 4000, "first should only see itself after shutdown");
+ }
+
/**
* This test creates a large number of documentnodestores which it starts,
* runs, stops in a random fashion, always testing to make sure the