You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2015/09/22 20:15:33 UTC
svn commit: r1704684 - in /jackrabbit/oak/branches/1.2: ./
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/
Author: reschke
Date: Tue Sep 22 18:15:30 2015
New Revision: 1704684
URL: http://svn.apache.org/viewvc?rev=1704684&view=rev
Log:
OAK-3435: LastRevRecovery: log failures to update ClusterInfo, keep track which node is running the recovery, improve logging (ported to 1.2)
Modified:
jackrabbit/oak/branches/1.2/ (props changed)
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java
Propchange: jackrabbit/oak/branches/1.2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 22 18:15:30 2015
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681955,1682042,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618,1684820
,1684868,1685023,1685075,1685370,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693209,1693421,1693525-1693526,1694007,1694393-1694394,1695050,1695122,1695280,1695299,1695457,1695482,1695507,1695521,1695540,1696194,1696242,1696285,1696578,1696759,1696916,1697363,1697373,1697410,1697
582,1697589,1697616,1697672,1700191,1700231,1700397,1700403,1700506,1700571,1700727,1700749,1700769,1700775,1701065,1701619,1701733,1701743,1701750,1701768,1701806,1701810,1701814,1701948,1701955,1701959,1701986,1702022,1702272,1702387,1702405,1702423,1702860,1702942,1702960,1703212,1703382,1703395,1703411,1703428,1703430,1703568,1703592,1703858,1703878,1704285,1704457,1704490,1704614,1704636
+/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681955,1682042,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618,1684820
,1684868,1685023,1685075,1685370,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693209,1693421,1693525-1693526,1694007,1694393-1694394,1695050,1695122,1695280,1695299,1695457,1695482,1695507,1695521,1695540,1696194,1696242,1696285,1696578,1696759,1696916,1697363,1697373,1697410,1697
582,1697589,1697616,1697672,1700191,1700231,1700397,1700403,1700506,1700571,1700727,1700749,1700769,1700775,1701065,1701619,1701733,1701743,1701750,1701768,1701806,1701810,1701814,1701948,1701955,1701959,1701986,1702022,1702272,1702387,1702405,1702423,1702860,1702942,1702960,1703212,1703382,1703395,1703411,1703428,1703430,1703568,1703592,1703858,1703878,1704285,1704457,1704490,1704614,1704636,1704670
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java?rev=1704684&r1=1704683&r2=1704684&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java Tue Sep 22 18:15:30 2015
@@ -112,6 +112,11 @@ public class ClusterNodeInfo {
}
/**
+ * Flag indicating which cluster node is running the recovery.
+ */
+ public static final String REV_RECOVERY_BY = "recoveryBy";
+
+ /**
* Additional info, such as the process id, for support.
*/
private static final String INFO_KEY = "info";
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java?rev=1704684&r1=1704683&r2=1704684&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java Tue Sep 22 18:15:30 2015
@@ -32,6 +32,7 @@ import java.util.Map;
import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
@@ -261,7 +262,9 @@ public class LastRevRecoveryAgent {
* @return the number of restored nodes
*/
private int recoverCandidates(final int clusterId, final long startTime) {
- boolean lockAcquired = missingLastRevUtil.acquireRecoveryLock(clusterId);
+
+ int myClusterId = nodeStore.getClusterId();
+ boolean lockAcquired = missingLastRevUtil.acquireRecoveryLock(clusterId, myClusterId);
//TODO What if recovery is being performed for current clusterNode by some other node
//should we halt the startup
@@ -272,7 +275,8 @@ public class LastRevRecoveryAgent {
}
Iterable<NodeDocument> suspects = missingLastRevUtil.getCandidates(startTime);
- log.debug("Performing Last Revision recovery for cluster {}", clusterId);
+
+ log.info("Performing Last Revision Recovery for clusterNodeId {}", clusterId);
try {
return recover(suspects.iterator(), clusterId);
@@ -327,11 +331,12 @@ public class LastRevRecoveryAgent {
return missingLastRevUtil.isRecoveryNeeded(nodeStore.getClock().getTime());
}
- public void performRecoveryIfNeeded(){
- if(isRecoveryNeeded()){
+ public void performRecoveryIfNeeded() {
+ if (isRecoveryNeeded()) {
List<Integer> clusterIds = getRecoveryCandidateNodes();
- log.info("Starting last revision recovery for following clusterId {}", clusterIds);
- for(int clusterId : clusterIds){
+ log.info("ClusterNodeId [{}] starting Last Revision Recovery for clusterNodeId(s) {}", nodeStore.getClusterId(),
+ clusterIds);
+ for (int clusterId : clusterIds) {
recover(clusterId);
}
}
@@ -343,29 +348,34 @@ public class LastRevRecoveryAgent {
* @return the recovery candidate nodes
*/
public List<Integer> getRecoveryCandidateNodes() {
+
Iterable<ClusterNodeInfoDocument> clusters = missingLastRevUtil.getAllClusters();
List<Integer> candidateClusterNodes = Lists.newArrayList();
+ List<String> beingRecoveredRightNow = Lists.newArrayList();
for (ClusterNodeInfoDocument nodeInfo : clusters) {
- if (isRecoveryNeeded(nodeInfo)) {
- candidateClusterNodes.add(Integer.valueOf(nodeInfo.getId()));
+ String id = nodeInfo.getId();
+ if (nodeInfo.isBeingRecovered()) {
+ Long recoveredBy = (Long) nodeInfo.get(ClusterNodeInfo.REV_RECOVERY_BY);
+ beingRecoveredRightNow.add(nodeInfo == null ? id : String.format("%s (by %d)", id, recoveredBy));
+ } else if (isRecoveryNeeded(nodeInfo)) {
+ candidateClusterNodes.add(Integer.valueOf(id));
}
}
+ if (!beingRecoveredRightNow.isEmpty()) {
+ log.info("Active cluster nodes already in the process of being recovered: " + beingRecoveredRightNow);
+ }
+
return candidateClusterNodes;
}
- private boolean isRecoveryNeeded(ClusterNodeInfoDocument nodeInfo) {
- if (nodeInfo != null) {
- // Check if _lastRev recovery needed for this cluster node
- // state is Active && currentTime past the leaseEnd time && recoveryLock not held by someone
- if (nodeInfo.isActive()
- && nodeStore.getClock().getTime() > nodeInfo.getLeaseEndTime()
- && !nodeInfo.isBeingRecovered()) {
- return true;
- }
- }
- return false;
+ /**
+ * Check if _lastRev recovery needed for this cluster node state is Active
+ * && currentTime past the leaseEnd time && recoveryLock not held by someone
+ */
+ private boolean isRecoveryNeeded(@Nonnull ClusterNodeInfoDocument nodeInfo) {
+ return nodeInfo.isActive() && nodeStore.getClock().getTime() > nodeInfo.getLeaseEndTime() && !nodeInfo.isBeingRecovered();
}
private static class ClusterPredicate implements Predicate<Revision> {
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java?rev=1704684&r1=1704683&r2=1704684&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java Tue Sep 22 18:15:30 2015
@@ -21,6 +21,8 @@ package org.apache.jackrabbit.oak.plugin
import org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.RecoverLockState;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
@@ -30,10 +32,14 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getSelectedDocuments;
/**
- * Utils to retrieve _lastRev missing update candidates.
+ * Utilities to retrieve _lastRev missing update candidates.
*/
public class MissingLastRevSeeker {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MissingLastRevSeeker.class);
+
protected final String ROOT_PATH = "/";
+
private final DocumentStore store;
public MissingLastRevSeeker(DocumentStore store) {
@@ -80,20 +86,35 @@ public class MissingLastRevSeeker {
});
}
- public boolean acquireRecoveryLock(int clusterId){
- //This approach has a race condition where two different cluster nodes
- //can acquire the lock simultaneously.
- UpdateOp update = new UpdateOp(Integer.toString(clusterId), true);
- update.set(ClusterNodeInfo.REV_RECOVERY_LOCK, RecoverLockState.ACQUIRED.name());
- store.createOrUpdate(Collection.CLUSTER_NODES, update);
- return true;
- }
-
- public void releaseRecoveryLock(int clusterId){
- UpdateOp update = new UpdateOp(Integer.toString(clusterId), true);
- update.set(ClusterNodeInfo.REV_RECOVERY_LOCK, null);
- update.set(ClusterNodeInfo.STATE, null);
- store.createOrUpdate(Collection.CLUSTER_NODES, update);
+ public boolean acquireRecoveryLock(int clusterId, int recoveredBy) {
+ try {
+ // This approach has a race condition where two different cluster
+ // nodes
+ // can acquire the lock simultaneously.
+ UpdateOp update = new UpdateOp(Integer.toString(clusterId), true);
+ update.set(ClusterNodeInfo.REV_RECOVERY_LOCK, RecoverLockState.ACQUIRED.name());
+ if (recoveredBy != 0) {
+ update.set(ClusterNodeInfo.REV_RECOVERY_BY, recoveredBy);
+ }
+ store.createOrUpdate(Collection.CLUSTER_NODES, update);
+ return true;
+ } catch (RuntimeException ex) {
+ LOG.error("Failed to acquire the recovery lock for clusterNodeId " + clusterId, ex);
+ throw (ex);
+ }
+ }
+
+ public void releaseRecoveryLock(int clusterId) {
+ try {
+ UpdateOp update = new UpdateOp(Integer.toString(clusterId), true);
+ update.set(ClusterNodeInfo.REV_RECOVERY_LOCK, null);
+ update.set(ClusterNodeInfo.REV_RECOVERY_BY, null);
+ update.set(ClusterNodeInfo.STATE, null);
+ store.createOrUpdate(Collection.CLUSTER_NODES, update);
+ } catch (RuntimeException ex) {
+ LOG.error("Failed to release the recovery lock for clusterNodeId " + clusterId, ex);
+ throw (ex);
+ }
}
public NodeDocument getRoot() {
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java?rev=1704684&r1=1704683&r2=1704684&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java Tue Sep 22 18:15:30 2015
@@ -73,7 +73,7 @@ public class MongoMissingLastRevSeeker e
}
@Override
- public boolean acquireRecoveryLock(int clusterId) {
+ public boolean acquireRecoveryLock(int clusterId, int recoveredBy) {
QueryBuilder query =
start().and(
start(Document.ID).is(Integer.toString(clusterId)).get(),
@@ -85,6 +85,9 @@ public class MongoMissingLastRevSeeker e
BasicDBObject setUpdates = new BasicDBObject();
setUpdates.append(ClusterNodeInfo.REV_RECOVERY_LOCK, RecoverLockState.ACQUIRED.name());
+ if (recoveredBy != 0) {
+ setUpdates.append(ClusterNodeInfo.REV_RECOVERY_BY, recoveredBy);
+ }
BasicDBObject update = new BasicDBObject();
update.append("$set", setUpdates);