You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mk...@apache.org on 2022/11/09 07:58:34 UTC
[jackrabbit-oak] branch trunk updated: OAK-9981: Add a option in purgecommand(oak-run) to skip purge operations on active base index (#742)
This is an automated email from the ASF dual-hosted git repository.
mkataria pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 76adce992b OAK-9981: Add a option in purgecommand(oak-run) to skip purge operations on active base index (#742)
76adce992b is described below
commit 76adce992ba1b426ba3603c71ee0432b92a120a3
Author: Mohit Kataria <mk...@apache.org>
AuthorDate: Wed Nov 9 13:28:28 2022 +0530
OAK-9981: Add a option in purgecommand(oak-run) to skip purge operations on active base index (#742)
---
.../oak/indexversion/IndexVersionOperation.java | 61 ++++++++--------------
.../oak/indexversion/PurgeOldIndexVersion.java | 29 ++++++++--
.../oak/run/PurgeOldIndexVersionCommand.java | 6 ++-
.../oak/indexversion/PurgeOldIndexVersionIT.java | 38 ++++++++++++--
4 files changed, 88 insertions(+), 46 deletions(-)
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/indexversion/IndexVersionOperation.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/indexversion/IndexVersionOperation.java
index 1ade99d8e0..14c6aa4a22 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/indexversion/IndexVersionOperation.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/indexversion/IndexVersionOperation.java
@@ -63,6 +63,21 @@ public class IndexVersionOperation {
return this.getIndexName() + " operation:" + this.getOperation();
}
+ /**
+ * Generate list of index version operation over a list of indexes have same index base. This will purge base index.
+ *
+ * @param rootNode NodeState of root
+ * @param parentPath parent path of baseIndex
+ * @param indexNameObjectList This is a list of IndexName Objects with same baseIndexName on which operations will be applied.
+ * @param purgeThresholdMillis after which a fully functional index is eligible for purge operations
+ *
+ * @return This method returns an IndexVersionOperation list i.e indexNameObjectList marked with operations
+ */
+ public static List<IndexVersionOperation> generateIndexVersionOperationList(NodeState rootNode, String parentPath,
+ List<IndexName> indexNameObjectList, long purgeThresholdMillis) {
+ return generateIndexVersionOperationList(rootNode, parentPath, indexNameObjectList, purgeThresholdMillis, true);
+ }
+
/**
* Generate list of index version operation over a list of indexes have same index base.
*
@@ -70,11 +85,12 @@ public class IndexVersionOperation {
* @param parentPath parent path of baseIndex
* @param indexNameObjectList This is a list of IndexName Objects with same baseIndexName on which operations will be applied.
* @param purgeThresholdMillis after which a fully functional index is eligible for purge operations
+ * @param shouldPurgeBaseIndex If set to true, will apply purge operations on active base index i.e. DELETE or DELETE_HIDDEN_AND_DISABLE
*
* @return This method returns an IndexVersionOperation list i.e indexNameObjectList marked with operations
*/
public static List<IndexVersionOperation> generateIndexVersionOperationList(NodeState rootNode, String parentPath,
- List<IndexName> indexNameObjectList, long purgeThresholdMillis) {
+ List<IndexName> indexNameObjectList, long purgeThresholdMillis, boolean shouldPurgeBaseIndex) {
NodeState indexDefParentNode = NodeStateUtils.getNode(rootNode, parentPath);
List<IndexName> reverseSortedIndexNameList = getReverseSortedIndexNameList(indexNameObjectList);
List<IndexVersionOperation> indexVersionOperationList = new LinkedList<>();
@@ -110,7 +126,11 @@ public class IndexVersionOperation {
// if active index not long enough, NOOP for all indexes
if (isActiveIndexOldEnough) {
if (indexNameObject.getProductVersion() == activeProductVersion && indexNameObject.getCustomerVersion() == 0) {
- indexVersionOperation.setOperation(Operation.DELETE_HIDDEN_AND_DISABLE);
+ if (shouldPurgeBaseIndex) {
+ indexVersionOperation.setOperation(Operation.DELETE_HIDDEN_AND_DISABLE);
+ } else {
+ indexVersionOperation.setOperation(Operation.NOOP);
+ }
} else if (indexNameObject.getProductVersion() <= activeProductVersion ) {
// the check hidden oak mount logic only works when passing through the proper composite store
if (isHiddenOakMountExists(indexNode)) {
@@ -131,7 +151,7 @@ public class IndexVersionOperation {
}
}
}
- if (indexVersionOperationList.isEmpty() || !isValidIndexVersionOperationList(indexVersionOperationList)) {
+ if (indexVersionOperationList.isEmpty()) {
LOG.info("Not valid version operation list: '{}', skip all", indexNameObjectList);
indexVersionOperationList = Collections.emptyList();
}
@@ -199,41 +219,6 @@ public class IndexVersionOperation {
return reverseSortedIndexNameObjectList;
}
- /**
- * @param indexVersionOperations
- * @return true if the IndexVersionOperation list passes following criteria.
- * For merging indexes we need baseIndex and latest custom index.
- * So we first validate that if there are custom indexes than OOTB index with same product must be marked with DELETE_HIDDEN_AND_DISABLE
- */
- private static boolean isValidIndexVersionOperationList(List<IndexVersionOperation> indexVersionOperations) {
- boolean isValid = false;
- IndexVersionOperation lastNoopOperationIndexVersion = null;
- IndexVersionOperation indexWithDeleteHiddenOp = null;
- for (IndexVersionOperation indexVersionOperation : indexVersionOperations) {
- if (indexVersionOperation.getOperation() == Operation.NOOP) {
- if (lastNoopOperationIndexVersion == null) {
- lastNoopOperationIndexVersion = indexVersionOperation;
- }
- }
- if (indexVersionOperation.getOperation() == Operation.DELETE_HIDDEN_AND_DISABLE) {
- if (indexWithDeleteHiddenOp == null) {
- indexWithDeleteHiddenOp = indexVersionOperation;
- }
- }
- }
- if (lastNoopOperationIndexVersion.getIndexName().getCustomerVersion() == 0) {
- isValid = true;
- } else if (lastNoopOperationIndexVersion.getIndexName().getCustomerVersion() != 0) {
- if (indexWithDeleteHiddenOp != null
- && lastNoopOperationIndexVersion.getIndexName().getProductVersion() == indexWithDeleteHiddenOp.getIndexName().getProductVersion()) {
- isValid = true;
- }
- }
- if (!isValid) {
- LOG.info("IndexVersionOperation List is not valid for index {}", lastNoopOperationIndexVersion.getIndexName().getNodeName());
- }
- return isValid;
- }
private static List<IndexName> removeDisabledCustomIndexesFromList(NodeState indexDefParentNode,
List<IndexName> indexNameObjectList) {
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersion.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersion.java
index cbd86f3f93..04dfa31995 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersion.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersion.java
@@ -46,7 +46,7 @@ public class PurgeOldIndexVersion {
private static final Logger LOG = LoggerFactory.getLogger(PurgeOldIndexVersion.class);
/**
- * Execute purging index based on the index version naming and last time index time
+ * Execute purging index based on the index version naming and last time index time. This will purge base index.
*
* @param nodeStore the node store
* @param isReadWriteRepository bool to indicate if it's read write repository, if yes, the purge index will not execute
@@ -58,7 +58,23 @@ public class PurgeOldIndexVersion {
*/
public void execute(NodeStore nodeStore, boolean isReadWriteRepository, long purgeThresholdMillis, List<String> indexPaths) throws
IOException, CommitFailedException {
- List<IndexVersionOperation> purgeIndexList = getPurgeIndexes(nodeStore, purgeThresholdMillis, indexPaths);
+ execute(nodeStore, isReadWriteRepository, purgeThresholdMillis, indexPaths, true);
+ }
+ /**
+ * Execute purging index based on the index version naming and last time index time
+ *
+ * @param nodeStore the node store
+ * @param isReadWriteRepository bool to indicate if it's read write repository, if yes, the purge index will not execute
+ * @param purgeThresholdMillis the threshold of time length since last time index time to determine, will purge if exceed that
+ * @param indexPaths the index path or parent path
+ * @param shouldPurgeBaseIndex If set to true, will apply purge operations on active base index i.e. DELETE or DELETE_HIDDEN_AND_DISABLE
+ *
+ * @throws IOException
+ * @throws CommitFailedException
+ */
+ public void execute(NodeStore nodeStore, boolean isReadWriteRepository, long purgeThresholdMillis, List<String> indexPaths, boolean shouldPurgeBaseIndex) throws
+ IOException, CommitFailedException {
+ List<IndexVersionOperation> purgeIndexList = getPurgeIndexes(nodeStore, purgeThresholdMillis, indexPaths, shouldPurgeBaseIndex);
if (!purgeIndexList.isEmpty()) {
if (isReadWriteRepository) {
LOG.info("Found indexes for purging: '{}'", purgeIndexList);
@@ -73,7 +89,7 @@ public class PurgeOldIndexVersion {
}
}
- public List<IndexVersionOperation> getPurgeIndexes(NodeStore nodeStore, long purgeThresholdMillis, List<String> indexPaths) throws IOException, CommitFailedException {
+ public List<IndexVersionOperation> getPurgeIndexes(NodeStore nodeStore, long purgeThresholdMillis, List<String> indexPaths, boolean shouldPurgeBaseIndex ) throws IOException, CommitFailedException {
List<IndexVersionOperation> purgeIndexList = new ArrayList<>();
LOG.info("Getting indexes to purge over index paths '{}'", indexPaths);
List<String> sanitisedIndexPaths = sanitiseUserIndexPaths(indexPaths);
@@ -85,7 +101,7 @@ public class PurgeOldIndexVersion {
List<IndexName> indexNameObjectList = getIndexNameObjectList(entry.getValue());
LOG.info("Validate purge index over base of '{}', which includes: '{}'", baseIndexPath, indexNameObjectList);
List<IndexVersionOperation> toDeleteIndexNameObjectList = IndexVersionOperation.generateIndexVersionOperationList(
- nodeStore.getRoot(), parentPath, indexNameObjectList, purgeThresholdMillis);
+ nodeStore.getRoot(), parentPath, indexNameObjectList, purgeThresholdMillis, shouldPurgeBaseIndex);
toDeleteIndexNameObjectList.removeIf(item -> (item.getOperation() == IndexVersionOperation.Operation.NOOP));
if (!toDeleteIndexNameObjectList.isEmpty()) {
LOG.info("Found some index need to be purged over base'{}': '{}'", baseIndexPath, toDeleteIndexNameObjectList);
@@ -97,6 +113,11 @@ public class PurgeOldIndexVersion {
return purgeIndexList;
}
+ // Purge operations will also be performed on base index i.e. DELETE or DELETE_HIDDEN_AND_DISABLE
+ public List<IndexVersionOperation> getPurgeIndexes(NodeStore nodeStore, long purgeThresholdMillis, List<String> indexPaths) throws IOException, CommitFailedException {
+ return getPurgeIndexes(nodeStore, purgeThresholdMillis, indexPaths, true);
+ }
+
/**
* @param userIndexPaths indexpaths provided by user
* @return a list of Indexpaths having baseIndexpaths or path till oak:index
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PurgeOldIndexVersionCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PurgeOldIndexVersionCommand.java
index 160db17284..0c8e19d099 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PurgeOldIndexVersionCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PurgeOldIndexVersionCommand.java
@@ -39,6 +39,7 @@ public class PurgeOldIndexVersionCommand implements Command {
private List<String> indexPaths;
private long DEFAULT_PURGE_THRESHOLD = TimeUnit.DAYS.toMillis(5); // 5 days in millis
private final static String DEFAULT_INDEX_PATH = "/oak:index";
+ private boolean shouldPurgeBaseIndex;
@Override
public void execute(String... args) throws Exception {
@@ -47,7 +48,7 @@ public class PurgeOldIndexVersionCommand implements Command {
if (!opts.getCommonOpts().isReadWrite()) {
LOG.info("Repository connected in read-only mode. Use '--read-write' for write operations");
}
- new PurgeOldIndexVersion().execute(fixture.getStore(), opts.getCommonOpts().isReadWrite(), threshold, indexPaths);
+ new PurgeOldIndexVersion().execute(fixture.getStore(), opts.getCommonOpts().isReadWrite(), threshold, indexPaths, shouldPurgeBaseIndex);
}
}
@@ -58,10 +59,13 @@ public class PurgeOldIndexVersionCommand implements Command {
OptionSpec<String> indexPathsOption = parser.accepts("index-paths", "Comma separated list of index paths for which the " +
"selected operations need to be performed")
.withOptionalArg().ofType(String.class).withValuesSeparatedBy(",").defaultsTo(DEFAULT_INDEX_PATH);
+ OptionSpec<Void> donotPurgeBaseIndexOption = parser.accepts("donot-purge-base-index", "Don't disable base index");
+
Options opts = new Options();
OptionSet optionSet = opts.parseAndConfigure(parser, args);
this.threshold = optionSet.valueOf(thresholdOption);
this.indexPaths = optionSet.valuesOf(indexPathsOption);
+ this.shouldPurgeBaseIndex = !optionSet.has(donotPurgeBaseIndexOption);
return opts;
}
}
diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersionIT.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersionIT.java
index dd2f12050c..35ad896637 100644
--- a/oak-run/src/test/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersionIT.java
+++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersionIT.java
@@ -73,12 +73,38 @@ public class PurgeOldIndexVersionIT {
NodeStore n = p.getCompositeNodestore();
purgeOldIndexVersion.execute(n, true, 1, Arrays.asList("/oak:index"));
- String purgeLog = "Found some index need to be purged over base'/oak:index/test':" +
- " '[/oak:index/test-1 base=/oak:index/test versioned product=1 custom=0 operation:DELETE_HIDDEN_AND_DISABLE]'";
+ String purgeLog = "Found some index need to be purged over base'/oak:index/test': '[" +
+ "/oak:index/test-2 base=/oak:index/test versioned product=2 custom=0 operation:DELETE_HIDDEN_AND_DISABLE," +
+ " /oak:index/test-1 base=/oak:index/test versioned product=1 custom=0 operation:DELETE_HIDDEN_AND_DISABLE]'";
+ Assert.assertEquals(1, purgeOldIndexVersionLogger.getLogs().size());
+ Assert.assertEquals(purgeLog, purgeOldIndexVersionLogger.getLogs().get(0));
+ Assert.assertTrue(IndexUtils.isIndexDisabledAndHiddenNodesDeleted(n, "/oak:index/test-1"));
+ Assert.assertTrue(IndexUtils.isIndexDisabledAndHiddenNodesDeleted(n, "/oak:index/test-2"));
+ Assert.assertTrue(IndexUtils.isIndexEnabledAndHiddenNodesPresent(n, "/oak:index/test-2-custom-1"));
+ }
+
+ @Test
+ public void testDonotPurgeBaseIndex() throws Exception {
+ createFolders();
+ config.blobStore = Persistence.getFileBlobStore(datastoreDir);
+ config.indexDir = indexDir;
+ initGlobal();
+ initLibs();
+ compositeLibs();
+
+ purgeOldIndexVersionLogger.starting();
+ PurgeOldIndexVersion purgeOldIndexVersion = new PurgeOldIndexVersion();
+ Persistence p = Persistence.openComposite(globalDir, libsDir, config);
+ NodeStore n = p.getCompositeNodestore();
+ purgeOldIndexVersion.execute(n, true, 1, Arrays.asList("/oak:index"), false);
+
+ String purgeLog = "Found some index need to be purged over base'/oak:index/test': '[" +
+ "/oak:index/test-1 base=/oak:index/test versioned product=1 custom=0 operation:DELETE_HIDDEN_AND_DISABLE]'";
Assert.assertEquals(1, purgeOldIndexVersionLogger.getLogs().size());
Assert.assertEquals(purgeLog, purgeOldIndexVersionLogger.getLogs().get(0));
Assert.assertTrue(IndexUtils.isIndexDisabledAndHiddenNodesDeleted(n, "/oak:index/test-1"));
Assert.assertTrue(IndexUtils.isIndexEnabledAndHiddenNodesPresent(n, "/oak:index/test-2"));
+ Assert.assertTrue(IndexUtils.isIndexEnabledAndHiddenNodesPresent(n, "/oak:index/test-2-custom-1"));
}
private void initGlobal() throws Exception {
@@ -102,6 +128,11 @@ public class PurgeOldIndexVersionIT {
"/jcr:root//*[@foo]",
"test-2",
"[/libs/test2]");
+ IndexUtils.createIndex(p, "test-2-custom-1", "foo", 30);
+ IndexUtils.assertQueryUsesIndexAndReturns(p,
+ "/jcr:root//*[@foo]",
+ "test-2-custom-1",
+ "[/libs/test2]");
p.close();
}
@@ -111,10 +142,11 @@ public class PurgeOldIndexVersionIT {
IndexUtils.createIndex(p, "test-1", "foo", 10);
IndexUtils.createIndex(p, "test-2", "foo", 20);
+ IndexUtils.createIndex(p, "test-2-custom-1", "foo", 30);
IndexUtils.assertQueryUsesIndexAndReturns(p,
"/jcr:root//*[@foo] order by @jcr:path",
- "test-2",
+ "test-2-custom-1",
"[/content/test, /libs/test2]");
p.close();
}