You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mk...@apache.org on 2020/10/14 12:28:19 UTC
svn commit: r1882487 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/
oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/
oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/
oa...
Author: mkataria
Date: Wed Oct 14 12:28:19 2020
New Revision: 1882487
URL: http://svn.apache.org/viewvc?rev=1882487&view=rev
Log:
OAK-9209: Index lane elastic-async traverses repository
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexlaneRepositoryTraversalTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImplTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdateTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java?rev=1882487&r1=1882486&r2=1882487&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java Wed Oct 14 12:28:19 2020
@@ -77,6 +77,7 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.commit.SimpleCommitContext;
import org.apache.jackrabbit.oak.spi.commit.ValidatorProvider;
import org.apache.jackrabbit.oak.spi.commit.VisibleEditor;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStateDiff;
@@ -196,6 +197,13 @@ public class AsyncIndexUpdate implements
= Integer.getInteger("oak.async.checkpointCleanupIntervalMinutes", 5);
/**
+ * Setting this to true lead to lane execution (node traversal) even if there
+ * is no index assigned to this lane under /oak:index.
+ */
+ private final boolean traverseNodesIfLaneNotPresentInIndex
+ = Boolean.getBoolean("oak.async.traverseNodesIfLaneNotPresentInIndex");
+
+ /**
* The time in minutes since the epoch when the last checkpoint cleanup ran.
*/
private long lastCheckpointCleanUpTime;
@@ -426,6 +434,9 @@ public class AsyncIndexUpdate implements
@Override
public synchronized void run() {
+ if (!shouldProceed()){
+ return;
+ }
boolean permitAcquired = false;
try{
if (runPermit.tryAcquire()){
@@ -441,7 +452,6 @@ public class AsyncIndexUpdate implements
}
}
-
@Override
public void close() {
if (closed) {
@@ -613,6 +623,43 @@ public class AsyncIndexUpdate implements
}
}
+ private boolean shouldProceed() {
+ NodeState asyncNode = store.getRoot().getChildNode(":async");
+ /*
+ If /:async node already have the lane(under consideration) info, we can proceed ahead, as
+ majorly this change is to stop repository traversal on very first run. If lane had already
+ traversed nodes in repository there is no point stopping this now.
+ */
+ if (asyncNode.exists() && asyncNode.hasProperty(name)) {
+ return true;
+ }
+ return traverseNodesIfLaneNotPresentInIndex || isIndexWithLanePresent();
+ }
+
+ /**
+ *
+ * @return true if there is at least one index present under /oak:index with indexingLane in action.
+ */
+ private boolean isIndexWithLanePresent() {
+ NodeState oakIndexNode = store.getRoot().getChildNode("oak:index");
+ if (!oakIndexNode.exists()) {
+ log.info("lane: {} - no indexes exist under /oak:index", name);
+ return false;
+ }
+ for (ChildNodeEntry childNodeEntry : oakIndexNode.getChildNodeEntries()) {
+ PropertyState async = childNodeEntry.getNodeState().getProperty("async");
+ if (async != null) {
+ for (String s : async.getValue(Type.STRINGS)) {
+ if (s.equals(name)) {
+ return true;
+ }
+ }
+ }
+ }
+ log.info("lane: {} not present for indexes under /oak:index", name);
+ return false;
+ }
+
private void markFailingIndexesAsCorrupt(NodeBuilder builder) {
for (Map.Entry<String, CorruptIndexInfo> index : corruptIndexHandler.getCorruptIndexData(name).entrySet()){
NodeBuilder indexBuilder = childBuilder(builder, index.getKey());
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImplTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImplTest.java?rev=1882487&r1=1882486&r2=1882487&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImplTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImplTest.java Wed Oct 14 12:28:19 2020
@@ -20,6 +20,7 @@
package org.apache.jackrabbit.oak.plugins.index;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import com.google.common.collect.ImmutableSet;
@@ -29,6 +30,8 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -40,6 +43,18 @@ public class AsyncIndexInfoServiceImplTe
private PropertyIndexEditorProvider provider = new PropertyIndexEditorProvider();
private AsyncIndexInfoServiceImpl service = new AsyncIndexInfoServiceImpl(store);
+ private Properties systemProperties;
+
+ @Before
+ public void setup(){
+ systemProperties =(Properties) System.getProperties().clone();
+ System.setProperty("oak.async.traverseNodesIfLaneNotPresentInIndex", "true");
+ }
+
+ @After
+ public void shutDown(){
+ System.setProperties(systemProperties);
+ }
@Test
public void names() throws Exception{
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdateTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdateTest.java?rev=1882487&r1=1882486&r2=1882487&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdateTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdateTest.java Wed Oct 14 12:28:19 2020
@@ -41,8 +41,10 @@ import java.lang.management.ManagementFa
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
@@ -94,6 +96,7 @@ import org.apache.jackrabbit.util.ISO860
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import ch.qos.logback.classic.Level;
@@ -107,8 +110,17 @@ public class AsyncIndexUpdateTest {
private MetricStatisticsProvider statsProvider =
new MetricStatisticsProvider(ManagementFactory.getPlatformMBeanServer(),executor);
+ private Properties systemProperties;
+
+ @Before
+ public void setup(){
+ systemProperties =(Properties) System.getProperties().clone();
+ System.setProperty("oak.async.traverseNodesIfLaneNotPresentInIndex", "true");
+ }
+
@After
public void shutDown(){
+ System.setProperties(systemProperties);
statsProvider.close();
new ExecutorCloser(executor).close();
}
@@ -503,7 +515,6 @@ public class AsyncIndexUpdateTest {
checkpoints.size() == 1);
assertEquals(store.getRoot().getChildNode(ASYNC)
.getString("async"), checkpoints.iterator().next());
-
async.run();
assertEquals("Expecting no checkpoint changes",
checkpoints, store.listCheckpoints());
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterTest.java?rev=1882487&r1=1882486&r2=1882487&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterTest.java Wed Oct 14 12:28:19 2020
@@ -59,6 +59,8 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.jetbrains.annotations.NotNull;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -87,6 +89,19 @@ public class IndexImporterTest {
private NodeStore store = new MemoryNodeStore();
private IndexEditorProvider provider = new PropertyIndexEditorProvider();
+ private Properties systemProperties;
+
+ @Before
+ public void setup(){
+ systemProperties =(Properties) System.getProperties().clone();
+ System.setProperty("oak.async.traverseNodesIfLaneNotPresentInIndex", "true");
+ }
+
+ @After
+ public void shutDown(){
+ System.setProperties(systemProperties);
+ }
+
@Test(expected = IllegalArgumentException.class)
public void importIndex_NoMeta() throws Exception{
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexlaneRepositoryTraversalTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexlaneRepositoryTraversalTest.java?rev=1882487&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexlaneRepositoryTraversalTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexlaneRepositoryTraversalTest.java Wed Oct 14 12:28:19 2020
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import ch.qos.logback.classic.Level;
+import com.google.common.collect.ImmutableList;
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.api.ContentSession;
+import org.apache.jackrabbit.oak.api.Root;
+import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.TrackingCorruptIndexHandler;
+import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.property.RecursiveDelete;
+import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexProvider;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider.compose;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests: index lane only traverses repository if atleast one index under /oak:index have
+ * the index-lane under consideration.
+ */
+public class IndexlaneRepositoryTraversalTest {
+
+ private final long INDEX_CORRUPT_INTERVAL_IN_MILLIS = 100;
+ private MemoryBlobStore blobStore;
+
+ protected Root root;
+
+ private AsyncIndexUpdate asyncIndexUpdate;
+ private AsyncIndexUpdate asyncIndexUpdateFulltext;
+
+ NodeStore nodeStore;
+ LogCustomizer customLogger;
+
+ @Before
+ public void before() throws Exception {
+ ContentSession session = createRepository().login(null, null);
+ root = session.getLatestRoot();
+ customLogger = LogCustomizer
+ .forLogger(AsyncIndexUpdate.class.getName())
+ .enable(Level.INFO).create();
+ customLogger.starting();
+ }
+
+ @After
+ public void after() {
+ customLogger.finished();
+ }
+
+ protected ContentRepository createRepository() {
+ nodeStore = new MemoryNodeStore();
+ blobStore = new MemoryBlobStore();
+ blobStore.setBlockSizeMin(48);//make it as small as possible
+
+ LuceneIndexEditorProvider luceneIndexEditorProvider = new LuceneIndexEditorProvider();
+ LuceneIndexProvider provider = new LuceneIndexProvider();
+ luceneIndexEditorProvider.setBlobStore(blobStore);
+
+ asyncIndexUpdate = new AsyncIndexUpdate("async", nodeStore, compose(newArrayList(
+ luceneIndexEditorProvider,
+ new NodeCounterEditorProvider()
+ )));
+ asyncIndexUpdateFulltext = new AsyncIndexUpdate("fulltext-async", nodeStore, compose(newArrayList(
+ luceneIndexEditorProvider,
+ new NodeCounterEditorProvider()
+ )));
+ TrackingCorruptIndexHandler trackingCorruptIndexHandler = new TrackingCorruptIndexHandler();
+ trackingCorruptIndexHandler.setCorruptInterval(INDEX_CORRUPT_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS);
+ asyncIndexUpdate.setCorruptIndexHandler(trackingCorruptIndexHandler);
+ return new Oak(nodeStore)
+ .with(new InitialContent())
+ .with(new OpenSecurityProvider())
+ .with((QueryIndexProvider) provider)
+ .with((Observer) provider)
+ .with(luceneIndexEditorProvider)
+ .with(new PropertyIndexEditorProvider())
+ .with(new NodeTypeIndexProvider())
+ .with(new PropertyIndexProvider())
+ .with(new PropertyIndexEditorProvider())
+ .createContentRepository();
+ }
+
+ @Test
+ public void repositoryTraversalIfLaneIsPresent() throws Exception {
+ Tree test1 = root.getTree("/").addChild(INDEX_DEFINITIONS_NAME).addChild("mynodetype");
+ test1.setProperty("jcr:primaryType", "oak:QueryIndexDefinition", Type.NAME);
+ test1.setProperty("type", "property");
+ test1.setProperty("propertyNames", ImmutableList.of("jcr:primaryType", "jcr:mixinTypes"), Type.NAMES);
+ test1.setProperty("declaringNodeTypes", ImmutableList.of("oak:QueryIndexDefinition"), Type.NAMES);
+ test1.setProperty("nodeTypeListDefined", true);
+ test1.setProperty("reindex", true);
+ root.commit();
+ asyncIndexUpdate.run();
+ List<String> logs = customLogger.getLogs();
+ assertFalse(isAssertionLogPresent(logs, indexLaneLog("async")));
+ assertFalse(isAssertionLogPresent(logs, indexNotPresentLog("async")));
+ }
+
+ @Test
+ public void noRepositoryTraversalIfLaneIsNotPresent() throws Exception {
+ Iterable<Tree> indexTree = root.getTree("/oak:index").getChildren();
+ for (Tree children : indexTree) {
+ deletePathRecursively(children.getPath());
+ }
+ asyncIndexUpdate.run();
+ List<String> logs = customLogger.getLogs();
+ assertTrue(isAssertionLogPresent(logs, indexLaneLog("async")));
+ assertFalse(isAssertionLogPresent(logs, indexNotPresentLog("async")));
+ }
+
+ @Test
+ public void noRepositoryTraversalIfNoIndexPresent() throws Exception {
+ deletePathRecursively("/oak:index");
+ asyncIndexUpdate.run();
+ List<String> logs = customLogger.getLogs();
+ assertTrue(isAssertionLogPresent(logs, indexNotPresentLog("async")));
+ assertFalse(isAssertionLogPresent(logs, indexLaneLog("async")));
+ }
+
+ @Test
+ public void repositoryTraversalAsyncNodeContainsAsyncProperty() throws Exception {
+ // first run to populate /:async node.
+ asyncIndexUpdate.run();
+ asyncIndexUpdate.run();
+ List<String> logs = customLogger.getLogs();
+ assertFalse(isAssertionLogPresent(logs, indexLaneLog("async")));
+ assertFalse(isAssertionLogPresent(logs, indexNotPresentLog("async")));
+ }
+
+ @Test
+ public void repositoryTraversalAsyncNodeDonotContainsFulltextAsyncProperty() throws Exception {
+ // first run to populate /:async node.
+ asyncIndexUpdate.run();
+ List<String> logs = customLogger.getLogs();
+ assertFalse(isAssertionLogPresent(logs, indexLaneLog("async")));
+ assertFalse(isAssertionLogPresent(logs, indexNotPresentLog("async")));
+ asyncIndexUpdateFulltext.run();
+ logs = customLogger.getLogs();
+ assertFalse(isAssertionLogPresent(logs, indexNotPresentLog("fulltext-async")));
+ assertTrue(isAssertionLogPresent(logs, indexLaneLog("fulltext-async")));
+ }
+
+ private void deletePathRecursively(String path) throws CommitFailedException {
+ RecursiveDelete rd = new RecursiveDelete(nodeStore, EmptyHook.INSTANCE, () -> CommitInfo.EMPTY);
+ rd.setBatchSize(100);
+ rd.run(path);
+ }
+
+ private boolean isAssertionLogPresent(List<String> logs, String assertionLog) {
+ for (String log : logs) {
+ if (log.equals(assertionLog)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private String indexNotPresentLog(String lane) {
+ return "lane: " + lane + " - no indexes exist under /oak:index";
+ }
+
+ private String indexLaneLog(String lane) {
+ return "lane: " + lane + " not present for indexes under /oak:index";
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexlaneRepositoryTraversalTest.java
------------------------------------------------------------------------------
svn:eol-style = native