You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2022/07/28 14:43:18 UTC
[jackrabbit-oak] branch OAK-9873 created (now 8f630a3941)
This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a change to branch OAK-9873
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at 8f630a3941 OAK-9873 Prefetch node states: composite node store support
This branch includes the following new commits:
new 8f630a3941 OAK-9873 Prefetch node states: composite node store support
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[jackrabbit-oak] 01/01: OAK-9873 Prefetch node states: composite node store support
Posted by th...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a commit to branch OAK-9873
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 8f630a3941447cfb6b59544604620848bc54f503
Author: Thomas Mueller <th...@apache.org>
AuthorDate: Thu Jul 28 16:43:06 2022 +0200
OAK-9873 Prefetch node states: composite node store support
---
.../apache/jackrabbit/oak/query/QueryOptions.java | 10 +
.../apache/jackrabbit/oak/query/SQL2Parser.java | 3 +
.../jackrabbit/oak/query/ast/SelectorImpl.java | 4 +-
.../jackrabbit/oak/query/xpath/Statement.java | 3 +
.../oak/query/xpath/XPathToSQL2Converter.java | 2 +
.../jackrabbit/oak/composite/PrefetchTest.java | 225 +++++++++++++++++++++
.../oak/composite/CompositionContext.java | 4 +
.../plugins/document/prefetch/CacheWarming.java | 7 +
8 files changed, 257 insertions(+), 1 deletion(-)
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryOptions.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryOptions.java
index d244378f46..1edde3b04e 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryOptions.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryOptions.java
@@ -39,6 +39,7 @@ public class QueryOptions {
public Optional<Long> limit = Optional.empty();
public Optional<Long> offset = Optional.empty();
public List<String> prefetch = Collections.emptyList();
+ public Optional<Integer> prefetchCount = Optional.empty();
public enum Traversal {
// traversing without index is OK for this query, and does not fail or log a warning
@@ -61,6 +62,7 @@ public class QueryOptions {
limit = defaultValues.limit;
offset = defaultValues.offset;
prefetch = defaultValues.prefetch;
+ prefetchCount = defaultValues.prefetchCount;
}
QueryOptions(JsonObject json) {
@@ -85,6 +87,14 @@ public class QueryOptions {
LOG.warn("Invalid limit {}", x);
}
}
+ x = map.get("prefetches");
+ if (x != null) {
+ try {
+ prefetchCount = Optional.of(Integer.parseInt(x));
+ } catch (NumberFormatException e) {
+ LOG.warn("Invalid prefetch count {}", x);
+ }
+ }
x = map.get("prefetch");
if (x != null) {
ArrayList<String> list = new ArrayList<>();
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/SQL2Parser.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/SQL2Parser.java
index 949d136c26..6e995844e8 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/SQL2Parser.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/SQL2Parser.java
@@ -26,6 +26,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import javax.jcr.PropertyType;
import javax.jcr.RepositoryException;
@@ -194,6 +195,8 @@ public class SQL2Parser {
q.setOffset(readNumber());
} else if (readIf("LIMIT")) {
q.setLimit(readNumber());
+ } else if (readIf("PREFETCHES")) {
+ options.prefetchCount = Optional.of((int) readNumber());
} else if (readIf("PREFETCH")) {
read("(");
ArrayList<String> list = new ArrayList<String>();
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java
index 8610e770c4..dcac7cb34a 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java
@@ -25,6 +25,7 @@ import static org.apache.jackrabbit.JcrConstants.NT_BASE;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -368,7 +369,8 @@ public class SelectorImpl extends SourceImpl {
planIndexName = index.getIndexName(f, rootState);
cursor = index.query(f, rootState);
}
- int prefetchCount = query.getExecutionContext().getSettings().getPrefetchCount();
+ int prefetchCount = query.getQueryOptions().prefetchCount.
+ orElse(query.getExecutionContext().getSettings().getPrefetchCount());
if (prefetchCount > 0) {
PrefetchNodeStore store = query.getExecutionContext().getPrefetchNodeStore();
cursor = Cursors.newPrefetchCursor(cursor, store, prefetchCount,
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java
index 149987ef09..8c3a953f07 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java
@@ -373,6 +373,9 @@ public class Statement {
if (queryOptions.limit.isPresent()) {
optionValues.add("limit " + queryOptions.limit.get());
}
+ if (queryOptions.prefetchCount.isPresent()) {
+ optionValues.add("prefetches " + queryOptions.prefetchCount.get());
+ }
if (!queryOptions.prefetch.isEmpty()) {
String list = String.join(", ",
Lists.transform(queryOptions.prefetch,
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/XPathToSQL2Converter.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/XPathToSQL2Converter.java
index a0e9c549fe..74ab7e3569 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/XPathToSQL2Converter.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/XPathToSQL2Converter.java
@@ -418,6 +418,8 @@ public class XPathToSQL2Converter {
options.offset = Optional.of(readNumber());
} else if (readIf("limit")) {
options.limit = Optional.of(readNumber());
+ } else if (readIf("prefetches")) {
+ options.prefetchCount = Optional.of((int) readNumber());
} else if (readIf("prefetch")) {
read("(");
ArrayList<String> list = new ArrayList<String>();
diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/composite/PrefetchTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/composite/PrefetchTest.java
new file mode 100644
index 0000000000..6e2823722b
--- /dev/null
+++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/composite/PrefetchTest.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.composite;
+
+import static org.apache.jackrabbit.JcrConstants.NT_UNSTRUCTURED;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.shutdown;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import javax.jcr.Node;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.query.RowIterator;
+
+import org.apache.jackrabbit.api.JackrabbitSession;
+import org.apache.jackrabbit.oak.commons.junit.TemporarySystemProperty;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.document.prefetch.CacheWarming;
+import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
+import org.apache.jackrabbit.oak.spi.mount.Mounts;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.contrib.java.lang.system.ProvideSystemProperty;
+import org.junit.contrib.java.lang.system.RestoreSystemProperties;
+import org.junit.runners.Parameterized.Parameters;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
+
+/**
+ * Test if prefetch works, when using the composite node store.
+ */
+public class PrefetchTest extends CompositeNodeStoreQueryTestBase {
+
+ @Parameters(name = "Root: {0}, Mounts: {1}")
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][]{
+ {NodeStoreKind.DOCUMENT_H2, NodeStoreKind.DOCUMENT_H2},
+ {NodeStoreKind.DOCUMENT_H2, NodeStoreKind.SEGMENT},
+ {NodeStoreKind.DOCUMENT_MEMORY, NodeStoreKind.DOCUMENT_MEMORY}
+ });
+ }
+
+ private static String READ_ONLY_MOUNT_V1_NAME = "readOnlyV1";
+
+ // JCR repository
+ private CompositeRepo repoV1;
+ private CompositeRepo repoV2;
+
+ private ListAppender<ILoggingEvent> listAppender;
+ private final String cacheWarmingLogger = CacheWarming.class.getName();
+
+
+ @Rule
+ public final ProvideSystemProperty updateSystemProperties
+ = new ProvideSystemProperty(DocumentNodeStore.SYS_PROP_PREFETCH, "true");
+
+ @Rule
+ public final RestoreSystemProperties restoreSystemProperties
+ = new RestoreSystemProperties();
+
+ public PrefetchTest(NodeStoreKind root, NodeStoreKind mounts) {
+ super(root, mounts);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> T getNodeStore(Class<T> type) {
+ for(NodeStoreRegistration r : registrations) {
+ NodeStore store = r.getInstance();
+ if (type.isAssignableFrom(store.getClass())) {
+ return (T) store;
+ }
+ }
+ return null;
+ }
+
+ @Rule
+ public TemporarySystemProperty temporarySystemProperty = new TemporarySystemProperty();
+
+ @Before
+ public void loggingAppenderStart() {
+ LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+ listAppender = new ListAppender<>();
+ listAppender.start();
+ context.getLogger(cacheWarmingLogger).setLevel(Level.DEBUG);
+ context.getLogger(cacheWarmingLogger).addAppender(listAppender);
+ }
+
+ @After
+ public void loggingAppenderStop() {
+ listAppender.stop();
+ }
+
+ @Override
+ @Before
+ public void initStore() throws Exception {
+ globalStore = register(nodeStoreRoot.create(null));
+ repoV1 = new CompositeRepo(READ_ONLY_MOUNT_V1_NAME);
+ repoV1.initCompositeRepo();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ repoV1.cleanup();
+ if (repoV2 != null) {
+ repoV2.cleanup();
+ }
+ }
+
+ @Test
+ public void prefetch() throws Exception {
+ // we run a query with prefetch,
+ // and see if the cacheWarmingLogger did log a message about it
+ QueryResult result = repoV1.executeQuery("/jcr:root//*[@foo = 'bar'] option(prefetches 10)", "xpath");
+ getResult(result, "jcr:path");
+ assertTrue(isMessagePresent(listAppender, "Prefetch"));
+ }
+
+ private boolean isMessagePresent(ListAppender<ILoggingEvent> listAppender, String pattern) {
+ for (ILoggingEvent loggingEvent : listAppender.list) {
+ if (loggingEvent.getMessage().contains(pattern)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static String getResult(QueryResult result, String propertyName) throws RepositoryException {
+ StringBuilder buff = new StringBuilder();
+ RowIterator it = result.getRows();
+ while (it.hasNext()) {
+ if (buff.length() > 0) {
+ buff.append(", ");
+ }
+ buff.append(it.nextRow().getValue(propertyName).getString());
+ }
+ return buff.toString();
+ }
+
+ private class CompositeRepo {
+ private Repository compositeRepository;
+ private JackrabbitSession compositeSession;
+ private QueryManager compositeQueryManager;
+
+ private NodeStore readOnlyStore;
+ private Repository readOnlyRepository;
+ private CompositeNodeStore store;
+ private MountInfoProvider mip;
+ private JackrabbitSession readOnlySession;
+ private Node readOnlyRoot;
+
+ private boolean cleanedUp;
+
+ public QueryResult executeQuery(String statement, String language) throws RepositoryException {
+ return compositeQueryManager.createQuery(statement, language).execute();
+ }
+
+ CompositeRepo(String readOnlyMountName) throws Exception {
+ this.readOnlyStore = register(mounts.create(readOnlyMountName));
+
+ this.mip = Mounts.newBuilder().readOnlyMount(readOnlyMountName, "/libs").build();
+
+ initReadOnlySeedRepo();
+ List<MountedNodeStore> nonDefaultStores = Lists.newArrayList();
+ nonDefaultStores.add(new MountedNodeStore(this.mip.getMountByName(readOnlyMountName), readOnlyStore));
+ this.store = new CompositeNodeStore(this.mip, globalStore, nonDefaultStores);
+
+ }
+
+ private void initCompositeRepo() throws Exception {
+ compositeRepository = createJCRRepository(this.store, this.mip);
+ compositeSession = (JackrabbitSession) compositeRepository.login(new SimpleCredentials("admin", "admin".toCharArray()));
+ compositeQueryManager = compositeSession.getWorkspace().getQueryManager();
+ }
+
+ private void initReadOnlySeedRepo() throws Exception {
+ readOnlyRepository = createJCRRepository(readOnlyStore, this.mip);
+ readOnlySession = (JackrabbitSession) readOnlyRepository.login(new SimpleCredentials("admin", "admin".toCharArray()));
+ readOnlyRoot = readOnlySession.getRootNode();
+ Node libs = readOnlyRoot.addNode("libs", NT_UNSTRUCTURED);
+ libs.setPrimaryType(NT_UNSTRUCTURED);
+ }
+
+ private void cleanup() {
+ if (!cleanedUp) {
+ compositeSession.logout();
+ shutdown(compositeRepository);
+ readOnlySession.logout();
+ shutdown(readOnlyRepository);
+ }
+ cleanedUp = true;
+ }
+
+ }
+}
diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositionContext.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositionContext.java
index 4ec0569c13..321974e2f0 100644
--- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositionContext.java
+++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositionContext.java
@@ -210,6 +210,10 @@ class CompositionContext {
}
public void prefetch(Collection<String> paths, NodeState rootState) {
+ if (rootState instanceof CompositeNodeState) {
+ CompositeNodeState compositeRoot = (CompositeNodeState) rootState;
+ rootState = compositeRoot.getNodeState(globalStore);
+ }
prefetchNodeStore.prefetch(paths, rootState);
}
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/prefetch/CacheWarming.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/prefetch/CacheWarming.java
index 132c2b9a32..b5a93a0d6d 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/prefetch/CacheWarming.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/prefetch/CacheWarming.java
@@ -25,11 +25,15 @@ import org.apache.jackrabbit.oak.plugins.document.DocumentNodeState;
import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static java.util.Objects.requireNonNull;
public class CacheWarming {
+ private static final Logger LOG = LoggerFactory.getLogger(CacheWarming.class);
+
private final DocumentStore store;
public CacheWarming(DocumentStore store) {
@@ -48,6 +52,9 @@ public class CacheWarming {
ids.add(id);
}
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Prefetch {} nodes", ids.size());
+ }
store.prefetch(Collection.NODES, ids);
}