You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2022/06/23 15:33:17 UTC
[jackrabbit-oak] branch OAK-9780 updated: OAK-9780 PoC for DocumentNodeStore cache warming - support relative prefetch (WIP)
This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a commit to branch OAK-9780
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/OAK-9780 by this push:
new 7def7d18c2 OAK-9780 PoC for DocumentNodeStore cache warming - support relative prefetch (WIP)
7def7d18c2 is described below
commit 7def7d18c23df16ef34c931fb8add3b94b5f8e32
Author: Thomas Mueller <th...@apache.org>
AuthorDate: Thu Jun 23 17:25:40 2022 +0200
OAK-9780 PoC for DocumentNodeStore cache warming - support relative prefetch (WIP)
---
.../oak/api/jmx/QueryEngineSettingsMBean.java | 17 ++++-
.../main/java/org/apache/jackrabbit/oak/Oak.java | 10 +++
.../jackrabbit/oak/plugins/index/Cursors.java | 21 +++++-
.../jackrabbit/oak/query/QueryEngineImpl.java | 4 +-
.../jackrabbit/oak/query/QueryEngineSettings.java | 19 ++++-
.../apache/jackrabbit/oak/query/QueryOptions.java | 86 ++++++++++++++++++++++
.../apache/jackrabbit/oak/query/SQL2Parser.java | 19 ++++-
.../jackrabbit/oak/query/ast/SelectorImpl.java | 3 +-
.../jackrabbit/oak/query/stats/QueryRecorder.java | 9 ++-
.../jackrabbit/oak/query/stats/QueryStatsData.java | 6 --
.../jackrabbit/oak/query/xpath/Statement.java | 10 ++-
.../oak/query/xpath/XPathToSQL2Converter.java | 42 ++++++++++-
12 files changed, 223 insertions(+), 23 deletions(-)
diff --git a/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/QueryEngineSettingsMBean.java b/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/QueryEngineSettingsMBean.java
index 7e4f7cc2e3..df9a550029 100644
--- a/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/QueryEngineSettingsMBean.java
+++ b/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/QueryEngineSettingsMBean.java
@@ -75,6 +75,21 @@ public interface QueryEngineSettingsMBean {
@Description("Get the prefetch count. This is the number of entries pre-fetched from the node store at a time.")
int getPrefetchCount();
+ /**
+ * Change the automatic query options mapping.
+ *
+ * @param json the new mapping, in Json format
+ */
+ void setAutoOptionsMappingJson(String json);
+
+ /**
+ * Get the automatic query options mapping.
+ *
+ * @return the mapping, in Json format
+ */
+ @Description("Get the automatic query options mapping, in Json format.")
+ String getAutoOptionsMappingJson();
+
/**
* Whether queries that don't use an index will fail (throw an exception).
* The default is false.
@@ -170,7 +185,7 @@ public interface QueryEngineSettingsMBean {
@Name("class names")
@NotNull String[] classNames);
-// @Description("Get the Java package / fully qualified class names to ignore when finding the caller of query")
+ // @Description("Get the Java package / fully qualified class names to ignore when finding the caller of query")
@NotNull
String[] getIgnoredClassNamesInCallTrace();
}
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java
index 2610d1085a..5e80033d00 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java
@@ -930,6 +930,16 @@ public class Oak {
settings.setPrefetchCount(prefetchCount);
}
+ @Override
+ public String getAutoOptionsMappingJson() {
+ return settings.getAutoOptionsMappingJson();
+ }
+
+ @Override
+ public void setAutoOptionsMappingJson(String json) {
+ settings.setAutoOptionsMappingJson(json);
+ }
+
@Override
public boolean getFailTraversal() {
return settings.getFailTraversal();
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/Cursors.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/Cursors.java
index 1df989bb30..e5125febd6 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/Cursors.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/Cursors.java
@@ -22,6 +22,8 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
import org.apache.jackrabbit.oak.api.Result.SizePrecision;
import org.apache.jackrabbit.oak.commons.PathUtils;
@@ -77,8 +79,9 @@ public class Cursors {
return new ConcatCursor(cursors, settings);
}
- public static Cursor newPrefetchCursor(Cursor cursor, PrefetchNodeStore store, int prefetchCount, NodeState rootState) {
- return new PrefetchCursor(cursor, store, prefetchCount, rootState);
+ public static Cursor newPrefetchCursor(Cursor cursor, PrefetchNodeStore store, int prefetchCount,
+ NodeState rootState, List<String> prefetchRelative) {
+ return new PrefetchCursor(cursor, store, prefetchCount, rootState, prefetchRelative);
}
/**
@@ -573,24 +576,27 @@ public class Cursors {
private final int prefetchCount;
private final NodeState rootState;
private Iterator<IndexRow> prefetched;
+ private final List<String> prefetchRelative;
- PrefetchCursor(Cursor cursor, PrefetchNodeStore store, int prefetchCount, NodeState rootState) {
+ PrefetchCursor(Cursor cursor, PrefetchNodeStore store, int prefetchCount, NodeState rootState, List<String> prefetchRelative) {
this.cursor = cursor;
this.store = store;
this.prefetchCount = prefetchCount;
this.rootState = rootState;
this.prefetched = Iterators.emptyIterator();
+ this.prefetchRelative = prefetchRelative;
}
@Override
public IndexRow next() {
if (!prefetched.hasNext()) {
ArrayList<IndexRow> rows = new ArrayList<>();
- HashSet<String> paths = new HashSet<>();
+ TreeSet<String> paths = new TreeSet<>();
for (int i = 0; i < prefetchCount && cursor.hasNext(); i++) {
IndexRow row = cursor.next();
rows.add(row);
String p = row.getPath();
+ prefetchRelative(paths, p);
do {
paths.add(p);
p = PathUtils.getParentPath(p);
@@ -602,6 +608,13 @@ public class Cursors {
return prefetched.next();
}
+ private void prefetchRelative(Set<String> target, String p) {
+ for (String r : prefetchRelative) {
+ String concat = PathUtils.concat(p, r);
+ target.add(concat);
+ }
+ }
+
@Override
public boolean hasNext() {
return prefetched.hasNext() || cursor.hasNext();
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java
index 57062f5bd9..1820ed5ef6 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java
@@ -176,7 +176,7 @@ public abstract class QueryEngineImpl implements QueryEngine {
parser.setAllowNumberLiterals(false);
parser.setAllowTextLiterals(false);
}
-
+
ArrayList<Query> queries = new ArrayList<Query>();
Query q;
@@ -187,7 +187,7 @@ public abstract class QueryEngineImpl implements QueryEngine {
parser.setSupportSQL1(true);
q = parser.parse(statement, false);
} else if (XPATH.equals(language)) {
- XPathToSQL2Converter converter = new XPathToSQL2Converter();
+ XPathToSQL2Converter converter = new XPathToSQL2Converter(settings);
String sql2 = converter.convert(statement);
LOG.debug("XPath > SQL2: {}", sql2);
try {
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java
index 5dd6af4a65..4719ddc1f5 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java
@@ -106,6 +106,8 @@ public class QueryEngineSettings implements QueryEngineSettingsMBean, QueryLimit
private final long queryLengthWarnLimit = Long.getLong(OAK_QUERY_LENGTH_WARN_LIMIT, 1024 * 1024); // 1 MB
private final long queryLengthErrorLimit = Long.getLong(OAK_QUERY_LENGTH_ERROR_LIMIT, 100 * 1024 * 1024); //100MB
+ private String autoOptionsMappingJson = "{}";
+ private QueryOptions.AutomaticQueryOptionsMapping autoOptionsMapping = new QueryOptions.AutomaticQueryOptionsMapping(autoOptionsMappingJson);
public long getQueryLengthWarnLimit() {
return queryLengthWarnLimit;
@@ -153,6 +155,21 @@ public class QueryEngineSettings implements QueryEngineSettingsMBean, QueryLimit
return prefetchCount;
}
+ @Override
+ public void setAutoOptionsMappingJson(String json) {
+ autoOptionsMappingJson = json;
+ autoOptionsMapping = new QueryOptions.AutomaticQueryOptionsMapping(json);
+ }
+
+ @Override
+ public String getAutoOptionsMappingJson() {
+ return autoOptionsMappingJson;
+ }
+
+ public QueryOptions.AutomaticQueryOptionsMapping getAutomaticQueryOptions() {
+ return autoOptionsMapping;
+ }
+
@Override
public boolean getFailTraversal() {
return failTraversal;
@@ -223,7 +240,7 @@ public class QueryEngineSettings implements QueryEngineSettingsMBean, QueryLimit
public void setIgnoredClassNamesInCallTrace(@NotNull String[] packageNames) {
classNamesIgnoredInCallTrace = packageNames;
}
-
+
public @NotNull String[] getIgnoredClassNamesInCallTrace() {
return classNamesIgnoredInCallTrace;
}
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryOptions.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryOptions.java
index 8f2a4625b7..832fa6683c 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryOptions.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryOptions.java
@@ -13,18 +13,32 @@
*/
package org.apache.jackrabbit.oak.query;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.Optional;
+import org.apache.jackrabbit.oak.commons.json.JsonObject;
+import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.query.stats.QueryRecorder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* A query options (or "hints") that are used to customize the way the query is processed.
*/
public class QueryOptions {
+ private static final Logger LOG = LoggerFactory.getLogger(QueryOptions.class);
+
public Traversal traversal = Traversal.DEFAULT;
public String indexName;
public String indexTag;
public Optional<Long> limit = Optional.empty();
public Optional<Long> offset = Optional.empty();
+ public List<String> prefetch = Collections.emptyList();
public enum Traversal {
// traversing without index is OK for this query, and does not fail or log a warning
@@ -37,4 +51,76 @@ public class QueryOptions {
DEFAULT
};
+ public QueryOptions() {
+ }
+
+ public QueryOptions(QueryOptions defaultValues) {
+ traversal = defaultValues.traversal;
+ indexName = defaultValues.indexName;
+ indexTag = defaultValues.indexTag;
+ limit = defaultValues.limit;
+ offset = defaultValues.offset;
+ prefetch = defaultValues.prefetch;
+ }
+
+ QueryOptions(JsonObject json) {
+ Map<String, String> map = json.getProperties();
+ String x = map.get("traversal");
+ if (x != null) {
+ traversal = Traversal.valueOf(x);
+ }
+ x = map.get("indexName");
+ if (x != null) {
+ indexName = x;
+ }
+ x = map.get("indexTag");
+ if (x != null) {
+ indexTag = x;
+ }
+ x = map.get("limit");
+ if (x != null) {
+ try {
+ limit = Optional.of(Long.parseLong(x));
+ } catch (NumberFormatException e) {
+ LOG.warn("Invalid limit {}", x);
+ }
+ }
+ x = map.get("prefetch");
+ if (x != null) {
+ ArrayList<String> list = new ArrayList<>();
+ JsopTokenizer t = new JsopTokenizer(x);
+ t.read('[');
+ do {
+ list.add(t.readString());
+ } while (t.matches(','));
+ t.read(']');
+ prefetch = list;
+ }
+ }
+
+ public static class AutomaticQueryOptionsMapping {
+
+ private final HashMap<String, QueryOptions> map = new HashMap<>();
+
+ public AutomaticQueryOptionsMapping(String json) {
+ try {
+ JsonObject obj = JsonObject.fromJson(json, true);
+ for (Map.Entry<String, JsonObject> e : obj.getChildren().entrySet()) {
+ String statement = e.getKey();
+ QueryOptions options = new QueryOptions(e.getValue());
+ String queryPattern = QueryRecorder.simplifySafely(statement);
+ map.put(queryPattern, options);
+ }
+ } catch (IllegalArgumentException e) {
+ LOG.warn("Can not parse {}", json, e);
+ }
+ }
+
+ public QueryOptions getDefaultValues(String statement) {
+ String queryPattern = QueryRecorder.simplifySafely(statement);
+ return map.computeIfAbsent(queryPattern, v -> new QueryOptions());
+ }
+
+ }
+
}
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/SQL2Parser.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/SQL2Parser.java
index 14835cdb0c..949d136c26 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/SQL2Parser.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/SQL2Parser.java
@@ -169,7 +169,15 @@ public class SQL2Parser {
read("BY");
orderings = parseOrder();
}
- QueryOptions options = new QueryOptions();
+
+ QueryOptions defaultOptions = settings.getAutomaticQueryOptions().getDefaultValues(query);
+ QueryOptions options = new QueryOptions(defaultOptions);
+ if (options.limit.isPresent()) {
+ q.setLimit(options.limit.get());
+ }
+ if (options.offset.isPresent()) {
+ q.setLimit(options.offset.get());
+ }
if (readIf("OPTION")) {
read("(");
while (true) {
@@ -186,6 +194,15 @@ public class SQL2Parser {
q.setOffset(readNumber());
} else if (readIf("LIMIT")) {
q.setLimit(readNumber());
+ } else if (readIf("PREFETCH")) {
+ read("(");
+ ArrayList<String> list = new ArrayList<String>();
+ do {
+ String x = readString().getValue(Type.STRING);
+ list.add(x);
+ } while (readIf(","));
+ read(")");
+ options.prefetch = list;
} else {
break;
}
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java
index a960ef6fff..db1daaa6ee 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java
@@ -371,7 +371,8 @@ public class SelectorImpl extends SourceImpl {
int prefetchCount = query.getExecutionContext().getSettings().getPrefetchCount();
if (prefetchCount > 0) {
PrefetchNodeStore store = query.getExecutionContext().getPrefetchNodeStore();
- cursor = Cursors.newPrefetchCursor(cursor, store, prefetchCount, rootState);
+ cursor = Cursors.newPrefetchCursor(cursor, store, prefetchCount,
+ rootState, query.getQueryOptions().prefetch);
}
}
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryRecorder.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryRecorder.java
index fe83f08928..2117addab8 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryRecorder.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryRecorder.java
@@ -66,12 +66,17 @@ public class QueryRecorder {
if (LOG.isTraceEnabled()) {
LOG.trace("query:\t{}", query);
}
+ query = simplifySafely(query);
+ record(query);
+ }
+
+ public static String simplifySafely(String query) {
try {
- query = simplify(query);
+ return simplify(query);
} catch (Exception e) {
LOG.trace("failed to simplify {}", query, e);
+ return query;
}
- record(query);
}
public static String simplify(String query) {
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsData.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsData.java
index 0477eb024b..389b272683 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsData.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsData.java
@@ -138,12 +138,6 @@ public class QueryStatsData {
public class QueryExecutionStats {
long time;
- private final long SLOW_QUERY_HISTOGRAM = 1;
- private final long TOTAL_QUERY_HISTOGRAM = 0;
- private final String SLOW_QUERY_PERCENTILE_METRICS_NAME = "SLOW_QUERY_PERCENTILE_METRICS";
- private final String SLOW_QUERY_COUNT_NAME = "SLOW_QUERY_COUNT";
- private final int SLOW_QUERY_LIMIT_SCANNED =
- Integer.getInteger("oak.query.slowScanLimit", 100000);
public void execute(long nanos) {
QueryRecorder.record(query, internal);
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java
index 59fc26ff08..149987ef09 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java
@@ -18,15 +18,17 @@ package org.apache.jackrabbit.oak.query.xpath;
import java.util.ArrayList;
import java.util.List;
-import java.util.stream.Collectors;
import org.apache.jackrabbit.oak.query.QueryOptions;
+import org.apache.jackrabbit.oak.query.SQL2Parser;
import org.apache.jackrabbit.oak.query.QueryOptions.Traversal;
import org.apache.jackrabbit.oak.query.xpath.Expression.AndCondition;
import org.apache.jackrabbit.oak.query.xpath.Expression.OrCondition;
import org.apache.jackrabbit.oak.query.xpath.Expression.Property;
import org.apache.jackrabbit.oak.spi.query.QueryConstants;
+import com.google.common.collect.Lists;
+
/**
* An xpath statement.
*/
@@ -371,6 +373,12 @@ public class Statement {
if (queryOptions.limit.isPresent()) {
optionValues.add("limit " + queryOptions.limit.get());
}
+ if (!queryOptions.prefetch.isEmpty()) {
+ String list = String.join(", ",
+ Lists.transform(queryOptions.prefetch,
+ SQL2Parser::escapeStringLiteral));
+ optionValues.add("prefetch (" + list + ")");
+ }
buff.append(String.join(", ", optionValues));
buff.append(")");
}
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/XPathToSQL2Converter.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/XPathToSQL2Converter.java
index d2f8aa7f03..a0e9c549fe 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/XPathToSQL2Converter.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/XPathToSQL2Converter.java
@@ -23,6 +23,7 @@ import java.util.Locale;
import java.util.Optional;
import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.query.QueryEngineSettings;
import org.apache.jackrabbit.oak.query.QueryOptions;
import org.apache.jackrabbit.oak.query.QueryOptions.Traversal;
import org.apache.jackrabbit.oak.query.xpath.Statement.UnionStatement;
@@ -77,6 +78,16 @@ public class XPathToSQL2Converter {
private Selector currentSelector = new Selector();
private ArrayList<Selector> selectors = new ArrayList<Selector>();
+ private final QueryEngineSettings settings;
+
+ public XPathToSQL2Converter() {
+ this(null);
+ }
+
+ public XPathToSQL2Converter(QueryEngineSettings settings) {
+ this.settings = settings;
+ }
+
/**
* Convert the query to SQL2.
*
@@ -387,7 +398,12 @@ public class XPathToSQL2Converter {
QueryOptions options = null;
if (readIf("option")) {
read("(");
- options = new QueryOptions();
+ if (settings == null) {
+ options = new QueryOptions();
+ } else {
+ QueryOptions defaultOptions = settings.getAutomaticQueryOptions().getDefaultValues(query);
+ options = new QueryOptions(defaultOptions);
+ }
while (true) {
if (readIf("traversal")) {
String type = readIdentifier().toUpperCase(Locale.ENGLISH);
@@ -402,7 +418,16 @@ public class XPathToSQL2Converter {
options.offset = Optional.of(readNumber());
} else if (readIf("limit")) {
options.limit = Optional.of(readNumber());
- } else {
+ } else if (readIf("prefetch")) {
+ read("(");
+ ArrayList<String> list = new ArrayList<String>();
+ do {
+ String x = readString();
+ list.add(x);
+ } while (readIf(","));
+ read(")");
+ options.prefetch = list;
+ } else {
break;
}
readIf(",");
@@ -853,6 +878,15 @@ public class XPathToSQL2Converter {
}
}
+ private String readString() throws ParseException {
+ if (currentTokenType != VALUE_STRING) {
+ throw getSyntaxError("string value");
+ }
+ String result = currentToken;
+ read();
+ return result;
+ }
+
private Expression.Property readProperty() throws ParseException {
if (readIf("*")) {
return new Expression.Property(currentSelector, "*", false);
@@ -1190,7 +1224,7 @@ public class XPathToSQL2Converter {
int startParseIndex) throws ParseException {
int start = query.indexOf("(", startParseIndex);
String begin = query.substring(0, start);
- XPathToSQL2Converter converter = new XPathToSQL2Converter();
+ XPathToSQL2Converter converter = new XPathToSQL2Converter(settings);
String partList = query.substring(start);
converter.initialize(partList);
converter.read();
@@ -1226,7 +1260,7 @@ public class XPathToSQL2Converter {
QueryOptions queryOptions = null;
for(String p : parts) {
String q = begin + p + end;
- converter = new XPathToSQL2Converter();
+ converter = new XPathToSQL2Converter(settings);
Statement stat = converter.convertToStatement(q);
orderList = stat.orderList;
queryOptions = stat.queryOptions;