You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2021/10/13 21:37:28 UTC
[lucene-solr] branch branch_8x updated: LUCENE-10103 Make
QueryCache respect Accountable queries (#346) (#2591)
This is an automated email from the ASF dual-hosted git repository.
mikemccand pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 55a05f1 LUCENE-10103 Make QueryCache respect Accountable queries (#346) (#2591)
55a05f1 is described below
commit 55a05f14ded09c162313ee930067c20aa354d44f
Author: Patrick Zhai <zh...@users.noreply.github.com>
AuthorDate: Wed Oct 13 14:37:09 2021 -0700
LUCENE-10103 Make QueryCache respect Accountable queries (#346) (#2591)
---
lucene/CHANGES.txt | 2 ++
.../org/apache/lucene/search/LRUQueryCache.java | 7 +++-
.../search/MultiTermQueryConstantScoreWrapper.java | 17 ++++++++-
.../apache/lucene/search/TestLRUQueryCache.java | 41 ++++++++++++++++++++++
4 files changed, 65 insertions(+), 2 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 07fd306..291d080 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -19,6 +19,8 @@ Improvements
* LUCENE-9662: Make CheckIndex concurrent by parallelizing index check across segments.
(Zach Chen, Mike McCandless, Dawid Weiss, Robert Muir)
+* LUCENE-10103: Make QueryCache respect Accountable queries. (Haoyu Zhai)
+
Optimizations
---------------------
(No changes)
diff --git a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
index 4291222..9cb5f7b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
+++ b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
@@ -301,7 +301,12 @@ public class LRUQueryCache implements QueryCache, Accountable {
try {
Query singleton = uniqueQueries.putIfAbsent(query, query);
if (singleton == null) {
- onQueryCache(query, LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY + QUERY_DEFAULT_RAM_BYTES_USED);
+ if (query instanceof Accountable) {
+ onQueryCache(
+ query, LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY + ((Accountable) query).ramBytesUsed());
+ } else {
+ onQueryCache(query, LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY + QUERY_DEFAULT_RAM_BYTES_USED);
+ }
} else {
query = singleton;
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
index 2d7ac9b..c195bca 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java
@@ -30,8 +30,10 @@ import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.DocIdSetBuilder;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* This class also provides the functionality behind
@@ -41,11 +43,24 @@ import org.apache.lucene.util.DocIdSetBuilder;
* bit set with matches and builds a Scorer on top of
* this bit set.
*/
-final class MultiTermQueryConstantScoreWrapper<Q extends MultiTermQuery> extends Query {
+final class MultiTermQueryConstantScoreWrapper<Q extends MultiTermQuery> extends Query
+ implements Accountable {
// mtq that matches 16 terms or less will be executed as a regular disjunction
private static final int BOOLEAN_REWRITE_TERM_COUNT_THRESHOLD = 16;
+ @Override
+ public long ramBytesUsed() {
+ if (query instanceof Accountable) {
+ return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
+ + RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ + ((Accountable) query).ramBytesUsed();
+ }
+ return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
+ + RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ + RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED;
+ }
+
private static class TermAndState {
final BytesRef term;
final TermState state;
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
index 728d263..63e7bc6 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
@@ -59,6 +59,7 @@ import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@@ -465,6 +466,46 @@ public class TestLRUQueryCache extends LuceneTestCase {
dir.close();
}
+ /** DummyQuery with Accountable, pretending to be a memory-eating query */
+ private class AccountableDummyQuery extends DummyQuery implements Accountable {
+
+ @Override
+ public long ramBytesUsed() {
+ return 10 * QUERY_DEFAULT_RAM_BYTES_USED;
+ }
+ }
+
+ public void testCachingAccountableQuery() throws IOException {
+ final LRUQueryCache queryCache =
+ new LRUQueryCache(1000000, 10000000, context -> true, Float.POSITIVE_INFINITY);
+
+ Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ final int numDocs = atLeast(100);
+ for (int i = 0; i < numDocs; ++i) {
+ w.addDocument(doc);
+ }
+ final DirectoryReader reader = w.getReader();
+ final IndexSearcher searcher = new IndexSearcher(reader);
+ searcher.setQueryCache(queryCache);
+ searcher.setQueryCachingPolicy(ALWAYS_CACHE);
+
+ final int numQueries = random().nextInt(100) + 100;
+ for (int i = 0; i < numQueries; ++i) {
+ final Query query = new AccountableDummyQuery();
+ searcher.count(query);
+ }
+ long queryRamBytesUsed =
+ numQueries * (10 * QUERY_DEFAULT_RAM_BYTES_USED + LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY);
+ // allow 10% error for other ram bytes used estimation inside query cache
+ assertEquals(queryRamBytesUsed, queryCache.ramBytesUsed(), 10 * queryRamBytesUsed / 100);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
public void testOnUse() throws IOException {
final LRUQueryCache queryCache = new LRUQueryCache(1 + random().nextInt(5), 1 + random().nextInt(1000), context -> random().nextBoolean(), Float.POSITIVE_INFINITY);