You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2017/02/28 17:27:12 UTC
[45/50] [abbrv] lucene-solr:jira/solr-9045: LUCENE-7410: Make cache
keys and close listeners less trappy.
LUCENE-7410: Make cache keys and close listeners less trappy.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/df6f8307
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/df6f8307
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/df6f8307
Branch: refs/heads/jira/solr-9045
Commit: df6f83072303b4891a296b700a50c743284d3c30
Parents: 8e65aca
Author: Adrien Grand <jp...@gmail.com>
Authored: Tue Feb 28 14:21:30 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Tue Feb 28 14:46:45 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
lucene/MIGRATE.txt | 8 +
.../apache/lucene/index/FixBrokenOffsets.java | 10 +
.../apache/lucene/codecs/DocValuesConsumer.java | 4 +-
.../lucene/index/ExitableDirectoryReader.java | 21 +-
.../apache/lucene/index/FilterCodecReader.java | 13 +-
.../apache/lucene/index/FilterLeafReader.java | 73 +---
.../org/apache/lucene/index/IndexReader.java | 115 +++--
.../org/apache/lucene/index/LeafReader.java | 84 +---
.../apache/lucene/index/MergeReaderWrapper.java | 20 +-
.../org/apache/lucene/index/MultiDocValues.java | 22 +-
.../org/apache/lucene/index/MultiReader.java | 11 +
.../lucene/index/ParallelCompositeReader.java | 16 +-
.../apache/lucene/index/ParallelLeafReader.java | 36 +-
.../apache/lucene/index/SegmentCoreReaders.java | 39 +-
.../org/apache/lucene/index/SegmentReader.java | 57 ++-
.../lucene/index/SlowCodecReaderWrapper.java | 8 +-
.../apache/lucene/index/SortingLeafReader.java | 12 +
.../lucene/index/StandardDirectoryReader.java | 42 ++
.../org/apache/lucene/search/LRUQueryCache.java | 39 +-
.../index/TestDemoParallelLeafReader.java | 11 +-
.../lucene/index/TestDirectoryReader.java | 8 +-
.../lucene/index/TestDirectoryReaderReopen.java | 12 +-
.../index/TestExitableDirectoryReader.java | 10 +
.../lucene/index/TestFilterDirectoryReader.java | 5 +
.../lucene/index/TestFilterLeafReader.java | 21 +-
.../lucene/index/TestIndexReaderClose.java | 62 ++-
.../apache/lucene/index/TestMultiTermsEnum.java | 10 +
.../index/TestParallelCompositeReader.java | 33 +-
.../lucene/search/TermInSetQueryTest.java | 17 +-
.../apache/lucene/search/TestLRUQueryCache.java | 61 ++-
.../lucene/search/TestSearcherManager.java | 15 +
.../org/apache/lucene/search/TestTermQuery.java | 15 +
.../apache/lucene/search/TestTermScorer.java | 10 +
.../DefaultSortedSetDocValuesReaderState.java | 3 +-
.../facet/taxonomy/CachedOrdinalsReader.java | 7 +-
.../taxonomy/OrdinalMappingLeafReader.java | 10 +
.../search/highlight/TermVectorLeafReader.java | 20 +-
.../highlight/WeightedSpanTermExtractor.java | 10 +
.../lucene/search/uhighlight/PhraseHelper.java | 10 +
.../TermVectorFilteredLeafReader.java | 10 +
.../search/uhighlight/UnifiedHighlighter.java | 15 +
.../TestUnifiedHighlighterTermVec.java | 15 +
.../lucene/search/join/QueryBitSetProducer.java | 14 +-
.../apache/lucene/search/join/TestJoinUtil.java | 10 +-
.../search/join/TestQueryBitSetProducer.java | 110 +++++
.../apache/lucene/index/memory/MemoryIndex.java | 20 +-
.../lucene/index/MultiPassIndexSplitter.java | 15 +
.../apache/lucene/index/PKIndexSplitter.java | 10 +
.../nrt/SegmentInfosSearcherManager.java | 8 +-
.../lucene/index/AllDeletedFilterReader.java | 10 +
.../lucene/index/AssertingDirectoryReader.java | 9 +-
.../lucene/index/AssertingLeafReader.java | 30 +-
.../index/BaseStoredFieldsFormatTestCase.java | 15 +
.../lucene/index/FieldFilterLeafReader.java | 12 +-
.../lucene/index/MismatchedDirectoryReader.java | 5 +
.../lucene/index/MismatchedLeafReader.java | 10 +
.../lucene/index/MockRandomMergePolicy.java | 13 +-
.../org/apache/lucene/search/QueryUtils.java | 43 +-
.../org/apache/lucene/util/LuceneTestCase.java | 30 +-
.../src/java/org/apache/solr/core/SolrCore.java | 13 +-
.../solr/handler/component/ExpandComponent.java | 21 +-
.../solr/highlight/DefaultSolrHighlighter.java | 10 +
.../solr/index/SlowCompositeReaderWrapper.java | 35 +-
.../schema/RptWithGeometrySpatialField.java | 7 +-
.../solr/search/CollapsingQParserPlugin.java | 19 +-
.../java/org/apache/solr/search/Insanity.java | 9 +-
.../org/apache/solr/uninverting/FieldCache.java | 18 +-
.../apache/solr/uninverting/FieldCacheImpl.java | 70 +--
.../uninverting/FieldCacheSanityChecker.java | 426 -------------------
.../solr/uninverting/UninvertingReader.java | 21 +-
.../apache/solr/update/SolrIndexSplitter.java | 10 +
.../test/org/apache/solr/core/TestNRTOpen.java | 2 +-
.../index/TestSlowCompositeReaderWrapper.java | 53 ++-
.../test/org/apache/solr/search/TestDocSet.java | 20 +-
.../apache/solr/search/TestSolr4Spatial2.java | 2 +-
.../solr/uninverting/TestDocTermOrds.java | 4 +-
.../apache/solr/uninverting/TestFieldCache.java | 4 +-
.../TestFieldCacheSanityChecker.java | 164 -------
.../solr/uninverting/TestLegacyFieldCache.java | 35 +-
80 files changed, 1089 insertions(+), 1242 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 21a29c3..6026654 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -33,6 +33,10 @@ API Changes
* LUCENE-7494: Points now have a per-field API, like doc values. (Adrien Grand)
+* LUCENE-7410: Cache keys and close listeners have been refactored in order
+ to be less trappy. See IndexReader.getReaderCacheHelper and
+ LeafReader.getCoreCacheHelper. (Adrien Grand)
+
Bug Fixes
* LUCENE-7626: IndexWriter will no longer accept broken token offsets
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/MIGRATE.txt
----------------------------------------------------------------------
diff --git a/lucene/MIGRATE.txt b/lucene/MIGRATE.txt
index 06e6a81..51f6435 100644
--- a/lucene/MIGRATE.txt
+++ b/lucene/MIGRATE.txt
@@ -47,3 +47,11 @@ queries.
This option has been removed as expanded terms are now normalized through
Analyzer#normalize.
+
+## Cache key and close listener refactoring (LUCENE-7410)
+
+The way to access cache keys and add close listeners has been refactored in
+order to be less trappy. You should now use IndexReader.getReaderCacheHelper()
+to have manage caches that take deleted docs and doc values updates into
+account, and LeafReader.getCoreCacheHelper() to manage per-segment caches that
+do not take deleted docs and doc values updates into account.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/backward-codecs/src/java/org/apache/lucene/index/FixBrokenOffsets.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/index/FixBrokenOffsets.java b/lucene/backward-codecs/src/java/org/apache/lucene/index/FixBrokenOffsets.java
index d4d6f85..e775a28 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/index/FixBrokenOffsets.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/index/FixBrokenOffsets.java
@@ -114,6 +114,16 @@ public class FixBrokenOffsets {
}
};
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
});
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
index 3d06b51..88e34f6 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
@@ -521,7 +521,7 @@ public abstract class DocValuesConsumer implements Closeable {
}
// step 2: create ordinal map (this conceptually does the "merging")
- final OrdinalMap map = OrdinalMap.build(this, liveTerms, weights, PackedInts.COMPACT);
+ final OrdinalMap map = OrdinalMap.build(null, liveTerms, weights, PackedInts.COMPACT);
// step 3: add field
addSortedField(fieldInfo,
@@ -689,7 +689,7 @@ public abstract class DocValuesConsumer implements Closeable {
}
// step 2: create ordinal map (this conceptually does the "merging")
- final OrdinalMap map = OrdinalMap.build(this, liveTerms, weights, PackedInts.COMPACT);
+ final OrdinalMap map = OrdinalMap.build(null, liveTerms, weights, PackedInts.COMPACT);
// step 3: add field
addSortedSetField(mergeFieldInfo,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java
index ee1c0ce..a9b7472 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java
@@ -88,17 +88,19 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {
return fields; // break out of wrapper as soon as possible
}
}
-
+
+ // this impl does not change deletes or data so we can delegate the
+ // CacheHelpers
@Override
- public Object getCoreCacheKey() {
- return in.getCoreCacheKey();
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
}
-
+
@Override
- public Object getCombinedCoreAndDeletesKey() {
- return in.getCombinedCoreAndDeletesKey();
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
}
-
+
}
/**
@@ -211,6 +213,11 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {
}
@Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
+
+ @Override
public String toString() {
return "ExitableDirectoryReader(" + in.toString() + ")";
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
index c0ea8fc..5949fca 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
@@ -35,6 +35,9 @@ import org.apache.lucene.util.Bits;
* A <code>FilterCodecReader</code> contains another CodecReader, which it
* uses as its basic source of data, possibly transforming the data along the
* way or providing additional functionality.
+ * <p><b>NOTE</b>: If this {@link FilterCodecReader} does not change the
+ * content the contained reader, you could consider delegating calls to
+ * {@link #getCoreCacheHelper()} and {@link #getReaderCacheHelper()}.
*/
public abstract class FilterCodecReader extends CodecReader {
/**
@@ -106,16 +109,6 @@ public abstract class FilterCodecReader extends CodecReader {
}
@Override
- public void addCoreClosedListener(CoreClosedListener listener) {
- in.addCoreClosedListener(listener);
- }
-
- @Override
- public void removeCoreClosedListener(CoreClosedListener listener) {
- in.removeCoreClosedListener(listener);
- }
-
- @Override
protected void doClose() throws IOException {
in.doClose();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
index 9ed62e7..0a3ec7f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
@@ -19,9 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Iterator;
-import java.util.Objects;
-import org.apache.lucene.search.QueryCache;
import org.apache.lucene.search.Sort;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Bits;
@@ -38,12 +36,8 @@ import org.apache.lucene.util.BytesRef;
* <p><b>NOTE</b>: If you override {@link #getLiveDocs()}, you will likely need
* to override {@link #numDocs()} as well and vice-versa.
* <p><b>NOTE</b>: If this {@link FilterLeafReader} does not change the
- * content the contained reader, you could consider overriding
- * {@link #getCoreCacheKey()} so that
- * {@link QueryCache} impls share the same entries for this atomic reader
- * and the wrapped one. {@link #getCombinedCoreAndDeletesKey()} could be
- * overridden as well if the {@link #getLiveDocs() live docs} are not changed
- * either.
+ * content the contained reader, you could consider delegating calls to
+ * {@link #getCoreCacheHelper()} and {@link #getReaderCacheHelper()}.
*/
public abstract class FilterLeafReader extends LeafReader {
@@ -307,69 +301,6 @@ public abstract class FilterLeafReader extends LeafReader {
in.registerParentReader(this);
}
- /**
- * A CoreClosedListener wrapper that adjusts the core cache key that
- * the wrapper is called with. This is useful if the core cache key
- * of a reader is different from the key of the wrapped reader.
- */
- private static class CoreClosedListenerWrapper implements CoreClosedListener {
-
- public static CoreClosedListener wrap(CoreClosedListener listener, Object thisCoreKey, Object inCoreKey) {
- if (thisCoreKey == inCoreKey) {
- // this reader has the same core cache key as its parent, nothing to do
- return listener;
- } else {
- // we don't have the same cache key as the wrapped reader, we need to wrap
- // the listener to call it with the correct cache key
- return new CoreClosedListenerWrapper(listener, thisCoreKey, inCoreKey);
- }
- }
-
- private final CoreClosedListener in;
- private final Object thisCoreKey;
- private final Object inCoreKey;
-
- private CoreClosedListenerWrapper(CoreClosedListener in, Object thisCoreKey, Object inCoreKey) {
- this.in = in;
- this.thisCoreKey = thisCoreKey;
- this.inCoreKey = inCoreKey;
- }
-
- @Override
- public void onClose(Object ownerCoreCacheKey) throws IOException {
- assert inCoreKey == ownerCoreCacheKey;
- in.onClose(thisCoreKey);
- }
-
- // NOTE: equals/hashcore are important for removeCoreClosedListener to work
- // correctly
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null || obj.getClass() != CoreClosedListenerWrapper.class) {
- return false;
- }
- CoreClosedListenerWrapper that = (CoreClosedListenerWrapper) obj;
- return in.equals(that.in) && thisCoreKey == that.thisCoreKey;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(getClass(), in, thisCoreKey);
- }
-
- }
-
- @Override
- public void addCoreClosedListener(final CoreClosedListener listener) {
- in.addCoreClosedListener(CoreClosedListenerWrapper.wrap(listener, getCoreCacheKey(), in.getCoreCacheKey()));
- }
-
- @Override
- public void removeCoreClosedListener(CoreClosedListener listener) {
- in.removeCoreClosedListener(CoreClosedListenerWrapper.wrap(listener, getCoreCacheKey(), in.getCoreCacheKey()));
- }
-
@Override
public Bits getLiveDocs() {
ensureOpen();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexReader.java b/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
index 976f548..eb3a6db 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collections;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.WeakHashMap;
@@ -88,42 +87,48 @@ public abstract class IndexReader implements Closeable {
if (!(this instanceof CompositeReader || this instanceof LeafReader))
throw new Error("IndexReader should never be directly extended, subclass LeafReader or CompositeReader instead.");
}
-
+
/**
- * A custom listener that's invoked when the IndexReader
- * is closed.
- *
+ * A utility class that gives hooks in order to help build a cache based on
+ * the data that is contained in this index.
* @lucene.experimental
*/
- public static interface ReaderClosedListener {
- /** Invoked when the {@link IndexReader} is closed. */
- public void onClose(IndexReader reader) throws IOException;
- }
+ public static interface CacheHelper {
- private final Set<ReaderClosedListener> readerClosedListeners =
- Collections.synchronizedSet(new LinkedHashSet<ReaderClosedListener>());
+ /**
+ * Get a key that the resource can be cached on. The given entry can be
+ * compared using identity, ie. {@link Object#equals} is implemented as
+ * {@code ==} and {@link Object#hashCode} is implemented as
+ * {@link System#identityHashCode}.
+ */
+ CacheKey getKey();
- private final Set<IndexReader> parentReaders =
- Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<IndexReader,Boolean>()));
+ /**
+ * Add a {@link ClosedListener} which will be called when the resource
+ * guarded by {@link #getKey()} is closed.
+ */
+ void addClosedListener(ClosedListener listener);
- /** Expert: adds a {@link ReaderClosedListener}. The
- * provided listener will be invoked when this reader is closed.
- * At this point, it is safe for apps to evict this reader from
- * any caches keyed on {@link #getCombinedCoreAndDeletesKey()}.
- *
- * @lucene.experimental */
- public final void addReaderClosedListener(ReaderClosedListener listener) {
- ensureOpen();
- readerClosedListeners.add(listener);
}
- /** Expert: remove a previously added {@link ReaderClosedListener}.
- *
- * @lucene.experimental */
- public final void removeReaderClosedListener(ReaderClosedListener listener) {
- ensureOpen();
- readerClosedListeners.remove(listener);
+ /** A cache key identifying a resource that is being cached on. */
+ public static final class CacheKey {
+ CacheKey() {} // only instantiable by core impls
+ }
+
+ /**
+ * A listener that is called when a resource gets closed.
+ * @lucene.experimental
+ */
+ @FunctionalInterface
+ public static interface ClosedListener {
+ /** Invoked when the resource (segment core, or index reader) that is
+ * being cached on is closed. */
+ void onClose(CacheKey key) throws IOException;
}
+
+ private final Set<IndexReader> parentReaders =
+ Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<IndexReader,Boolean>()));
/** Expert: This method is called by {@code IndexReader}s which wrap other readers
* (e.g. {@link CompositeReader} or {@link FilterLeafReader}) to register the parent
@@ -136,21 +141,10 @@ public abstract class IndexReader implements Closeable {
parentReaders.add(reader);
}
- private void notifyReaderClosedListeners(Throwable th) throws IOException {
- synchronized(readerClosedListeners) {
- for(ReaderClosedListener listener : readerClosedListeners) {
- try {
- listener.onClose(this);
- } catch (Throwable t) {
- if (th == null) {
- th = t;
- } else {
- th.addSuppressed(t);
- }
- }
- }
- IOUtils.reThrow(th);
- }
+ // overridden by StandardDirectoryReader and SegmentReader
+ void notifyReaderClosedListeners(Throwable th) throws IOException {
+ // nothing to notify in the base impl, just rethrow
+ IOUtils.reThrow(th);
}
private void reportCloseToParentReaders() {
@@ -279,10 +273,8 @@ public abstract class IndexReader implements Closeable {
}
/** {@inheritDoc}
- * <p>For caching purposes, {@code IndexReader} subclasses are not allowed
+ * <p>{@code IndexReader} subclasses are not allowed
* to implement equals/hashCode, so methods are declared final.
- * To lookup instances from caches use {@link #getCoreCacheKey} and
- * {@link #getCombinedCoreAndDeletesKey}.
*/
@Override
public final boolean equals(Object obj) {
@@ -290,10 +282,8 @@ public abstract class IndexReader implements Closeable {
}
/** {@inheritDoc}
- * <p>For caching purposes, {@code IndexReader} subclasses are not allowed
+ * <p>{@code IndexReader} subclasses are not allowed
* to implement equals/hashCode, so methods are declared final.
- * To lookup instances from caches use {@link #getCoreCacheKey} and
- * {@link #getCombinedCoreAndDeletesKey}.
*/
@Override
public final int hashCode() {
@@ -436,24 +426,17 @@ public abstract class IndexReader implements Closeable {
return getContext().leaves();
}
- /** Expert: Returns a key for this IndexReader, so CachingWrapperFilter can find
- * it again.
- * This key must not have equals()/hashCode() methods, so "equals" means "identical". */
- public Object getCoreCacheKey() {
- // Don't call ensureOpen since FC calls this (to evict)
- // on close
- return this;
- }
+ /**
+ * Optional method: Return a {@link CacheHelper} that can be used to cache
+ * based on the content of this reader. Two readers that have different data
+ * or different sets of deleted documents will be considered different.
+ * <p>A return value of {@code null} indicates that this reader is not suited
+ * for caching, which is typically the case for short-lived wrappers that
+ * alter the content of the wrapped reader.
+ * @lucene.experimental
+ */
+ public abstract CacheHelper getReaderCacheHelper();
- /** Expert: Returns a key for this IndexReader that also includes deletions,
- * so CachingWrapperFilter can find it again.
- * This key must not have equals()/hashCode() methods, so "equals" means "identical". */
- public Object getCombinedCoreAndDeletesKey() {
- // Don't call ensureOpen since FC calls this (to evict)
- // on close
- return this;
- }
-
/** Returns the number of documents containing the
* <code>term</code>. This method returns 0 if the term or
* field does not exists. This method does not take into
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
index 73394f2..13c8646 100644
--- a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
@@ -18,7 +18,7 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.index.IndexReader.ReaderClosedListener;
+import org.apache.lucene.index.IndexReader.CacheHelper;
import org.apache.lucene.search.Sort;
import org.apache.lucene.util.Bits;
@@ -61,80 +61,18 @@ public abstract class LeafReader extends IndexReader {
}
/**
- * Called when the shared core for this {@link LeafReader}
- * is closed.
- * <p>
- * If this {@link LeafReader} impl has the ability to share
- * resources across instances that might only vary through
- * deleted documents and doc values updates, then this listener
- * will only be called when the shared core is closed.
- * Otherwise, this listener will be called when this reader is
- * closed.</p>
- * <p>
- * This is typically useful to manage per-segment caches: when
- * the listener is called, it is safe to evict this reader from
- * any caches keyed on {@link #getCoreCacheKey}.</p>
- *
+ * Optional method: Return a {@link CacheHelper} that can be used to cache
+ * based on the content of this leaf regardless of deletions. Two readers
+ * that have the same data but different sets of deleted documents or doc
+ * values updates may be considered equal. Consider using
+ * {@link #getReaderCacheHelper} if you need deletions or dv updates to be
+ * taken into account.
+ * <p>A return value of {@code null} indicates that this reader is not suited
+ * for caching, which is typically the case for short-lived wrappers that
+ * alter the content of the wrapped leaf reader.
* @lucene.experimental
*/
- public static interface CoreClosedListener {
- /** Invoked when the shared core of the original {@code
- * SegmentReader} has closed. The provided {@code
- * ownerCoreCacheKey} will be the same key as the one
- * returned by {@link LeafReader#getCoreCacheKey()}. */
- void onClose(Object ownerCoreCacheKey) throws IOException;
- }
-
- private static class CoreClosedListenerWrapper implements ReaderClosedListener {
-
- private final CoreClosedListener listener;
-
- CoreClosedListenerWrapper(CoreClosedListener listener) {
- this.listener = listener;
- }
-
- @Override
- public void onClose(IndexReader reader) throws IOException {
- listener.onClose(reader.getCoreCacheKey());
- }
-
- @Override
- public int hashCode() {
- return listener.hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (!(other instanceof CoreClosedListenerWrapper)) {
- return false;
- }
- return listener.equals(((CoreClosedListenerWrapper) other).listener);
- }
-
- }
-
- /** Add a {@link CoreClosedListener} as a {@link ReaderClosedListener}. This
- * method is typically useful for {@link LeafReader} implementations that
- * don't have the concept of a core that is shared across several
- * {@link LeafReader} instances in which case the {@link CoreClosedListener}
- * is called when closing the reader. */
- protected static void addCoreClosedListenerAsReaderClosedListener(IndexReader reader, CoreClosedListener listener) {
- reader.addReaderClosedListener(new CoreClosedListenerWrapper(listener));
- }
-
- /** Remove a {@link CoreClosedListener} which has been added with
- * {@link #addCoreClosedListenerAsReaderClosedListener(IndexReader, CoreClosedListener)}. */
- protected static void removeCoreClosedListenerAsReaderClosedListener(IndexReader reader, CoreClosedListener listener) {
- reader.removeReaderClosedListener(new CoreClosedListenerWrapper(listener));
- }
-
- /** Expert: adds a CoreClosedListener to this reader's shared core
- * @lucene.experimental */
- public abstract void addCoreClosedListener(CoreClosedListener listener);
-
- /** Expert: removes a CoreClosedListener from this reader's shared core
- * @lucene.experimental */
- public abstract void removeCoreClosedListener(CoreClosedListener listener);
+ public abstract CacheHelper getCoreCacheHelper();
/**
* Returns {@link Fields} for this reader.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
index 7eb90df..fffb693 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
@@ -71,16 +71,6 @@ class MergeReaderWrapper extends LeafReader {
}
@Override
- public void addCoreClosedListener(CoreClosedListener listener) {
- in.addCoreClosedListener(listener);
- }
-
- @Override
- public void removeCoreClosedListener(CoreClosedListener listener) {
- in.removeCoreClosedListener(listener);
- }
-
- @Override
public Fields fields() throws IOException {
return fields;
}
@@ -224,15 +214,15 @@ class MergeReaderWrapper extends LeafReader {
}
@Override
- public Object getCoreCacheKey() {
- return in.getCoreCacheKey();
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
}
@Override
- public Object getCombinedCoreAndDeletesKey() {
- return in.getCombinedCoreAndDeletesKey();
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
}
-
+
private void checkBounds(int docID) {
if (docID < 0 || docID >= maxDoc()) {
throw new IndexOutOfBoundsException("docID must be >= 0 and < maxDoc=" + maxDoc() + " (got docID=" + docID + ")");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
index 3970e0a..88dd6a1 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
@@ -598,7 +598,9 @@ public class MultiDocValues {
if (anyReal == false) {
return null;
} else {
- OrdinalMap mapping = OrdinalMap.build(r.getCoreCacheKey(), values, PackedInts.DEFAULT);
+ IndexReader.CacheHelper cacheHelper = r.getReaderCacheHelper();
+ IndexReader.CacheKey owner = cacheHelper == null ? null : cacheHelper.getKey();
+ OrdinalMap mapping = OrdinalMap.build(owner, values, PackedInts.DEFAULT);
return new MultiSortedDocValues(values, starts, mapping, totalCost);
}
}
@@ -640,7 +642,9 @@ public class MultiDocValues {
if (anyReal == false) {
return null;
} else {
- OrdinalMap mapping = OrdinalMap.build(r.getCoreCacheKey(), values, PackedInts.DEFAULT);
+ IndexReader.CacheHelper cacheHelper = r.getReaderCacheHelper();
+ IndexReader.CacheKey owner = cacheHelper == null ? null : cacheHelper.getKey();
+ OrdinalMap mapping = OrdinalMap.build(owner, values, PackedInts.DEFAULT);
return new MultiSortedSetDocValues(values, starts, mapping, totalCost);
}
}
@@ -710,9 +714,9 @@ public class MultiDocValues {
/**
* Create an ordinal map that uses the number of unique values of each
* {@link SortedDocValues} instance as a weight.
- * @see #build(Object, TermsEnum[], long[], float)
+ * @see #build(IndexReader.CacheKey, TermsEnum[], long[], float)
*/
- public static OrdinalMap build(Object owner, SortedDocValues[] values, float acceptableOverheadRatio) throws IOException {
+ public static OrdinalMap build(IndexReader.CacheKey owner, SortedDocValues[] values, float acceptableOverheadRatio) throws IOException {
final TermsEnum[] subs = new TermsEnum[values.length];
final long[] weights = new long[values.length];
for (int i = 0; i < values.length; ++i) {
@@ -725,9 +729,9 @@ public class MultiDocValues {
/**
* Create an ordinal map that uses the number of unique values of each
* {@link SortedSetDocValues} instance as a weight.
- * @see #build(Object, TermsEnum[], long[], float)
+ * @see #build(IndexReader.CacheKey, TermsEnum[], long[], float)
*/
- public static OrdinalMap build(Object owner, SortedSetDocValues[] values, float acceptableOverheadRatio) throws IOException {
+ public static OrdinalMap build(IndexReader.CacheKey owner, SortedSetDocValues[] values, float acceptableOverheadRatio) throws IOException {
final TermsEnum[] subs = new TermsEnum[values.length];
final long[] weights = new long[values.length];
for (int i = 0; i < values.length; ++i) {
@@ -748,7 +752,7 @@ public class MultiDocValues {
* to the other subs
* @throws IOException if an I/O error occurred.
*/
- public static OrdinalMap build(Object owner, TermsEnum subs[], long[] weights, float acceptableOverheadRatio) throws IOException {
+ public static OrdinalMap build(IndexReader.CacheKey owner, TermsEnum subs[], long[] weights, float acceptableOverheadRatio) throws IOException {
if (subs.length != weights.length) {
throw new IllegalArgumentException("subs and weights must have the same length");
}
@@ -761,7 +765,7 @@ public class MultiDocValues {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(OrdinalMap.class);
/** Cache key of whoever asked for this awful thing */
- public final Object owner;
+ public final IndexReader.CacheKey owner;
// globalOrd -> (globalOrd - segmentOrd) where segmentOrd is the the ordinal in the first segment that contains this term
final PackedLongValues globalOrdDeltas;
// globalOrd -> first segment container
@@ -773,7 +777,7 @@ public class MultiDocValues {
// ram usage
final long ramBytesUsed;
- OrdinalMap(Object owner, TermsEnum subs[], SegmentMap segmentMap, float acceptableOverheadRatio) throws IOException {
+ OrdinalMap(IndexReader.CacheKey owner, TermsEnum subs[], SegmentMap segmentMap, float acceptableOverheadRatio) throws IOException {
// create the ordinal mappings by pulling a termsenum over each sub's
// unique terms, and walking a multitermsenum over those
this.owner = owner;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/MultiReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiReader.java b/lucene/core/src/java/org/apache/lucene/index/MultiReader.java
index 8f1bb66..4d42382 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiReader.java
@@ -66,6 +66,17 @@ public class MultiReader extends BaseCompositeReader<IndexReader> {
}
@Override
+ public CacheHelper getReaderCacheHelper() {
+ // MultiReader instances can be short-lived, which would make caching trappy
+ // so we do not cache on them, unless they wrap a single reader in which
+ // case we delegate
+ if (getSequentialSubReaders().size() == 1) {
+ return getSequentialSubReaders().get(0).getReaderCacheHelper();
+ }
+ return null;
+ }
+
+ @Override
protected synchronized void doClose() throws IOException {
IOException ioe = null;
for (final IndexReader r : getSequentialSubReaders()) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java
index dd82976..4fc8a20 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java
@@ -51,6 +51,7 @@ public class ParallelCompositeReader extends BaseCompositeReader<LeafReader> {
private final boolean closeSubReaders;
private final Set<IndexReader> completeReaderSet =
Collections.newSetFromMap(new IdentityHashMap<IndexReader,Boolean>());
+ private final CacheHelper cacheHelper;
/** Create a ParallelCompositeReader based on the provided
* readers; auto-closes the given readers on {@link #close()}. */
@@ -80,6 +81,14 @@ public class ParallelCompositeReader extends BaseCompositeReader<LeafReader> {
}
// finally add our own synthetic readers, so we close or decRef them, too (it does not matter what we do)
completeReaderSet.addAll(getSequentialSubReaders());
+ // ParallelReader instances can be short-lived, which would make caching trappy
+ // so we do not cache on them, unless they wrap a single reader in which
+ // case we delegate
+ if (readers.length == 1 && storedFieldReaders.length == 1 && readers[0] == storedFieldReaders[0]) {
+ cacheHelper = readers[0].getReaderCacheHelper();
+ } else {
+ cacheHelper = null;
+ }
}
private static LeafReader[] prepareLeafReaders(CompositeReader[] readers, CompositeReader[] storedFieldsReaders) throws IOException {
@@ -142,7 +151,12 @@ public class ParallelCompositeReader extends BaseCompositeReader<LeafReader> {
}
}
}
-
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return cacheHelper;
+ }
+
@Override
protected synchronized void doClose() throws IOException {
IOException ioe = null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
index 60886ea..c67d07b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
@@ -159,16 +159,6 @@ public class ParallelLeafReader extends LeafReader {
return buffer.append(')').toString();
}
- @Override
- public void addCoreClosedListener(CoreClosedListener listener) {
- addCoreClosedListenerAsReaderClosedListener(this, listener);
- }
-
- @Override
- public void removeCoreClosedListener(CoreClosedListener listener) {
- removeCoreClosedListenerAsReaderClosedListener(this, listener);
- }
-
// Single instance of this, per ParallelReader instance
private final class ParallelFields extends Fields {
final Map<String,Terms> fields = new TreeMap<>();
@@ -242,6 +232,32 @@ public class ParallelLeafReader extends LeafReader {
}
@Override
+ public CacheHelper getCoreCacheHelper() {
+ // ParallelReader instances can be short-lived, which would make caching trappy
+ // so we do not cache on them, unless they wrap a single reader in which
+ // case we delegate
+ if (parallelReaders.length == 1
+ && storedFieldsReaders.length == 1
+ && parallelReaders[0] == storedFieldsReaders[0]) {
+ return parallelReaders[0].getCoreCacheHelper();
+ }
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ // ParallelReader instances can be short-lived, which would make caching trappy
+ // so we do not cache on them, unless they wrap a single reader in which
+ // case we delegate
+ if (parallelReaders.length == 1
+ && storedFieldsReaders.length == 1
+ && parallelReaders[0] == storedFieldsReaders[0]) {
+ return parallelReaders[0].getReaderCacheHelper();
+ }
+ return null;
+ }
+
+ @Override
public Fields getTermVectors(int docID) throws IOException {
ensureOpen();
ParallelFields fields = null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
index 270a2d5..99e503b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
@@ -33,7 +33,8 @@ import org.apache.lucene.codecs.PointsReader;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
-import org.apache.lucene.index.LeafReader.CoreClosedListener;
+import org.apache.lucene.index.IndexReader.CacheKey;
+import org.apache.lucene.index.IndexReader.ClosedListener;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -84,8 +85,8 @@ final class SegmentCoreReaders {
}
};
- private final Set<CoreClosedListener> coreClosedListeners =
- Collections.synchronizedSet(new LinkedHashSet<CoreClosedListener>());
+ private final Set<IndexReader.ClosedListener> coreClosedListeners =
+ Collections.synchronizedSet(new LinkedHashSet<IndexReader.ClosedListener>());
SegmentCoreReaders(Directory dir, SegmentCommitInfo si, IOContext context) throws IOException {
@@ -175,14 +176,32 @@ final class SegmentCoreReaders {
}
}
}
-
+
+ private final IndexReader.CacheHelper cacheHelper = new IndexReader.CacheHelper() {
+ private final IndexReader.CacheKey cacheKey = new IndexReader.CacheKey();
+
+ @Override
+ public CacheKey getKey() {
+ return cacheKey;
+ }
+
+ @Override
+ public void addClosedListener(ClosedListener listener) {
+ coreClosedListeners.add(listener);
+ }
+ };
+
+ IndexReader.CacheHelper getCacheHelper() {
+ return cacheHelper;
+ }
+
private void notifyCoreClosedListeners(Throwable th) throws IOException {
synchronized(coreClosedListeners) {
- for (CoreClosedListener listener : coreClosedListeners) {
+ for (IndexReader.ClosedListener listener : coreClosedListeners) {
// SegmentReader uses our instance as its
// coreCacheKey:
try {
- listener.onClose(this);
+ listener.onClose(cacheHelper.getKey());
} catch (Throwable t) {
if (th == null) {
th = t;
@@ -195,14 +214,6 @@ final class SegmentCoreReaders {
}
}
- void addCoreClosedListener(CoreClosedListener listener) {
- coreClosedListeners.add(listener);
- }
-
- void removeCoreClosedListener(CoreClosedListener listener) {
- coreClosedListeners.remove(listener);
- }
-
@Override
public String toString() {
return "SegmentCoreReader(" + segment + ")";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java b/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
index b01f0b8..5dbc492 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
@@ -19,6 +19,8 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesProducer;
@@ -32,6 +34,7 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.IOUtils;
/**
* IndexReader implementation over a single segment.
@@ -282,32 +285,48 @@ public final class SegmentReader extends CodecReader {
return si.info.dir;
}
- // This is necessary so that cloned SegmentReaders (which
- // share the underlying postings data) will map to the
- // same entry for CachingWrapperFilter. See LUCENE-1579.
- @Override
- public Object getCoreCacheKey() {
- // NOTE: if this ever changes, be sure to fix
- // SegmentCoreReader.notifyCoreClosedListeners to match!
- // Today it passes "this" as its coreCacheKey:
- return core;
- }
+ private final Set<ClosedListener> readerClosedListeners = new CopyOnWriteArraySet<>();
@Override
- public Object getCombinedCoreAndDeletesKey() {
- return this;
+ void notifyReaderClosedListeners(Throwable th) throws IOException {
+ synchronized(readerClosedListeners) {
+ for(ClosedListener listener : readerClosedListeners) {
+ try {
+ listener.onClose(cacheHelper.getKey());
+ } catch (Throwable t) {
+ if (th == null) {
+ th = t;
+ } else {
+ th.addSuppressed(t);
+ }
+ }
+ }
+ IOUtils.reThrow(th);
+ }
}
+ private final IndexReader.CacheHelper cacheHelper = new IndexReader.CacheHelper() {
+ private final IndexReader.CacheKey cacheKey = new IndexReader.CacheKey();
+
+ @Override
+ public CacheKey getKey() {
+ return cacheKey;
+ }
+
+ @Override
+ public void addClosedListener(ClosedListener listener) {
+ readerClosedListeners.add(listener);
+ }
+ };
+
@Override
- public void addCoreClosedListener(CoreClosedListener listener) {
- ensureOpen();
- core.addCoreClosedListener(listener);
+ public CacheHelper getReaderCacheHelper() {
+ return cacheHelper;
}
-
+
@Override
- public void removeCoreClosedListener(CoreClosedListener listener) {
- ensureOpen();
- core.removeCoreClosedListener(listener);
+ public CacheHelper getCoreCacheHelper() {
+ return core.getCacheHelper();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
index d5b5c33..99f35bc 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
@@ -113,13 +113,13 @@ public final class SlowCodecReaderWrapper {
}
@Override
- public void addCoreClosedListener(CoreClosedListener listener) {
- reader.addCoreClosedListener(listener);
+ public CacheHelper getCoreCacheHelper() {
+ return reader.getCoreCacheHelper();
}
@Override
- public void removeCoreClosedListener(CoreClosedListener listener) {
- reader.removeCoreClosedListener(listener);
+ public CacheHelper getReaderCacheHelper() {
+ return reader.getReaderCacheHelper();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
index f24a4d0..321b552 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
@@ -1246,4 +1246,16 @@ class SortingLeafReader extends FilterLeafReader {
public String toString() {
return "SortingLeafReader(" + in + ")";
}
+
+ // no caching on sorted views
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
index 7ac059e..46f81af 100644
--- a/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
@@ -25,6 +25,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
@@ -469,4 +471,44 @@ public final class StandardDirectoryReader extends DirectoryReader {
return reader;
}
}
+
+ private final Set<ClosedListener> readerClosedListeners = new CopyOnWriteArraySet<>();
+
+ private final CacheHelper cacheHelper = new CacheHelper() {
+ private final CacheKey cacheKey = new CacheKey();
+
+ @Override
+ public CacheKey getKey() {
+ return cacheKey;
+ }
+
+ @Override
+ public void addClosedListener(ClosedListener listener) {
+ readerClosedListeners.add(listener);
+ }
+
+ };
+
+ @Override
+ void notifyReaderClosedListeners(Throwable th) throws IOException {
+ synchronized(readerClosedListeners) {
+ for(ClosedListener listener : readerClosedListeners) {
+ try {
+ listener.onClose(cacheHelper.getKey());
+ } catch (Throwable t) {
+ if (th == null) {
+ th = t;
+ } else {
+ th.addSuppressed(t);
+ }
+ }
+ }
+ IOUtils.reThrow(th);
+ }
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return cacheHelper;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
index fcdf2a5..b1ba4e4 100644
--- a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
+++ b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
@@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Predicate;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.ReaderUtil;
@@ -107,7 +108,7 @@ public class LRUQueryCache implements QueryCache, Accountable {
// are only allowed to store sub-sets of the queries that are contained in
// mostRecentlyUsedQueries. This is why write operations are performed under a lock
private final Set<Query> mostRecentlyUsedQueries;
- private final Map<Object, LeafCache> cache;
+ private final Map<IndexReader.CacheKey, LeafCache> cache;
private final ReentrantLock lock;
// these variables are volatile so that we do not need to sync reads
@@ -264,11 +265,11 @@ public class LRUQueryCache implements QueryCache, Accountable {
}
}
- DocIdSet get(Query key, LeafReaderContext context) {
+ DocIdSet get(Query key, LeafReaderContext context, IndexReader.CacheHelper cacheHelper) {
assert lock.isHeldByCurrentThread();
assert key instanceof BoostQuery == false;
assert key instanceof ConstantScoreQuery == false;
- final Object readerKey = context.reader().getCoreCacheKey();
+ final IndexReader.CacheKey readerKey = cacheHelper.getKey();
final LeafCache leafCache = cache.get(readerKey);
if (leafCache == null) {
onMiss(readerKey, key);
@@ -289,7 +290,7 @@ public class LRUQueryCache implements QueryCache, Accountable {
return cached;
}
- void putIfAbsent(Query query, LeafReaderContext context, DocIdSet set) {
+ void putIfAbsent(Query query, LeafReaderContext context, DocIdSet set, IndexReader.CacheHelper cacheHelper) {
assert query instanceof BoostQuery == false;
assert query instanceof ConstantScoreQuery == false;
// under a lock to make sure that mostRecentlyUsedQueries and cache remain sync'ed
@@ -301,15 +302,15 @@ public class LRUQueryCache implements QueryCache, Accountable {
} else {
query = singleton;
}
- final Object key = context.reader().getCoreCacheKey();
+ final IndexReader.CacheKey key = cacheHelper.getKey();
LeafCache leafCache = cache.get(key);
if (leafCache == null) {
leafCache = new LeafCache(key);
- final LeafCache previous = cache.put(context.reader().getCoreCacheKey(), leafCache);
+ final LeafCache previous = cache.put(key, leafCache);
ramBytesUsed += HASHTABLE_RAM_BYTES_PER_ENTRY;
assert previous == null;
// we just created a new leaf cache, need to register a close listener
- context.reader().addCoreClosedListener(this::clearCoreCacheKey);
+ cacheHelper.addClosedListener(this::clearCoreCacheKey);
}
leafCache.putIfAbsent(query, set);
evictIfNecessary();
@@ -720,6 +721,14 @@ public class LRUQueryCache implements QueryCache, Accountable {
if (used.compareAndSet(false, true)) {
policy.onUse(getQuery());
}
+
+ // TODO: should it be pluggable, eg. for queries that run on doc values?
+ final IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper();
+ if (cacheHelper == null) {
+ // this segment is not suitable for caching
+ return in.scorer(context);
+ }
+
// Short-circuit: Check whether this segment is eligible for caching
// before we take a lock because of #get
if (shouldCache(context) == false) {
@@ -733,7 +742,7 @@ public class LRUQueryCache implements QueryCache, Accountable {
DocIdSet docIdSet;
try {
- docIdSet = get(in.getQuery(), context);
+ docIdSet = get(in.getQuery(), context, cacheHelper);
} finally {
lock.unlock();
}
@@ -741,7 +750,7 @@ public class LRUQueryCache implements QueryCache, Accountable {
if (docIdSet == null) {
if (policy.shouldCache(in.getQuery())) {
docIdSet = cache(context);
- putIfAbsent(in.getQuery(), context, docIdSet);
+ putIfAbsent(in.getQuery(), context, docIdSet, cacheHelper);
} else {
return in.scorer(context);
}
@@ -764,6 +773,14 @@ public class LRUQueryCache implements QueryCache, Accountable {
if (used.compareAndSet(false, true)) {
policy.onUse(getQuery());
}
+
+ // TODO: should it be pluggable, eg. for queries that run on doc values?
+ final IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper();
+ if (cacheHelper == null) {
+ // this segment is not suitable for caching
+ return in.bulkScorer(context);
+ }
+
// Short-circuit: Check whether this segment is eligible for caching
// before we take a lock because of #get
if (shouldCache(context) == false) {
@@ -777,7 +794,7 @@ public class LRUQueryCache implements QueryCache, Accountable {
DocIdSet docIdSet;
try {
- docIdSet = get(in.getQuery(), context);
+ docIdSet = get(in.getQuery(), context, cacheHelper);
} finally {
lock.unlock();
}
@@ -785,7 +802,7 @@ public class LRUQueryCache implements QueryCache, Accountable {
if (docIdSet == null) {
if (policy.shouldCache(in.getQuery())) {
docIdSet = cache(context);
- putIfAbsent(in.getQuery(), context, docIdSet);
+ putIfAbsent(in.getQuery(), context, docIdSet, cacheHelper);
} else {
return in.bulkScorer(context);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
index 7901214..34bde51 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
@@ -239,6 +239,11 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
// throw the first exception
IOUtils.reThrow(firstExc);
}
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
@Override
@@ -324,7 +329,7 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
}
}
- private class ParallelReaderClosed implements LeafReader.ReaderClosedListener {
+ private class ParallelReaderClosed implements IndexReader.ClosedListener {
private final SegmentIDAndGen segIDGen;
private final Directory dir;
@@ -334,7 +339,7 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
}
@Override
- public void onClose(IndexReader ignored) {
+ public void onClose(IndexReader.CacheKey ignored) {
try {
// TODO: make this sync finer, i.e. just the segment + schemaGen
synchronized(ReindexingReader.this) {
@@ -421,7 +426,7 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
// the pruning may remove our directory:
closedSegments.remove(segIDGen);
- parLeafReader.addReaderClosedListener(new ParallelReaderClosed(segIDGen, dir));
+ parLeafReader.getReaderCacheHelper().addClosedListener(new ParallelReaderClosed(segIDGen, dir));
} else {
// Used only for merged segment warming:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
index 9ac719e..7afcf7a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
@@ -927,14 +927,14 @@ public class TestDirectoryReader extends LuceneTestCase {
writer.commit();
final DirectoryReader reader = writer.getReader();
final int[] closeCount = new int[1];
- final IndexReader.ReaderClosedListener listener = new IndexReader.ReaderClosedListener() {
+ final IndexReader.ClosedListener listener = new IndexReader.ClosedListener() {
@Override
- public void onClose(IndexReader reader) {
+ public void onClose(IndexReader.CacheKey key) {
closeCount[0]++;
}
};
- reader.addReaderClosedListener(listener);
+ reader.getReaderCacheHelper().addClosedListener(listener);
reader.close();
@@ -943,7 +943,7 @@ public class TestDirectoryReader extends LuceneTestCase {
writer.close();
DirectoryReader reader2 = DirectoryReader.open(dir);
- reader2.addReaderClosedListener(listener);
+ reader2.getReaderCacheHelper().addClosedListener(listener);
closeCount[0] = 0;
reader2.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
index f415381..b38696a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
@@ -811,7 +811,8 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
assertEquals(1, oldest.leaves().size());
// sharing same core
- assertSame(latest.leaves().get(0).reader().getCoreCacheKey(), oldest.leaves().get(0).reader().getCoreCacheKey());
+ assertSame(latest.leaves().get(0).reader().getCoreCacheHelper().getKey(),
+ oldest.leaves().get(0).reader().getCoreCacheHelper().getKey());
latest.close();
oldest.close();
@@ -861,7 +862,8 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
assertEquals(1, oldest.leaves().size());
// sharing same core
- assertSame(latest.leaves().get(0).reader().getCoreCacheKey(), oldest.leaves().get(0).reader().getCoreCacheKey());
+ assertSame(latest.leaves().get(0).reader().getCoreCacheHelper().getKey(),
+ oldest.leaves().get(0).reader().getCoreCacheHelper().getKey());
latest.close();
oldest.close();
@@ -901,7 +903,8 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
assertEquals(1, oldest.leaves().size());
// sharing same core
- assertSame(latest.leaves().get(0).reader().getCoreCacheKey(), oldest.leaves().get(0).reader().getCoreCacheKey());
+ assertSame(latest.leaves().get(0).reader().getCoreCacheHelper().getKey(),
+ oldest.leaves().get(0).reader().getCoreCacheHelper().getKey());
NumericDocValues values = getOnlyLeafReader(oldest).getNumericDocValues("dv");
assertEquals(0, values.nextDoc());
@@ -948,7 +951,8 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
assertEquals(1, oldest.leaves().size());
// sharing same core
- assertSame(latest.leaves().get(0).reader().getCoreCacheKey(), oldest.leaves().get(0).reader().getCoreCacheKey());
+ assertSame(latest.leaves().get(0).reader().getCoreCacheHelper().getKey(),
+ oldest.leaves().get(0).reader().getCoreCacheHelper().getKey());
NumericDocValues values = getOnlyLeafReader(oldest).getNumericDocValues("dv");
assertEquals(0, values.nextDoc());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java
index 71406c8..3f424f5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java
@@ -86,6 +86,16 @@ public class TestExitableDirectoryReader extends LuceneTestCase {
public Fields fields() throws IOException {
return new TestFields(super.fields());
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/index/TestFilterDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterDirectoryReader.java
index 4ce86e2..62a4294 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFilterDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterDirectoryReader.java
@@ -49,6 +49,11 @@ public class TestFilterDirectoryReader extends LuceneTestCase {
protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
return new DummyFilterDirectoryReader(in);
}
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java
index e9f6fe2..79862fc 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java
@@ -106,6 +106,16 @@ public class TestFilterLeafReader extends LuceneTestCase {
public Fields fields() throws IOException {
return new TestFields(super.fields());
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
/**
@@ -196,7 +206,16 @@ public class TestFilterLeafReader extends LuceneTestCase {
w.addDocument(new Document());
DirectoryReader dr = w.getReader();
LeafReader r = dr.leaves().get(0).reader();
- FilterLeafReader r2 = new FilterLeafReader(r) {};
+ FilterLeafReader r2 = new FilterLeafReader(r) {
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
+ };
assertEquals(r, r2.getDelegate());
assertEquals(r, FilterLeafReader.unwrap(r2));
w.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java
index 91dcb6e..20088a5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
@@ -47,10 +46,21 @@ public class TestIndexReaderClose extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(open);
FilterLeafReader reader = new FilterLeafReader(leaf) {
@Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
+ @Override
protected void doClose() throws IOException {
- super.doClose();
- if (throwOnClose) {
- throw new IllegalStateException("BOOM!");
+ try {
+ super.doClose();
+ } finally {
+ if (throwOnClose) {
+ throw new IllegalStateException("BOOM!");
+ }
}
}
};
@@ -60,14 +70,14 @@ public class TestIndexReaderClose extends LuceneTestCase {
for (int i = 0; i < listenerCount; i++) {
if (rarely()) {
faultySet = true;
- reader.addReaderClosedListener(new FaultyListener());
+ reader.getReaderCacheHelper().addClosedListener(new FaultyListener());
} else {
count.incrementAndGet();
- reader.addReaderClosedListener(new CountListener(count));
+ reader.getReaderCacheHelper().addClosedListener(new CountListener(count));
}
}
if (!faultySet && !throwOnClose) {
- reader.addReaderClosedListener(new FaultyListener());
+ reader.getReaderCacheHelper().addClosedListener(new FaultyListener());
}
IllegalStateException expected = expectThrows(IllegalStateException.class, () -> {
@@ -106,31 +116,19 @@ public class TestIndexReaderClose extends LuceneTestCase {
w.close();
final IndexReader reader = DirectoryReader.open(w.w.getDirectory());
- // We explicitly define a different cache key
- final Object coreCacheKey = new Object();
- final LeafReader leafReader = new FilterLeafReader(getOnlyLeafReader(reader)) {
- @Override
- public Object getCoreCacheKey() {
- return coreCacheKey;
- }
- };
+ final LeafReader leafReader = new AssertingLeafReader(getOnlyLeafReader(reader));
final int numListeners = TestUtil.nextInt(random(), 1, 10);
- final List<LeafReader.CoreClosedListener> listeners = new ArrayList<>();
+ final List<IndexReader.ClosedListener> listeners = new ArrayList<>();
AtomicInteger counter = new AtomicInteger(numListeners);
-
+
for (int i = 0; i < numListeners; ++i) {
- CountCoreListener listener = new CountCoreListener(counter, coreCacheKey);
+ CountCoreListener listener = new CountCoreListener(counter, leafReader.getCoreCacheHelper().getKey());
listeners.add(listener);
- leafReader.addCoreClosedListener(listener);
+ leafReader.getCoreCacheHelper().addClosedListener(listener);
}
for (int i = 0; i < 100; ++i) {
- leafReader.addCoreClosedListener(listeners.get(random().nextInt(listeners.size())));
- }
- final int removed = random().nextInt(numListeners);
- Collections.shuffle(listeners, random());
- for (int i = 0; i < removed; ++i) {
- leafReader.removeCoreClosedListener(listeners.get(i));
+ leafReader.getCoreCacheHelper().addClosedListener(listeners.get(random().nextInt(listeners.size())));
}
assertEquals(numListeners, counter.get());
// make sure listeners are registered on the wrapped reader and that closing any of them has the same effect
@@ -139,11 +137,11 @@ public class TestIndexReaderClose extends LuceneTestCase {
} else {
leafReader.close();
}
- assertEquals(removed, counter.get());
+ assertEquals(0, counter.get());
w.w.getDirectory().close();
}
- private static final class CountCoreListener implements LeafReader.CoreClosedListener {
+ private static final class CountCoreListener implements IndexReader.ClosedListener {
private final AtomicInteger count;
private final Object coreCacheKey;
@@ -154,14 +152,14 @@ public class TestIndexReaderClose extends LuceneTestCase {
}
@Override
- public void onClose(Object coreCacheKey) {
+ public void onClose(IndexReader.CacheKey coreCacheKey) {
assertSame(this.coreCacheKey, coreCacheKey);
count.decrementAndGet();
}
}
- private static final class CountListener implements IndexReader.ReaderClosedListener {
+ private static final class CountListener implements IndexReader.ClosedListener {
private final AtomicInteger count;
public CountListener(AtomicInteger count) {
@@ -169,15 +167,15 @@ public class TestIndexReaderClose extends LuceneTestCase {
}
@Override
- public void onClose(IndexReader reader) {
+ public void onClose(IndexReader.CacheKey cacheKey) {
count.decrementAndGet();
}
}
- private static final class FaultyListener implements IndexReader.ReaderClosedListener {
+ private static final class FaultyListener implements IndexReader.ClosedListener {
@Override
- public void onClose(IndexReader reader) {
+ public void onClose(IndexReader.CacheKey cacheKey) {
throw new IllegalStateException("GRRRRRRRRRRRR!");
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
index ac352c1..a265c9c 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
@@ -265,5 +265,15 @@ public class TestMultiTermsEnum extends LuceneTestCase {
delegate.close();
}
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java
index 3efdc8b..d452306 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java
@@ -126,7 +126,7 @@ public class TestParallelCompositeReader extends LuceneTestCase {
dir2.close();
}
- private void testReaderClosedListener(boolean closeSubReaders, int wrapMultiReaderType) throws IOException {
+ private void testReaderClosedListener1(boolean closeSubReaders, int wrapMultiReaderType) throws IOException {
final Directory dir1 = getDir1(random());
final CompositeReader ir2, ir1 = DirectoryReader.open(dir1);
switch (wrapMultiReaderType) {
@@ -147,18 +147,19 @@ public class TestParallelCompositeReader extends LuceneTestCase {
new CompositeReader[] {ir2},
new CompositeReader[] {ir2});
- final int[] listenerClosedCount = new int[1];
-
assertEquals(3, pr.leaves().size());
+ assertEquals(ir1.getReaderCacheHelper(), pr.getReaderCacheHelper());
+ int i = 0;
for(LeafReaderContext cxt : pr.leaves()) {
- cxt.reader().addReaderClosedListener(reader -> listenerClosedCount[0]++);
+ LeafReader originalLeaf = ir1.leaves().get(i++).reader();
+ assertEquals(originalLeaf.getCoreCacheHelper(), cxt.reader().getCoreCacheHelper());
+ assertEquals(originalLeaf.getReaderCacheHelper(), cxt.reader().getReaderCacheHelper());
}
pr.close();
if (!closeSubReaders) {
ir1.close();
}
- assertEquals(3, listenerClosedCount[0]);
// We have to close the extra MultiReader, because it will not close its own subreaders:
if (wrapMultiReaderType == 2) {
@@ -168,23 +169,11 @@ public class TestParallelCompositeReader extends LuceneTestCase {
}
public void testReaderClosedListener1() throws Exception {
- testReaderClosedListener(false, 0);
- }
-
- public void testReaderClosedListener2() throws Exception {
- testReaderClosedListener(true, 0);
- }
-
- public void testReaderClosedListener3() throws Exception {
- testReaderClosedListener(false, 1);
- }
-
- public void testReaderClosedListener4() throws Exception {
- testReaderClosedListener(true, 1);
- }
-
- public void testReaderClosedListener5() throws Exception {
- testReaderClosedListener(false, 2);
+ testReaderClosedListener1(false, 0);
+ testReaderClosedListener1(true, 0);
+ testReaderClosedListener1(false, 1);
+ testReaderClosedListener1(true, 1);
+ testReaderClosedListener1(false, 2);
}
public void testCloseInnerReader() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java b/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
index 3878d59..9b8a285 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
@@ -237,7 +237,17 @@ public class TermInSetQueryTest extends LuceneTestCase {
}
};
}
-
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
+
}
@Override
@@ -245,6 +255,11 @@ public class TermInSetQueryTest extends LuceneTestCase {
return new TermsCountingDirectoryReaderWrapper(in, counter);
}
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
+
}
public void testPullOneTermsEnum() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
index 3acc3ea..91c1887 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
@@ -42,8 +42,11 @@ import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.RandomIndexWriter;
@@ -607,12 +610,12 @@ public class TestLRUQueryCache extends LuceneTestCase {
final int segmentCount2 = reader2.leaves().size();
final IndexSearcher searcher2 = new IndexSearcher(reader2);
- final Map<Object, Integer> indexId = new HashMap<>();
+ final Map<IndexReader.CacheKey, Integer> indexId = new HashMap<>();
for (LeafReaderContext ctx : reader1.leaves()) {
- indexId.put(ctx.reader().getCoreCacheKey(), 1);
+ indexId.put(ctx.reader().getCoreCacheHelper().getKey(), 1);
}
for (LeafReaderContext ctx : reader2.leaves()) {
- indexId.put(ctx.reader().getCoreCacheKey(), 2);
+ indexId.put(ctx.reader().getCoreCacheHelper().getKey(), 2);
}
final AtomicLong hitCount1 = new AtomicLong();
@@ -1218,4 +1221,56 @@ public class TestLRUQueryCache extends LuceneTestCase {
w.close();
dir.close();
}
+
+ // a reader whose sole purpose is to not be cacheable
+ private static class DummyDirectoryReader extends FilterDirectoryReader {
+
+ public DummyDirectoryReader(DirectoryReader in) throws IOException {
+ super(in, new SubReaderWrapper() {
+ @Override
+ public LeafReader wrap(LeafReader reader) {
+ return new FilterLeafReader(reader) {
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
+ };
+ }
+ });
+ }
+
+ @Override
+ protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
+ return new DummyDirectoryReader(in);
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
+ }
+
+ public void testReaderNotSuitedForCaching() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+ w.addDocument(new Document());
+ DirectoryReader reader = new DummyDirectoryReader(w.getReader());
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE);
+
+ // don't cache if the reader does not expose a cache helper
+ assertNull(reader.leaves().get(0).reader().getCoreCacheHelper());
+ LRUQueryCache cache = new LRUQueryCache(2, 10000, context -> true);
+ searcher.setQueryCache(cache);
+ assertEquals(0, searcher.count(new DummyQuery()));
+ assertEquals(0, cache.getCacheCount());
+ reader.close();
+ w.close();
+ dir.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java b/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
index 2fac35f..cc9a919 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
@@ -443,6 +443,16 @@ public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
public MyFilterLeafReader(LeafReader in) {
super(in);
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
}
private static class MyFilterDirectoryReader extends FilterDirectoryReader {
@@ -462,6 +472,11 @@ public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
return new MyFilterDirectoryReader(in);
}
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
}
// LUCENE-6087
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
index a994118..41d82d5 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
@@ -108,6 +108,11 @@ public class TestTermQuery extends LuceneTestCase {
protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
return new NoSeekDirectoryReader(in);
}
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
}
@@ -149,6 +154,16 @@ public class TestTermQuery extends LuceneTestCase {
};
}
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
+
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java
index 1ce1bc6..d00e520 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java
@@ -179,6 +179,16 @@ public class TestTermScorer extends LuceneTestCase {
// unreachable
return null;
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
};
// We don't use newSearcher because it sometimes runs checkIndex which loads norms
IndexSearcher indexSearcher = new IndexSearcher(forbiddenNorms);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
index b959d25..6bcfa46 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
@@ -116,7 +116,8 @@ public class DefaultSortedSetDocValuesReaderState extends SortedSetDocValuesRead
SortedSetDocValues dv = MultiDocValues.getSortedSetValues(origReader, field);
if (dv instanceof MultiDocValues.MultiSortedSetDocValues) {
map = ((MultiDocValues.MultiSortedSetDocValues)dv).mapping;
- if (map.owner == origReader.getCoreCacheKey()) {
+ IndexReader.CacheHelper cacheHelper = origReader.getReaderCacheHelper();
+ if (cacheHelper != null && map.owner == cacheHelper.getKey()) {
cachedOrdMaps.put(field, map);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java
index 0fbf4fb..a52b2af 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java
@@ -23,6 +23,7 @@ import java.util.WeakHashMap;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
@@ -67,7 +68,11 @@ public class CachedOrdinalsReader extends OrdinalsReader implements Accountable
}
private synchronized CachedOrds getCachedOrds(LeafReaderContext context) throws IOException {
- Object cacheKey = context.reader().getCoreCacheKey();
+ IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper();
+ if (cacheHelper == null) {
+ throw new IllegalStateException("Cannot cache ordinals on leaf: " + context.reader());
+ }
+ Object cacheKey = cacheHelper.getKey();
CachedOrds ords = ordsCache.get(cacheKey);
if (ords == null) {
ords = new CachedOrds(source.getReader(context), context.reader().maxDoc());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
index cb798af..341411d 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
@@ -157,5 +157,15 @@ public class OrdinalMappingLeafReader extends FilterLeafReader {
return in.getBinaryDocValues(field);
}
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}