You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/03/01 09:27:49 UTC
[41/50] [abbrv] lucene-solr:jira/solr-9858: LUCENE-7410: Make cache
keys and close listeners less trappy.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
index 608e3d4..2e3cdab 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
@@ -85,16 +85,6 @@ public class TermVectorLeafReader extends LeafReader {
}
@Override
- public void addCoreClosedListener(CoreClosedListener listener) {
- addCoreClosedListenerAsReaderClosedListener(this, listener);
- }
-
- @Override
- public void removeCoreClosedListener(CoreClosedListener listener) {
- removeCoreClosedListenerAsReaderClosedListener(this, listener);
- }
-
- @Override
protected void doClose() throws IOException {
}
@@ -178,4 +168,14 @@ public class TermVectorLeafReader extends LeafReader {
public Sort getIndexSort() {
return null;
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
index 0e0093b..a898854 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
@@ -474,6 +474,16 @@ public class WeightedSpanTermExtractor {
public NumericDocValues getNormValues(String field) throws IOException {
return super.getNormValues(FIELD_NAME);
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java
index d7e8671..ebc5d37 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java
@@ -584,6 +584,16 @@ public class PhraseHelper {
public NumericDocValues getNormValues(String field) throws IOException {
return super.getNormValues(fieldName);
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java
index 954024c..8555cbd 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java
@@ -127,4 +127,14 @@ final class TermVectorFilteredLeafReader extends FilterLeafReader {
}
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
index bbcfd5b..f1e2c44 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
@@ -1047,6 +1047,11 @@ public class UnifiedHighlighter {
protected void doClose() throws IOException {
reader.close();
}
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
};
}
@@ -1066,6 +1071,16 @@ public class UnifiedHighlighter {
return tvFields;
}
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
+
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermVec.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermVec.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermVec.java
index 57d398b..c462aee 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermVec.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermVec.java
@@ -132,6 +132,16 @@ public class TestUnifiedHighlighterTermVec extends LuceneTestCase {
return super.getTermVectors(docID);
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
};
}
};
@@ -144,6 +154,11 @@ public class TestUnifiedHighlighterTermVec extends LuceneTestCase {
protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
return new AssertOnceTermVecDirectoryReader(in);
}
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
private static boolean calledBy(Class<?> clazz) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/join/src/java/org/apache/lucene/search/join/QueryBitSetProducer.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/QueryBitSetProducer.java b/lucene/join/src/java/org/apache/lucene/search/join/QueryBitSetProducer.java
index 98d85cd..ac15664 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/QueryBitSetProducer.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/QueryBitSetProducer.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.Map;
import java.util.WeakHashMap;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
@@ -39,7 +40,7 @@ import org.apache.lucene.util.BitSet;
*/
public class QueryBitSetProducer implements BitSetProducer {
private final Query query;
- private final Map<Object,DocIdSet> cache = Collections.synchronizedMap(new WeakHashMap<>());
+ final Map<IndexReader.CacheKey,DocIdSet> cache = Collections.synchronizedMap(new WeakHashMap<>());
/** Wraps another query's result and caches it into bitsets.
* @param query Query to cache results of
@@ -59,9 +60,12 @@ public class QueryBitSetProducer implements BitSetProducer {
@Override
public BitSet getBitSet(LeafReaderContext context) throws IOException {
final LeafReader reader = context.reader();
- final Object key = reader.getCoreCacheKey();
+ final IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper();
- DocIdSet docIdSet = cache.get(key);
+ DocIdSet docIdSet = null;
+ if (cacheHelper != null) {
+ docIdSet = cache.get(cacheHelper.getKey());
+ }
if (docIdSet == null) {
final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(context);
final IndexSearcher searcher = new IndexSearcher(topLevelContext);
@@ -74,7 +78,9 @@ public class QueryBitSetProducer implements BitSetProducer {
} else {
docIdSet = new BitDocIdSet(BitSet.of(s.iterator(), context.reader().maxDoc()));
}
- cache.put(key, docIdSet);
+ if (cacheHelper != null) {
+ cache.put(cacheHelper.getKey(), docIdSet);
+ }
}
return docIdSet == DocIdSet.EMPTY ? null : ((BitDocIdSet) docIdSet).bits();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
index 39979ac..72b6bf5 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
@@ -267,7 +267,7 @@ public class TestJoinUtil extends LuceneTestCase {
values[i] = DocValues.getSorted(leafReader, joinField);
}
MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(
- r.getCoreCacheKey(), values, PackedInts.DEFAULT
+ null, values, PackedInts.DEFAULT
);
Query toQuery = new TermQuery(new Term(typeField, "price"));
@@ -372,7 +372,7 @@ public class TestJoinUtil extends LuceneTestCase {
values[i] = DocValues.getSorted(leafReader, joinField);
}
MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(
- r.getCoreCacheKey(), values, PackedInts.DEFAULT
+ null, values, PackedInts.DEFAULT
);
Query toQuery = new TermQuery(new Term("price", "5.0"));
@@ -500,7 +500,7 @@ public class TestJoinUtil extends LuceneTestCase {
values[leadContext.ord] = DocValues.getSorted(leadContext.reader(), "join_field");
}
MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(
- searcher.getIndexReader().getCoreCacheKey(), values, PackedInts.DEFAULT
+ null, values, PackedInts.DEFAULT
);
BooleanQuery.Builder fromQuery = new BooleanQuery.Builder();
fromQuery.add(priceQuery, BooleanClause.Occur.MUST);
@@ -621,7 +621,7 @@ public class TestJoinUtil extends LuceneTestCase {
values[leadContext.ord] = DocValues.getSorted(leadContext.reader(), "join_field");
}
MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(
- searcher.getIndexReader().getCoreCacheKey(), values, PackedInts.DEFAULT
+ null, values, PackedInts.DEFAULT
);
Query fromQuery = new TermQuery(new Term("type", "from"));
Query toQuery = new TermQuery(new Term("type", "to"));
@@ -1336,7 +1336,7 @@ public class TestJoinUtil extends LuceneTestCase {
values[leadContext.ord] = DocValues.getSorted(leadContext.reader(), "join_field");
}
context.ordinalMap = MultiDocValues.OrdinalMap.build(
- topLevelReader.getCoreCacheKey(), values, PackedInts.DEFAULT
+ null, values, PackedInts.DEFAULT
);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/join/src/test/org/apache/lucene/search/join/TestQueryBitSetProducer.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestQueryBitSetProducer.java b/lucene/join/src/test/org/apache/lucene/search/join/TestQueryBitSetProducer.java
new file mode 100644
index 0000000..c90b0a8
--- /dev/null
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestQueryBitSetProducer.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.join;
+
+import java.io.IOException;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BitSet;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestQueryBitSetProducer extends LuceneTestCase {
+
+ public void testSimple() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+ w.addDocument(new Document());
+ DirectoryReader reader = w.getReader();
+
+ QueryBitSetProducer producer = new QueryBitSetProducer(new MatchNoDocsQuery());
+ assertNull(producer.getBitSet(reader.leaves().get(0)));
+ assertEquals(1, producer.cache.size());
+
+ producer = new QueryBitSetProducer(new MatchAllDocsQuery());
+ BitSet bitSet = producer.getBitSet(reader.leaves().get(0));
+ assertEquals(1, bitSet.length());
+ assertEquals(true, bitSet.get(0));
+ assertEquals(1, producer.cache.size());
+
+ IOUtils.close(reader, w, dir);
+ }
+
+ public void testReaderNotSuitedForCaching() throws IOException{
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+ w.addDocument(new Document());
+ DirectoryReader reader = new DummyDirectoryReader(w.getReader());
+
+ QueryBitSetProducer producer = new QueryBitSetProducer(new MatchNoDocsQuery());
+ assertNull(producer.getBitSet(reader.leaves().get(0)));
+ assertEquals(0, producer.cache.size());
+
+ producer = new QueryBitSetProducer(new MatchAllDocsQuery());
+ BitSet bitSet = producer.getBitSet(reader.leaves().get(0));
+ assertEquals(1, bitSet.length());
+ assertEquals(true, bitSet.get(0));
+ assertEquals(0, producer.cache.size());
+
+ IOUtils.close(reader, w, dir);
+ }
+
+ // a reader whose sole purpose is to not be cacheable
+ private static class DummyDirectoryReader extends FilterDirectoryReader {
+
+ public DummyDirectoryReader(DirectoryReader in) throws IOException {
+ super(in, new SubReaderWrapper() {
+ @Override
+ public LeafReader wrap(LeafReader reader) {
+ return new FilterLeafReader(reader) {
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }};
+ }
+ });
+ }
+
+ @Override
+ protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
+ return new DummyDirectoryReader(in);
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index 7587406..dc4a666 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -1182,16 +1182,6 @@ public class MemoryIndex {
}
}
- @Override
- public void addCoreClosedListener(CoreClosedListener listener) {
- addCoreClosedListenerAsReaderClosedListener(this, listener);
- }
-
- @Override
- public void removeCoreClosedListener(CoreClosedListener listener) {
- removeCoreClosedListenerAsReaderClosedListener(this, listener);
- }
-
private Info getInfoForExpectedDocValuesType(String fieldName, DocValuesType expectedType) {
if (expectedType == DocValuesType.NONE) {
return null;
@@ -1684,6 +1674,16 @@ public class MemoryIndex {
public Sort getIndexSort() {
return null;
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java b/lucene/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
index b4e2131..6d7e4ea 100644
--- a/lucene/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
+++ b/lucene/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
@@ -206,6 +206,11 @@ public class MultiPassIndexSplitter {
@Override
protected void doClose() {}
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
+
// no need to override numDocs/hasDeletions,
// as we pass the subreaders directly to IW.addIndexes().
}
@@ -247,5 +252,15 @@ public class MultiPassIndexSplitter {
public Bits getLiveDocs() {
return liveDocs;
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java b/lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java
index 49ec6a2..c95cda6 100644
--- a/lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java
+++ b/lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java
@@ -167,5 +167,15 @@ public class PKIndexSplitter {
public Bits getLiveDocs() {
return liveDocs;
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/SegmentInfosSearcherManager.java
----------------------------------------------------------------------
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/SegmentInfosSearcherManager.java b/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/SegmentInfosSearcherManager.java
index 4cb49c4..a04464a 100644
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/SegmentInfosSearcherManager.java
+++ b/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/SegmentInfosSearcherManager.java
@@ -111,10 +111,14 @@ class SegmentInfosSearcherManager extends ReferenceManager<IndexSearcher> {
}
private void addReaderClosedListener(IndexReader r) {
+ IndexReader.CacheHelper cacheHelper = r.getReaderCacheHelper();
+ if (cacheHelper == null) {
+ throw new IllegalStateException("StandardDirectoryReader must support caching");
+ }
openReaderCount.incrementAndGet();
- r.addReaderClosedListener(new IndexReader.ReaderClosedListener() {
+ cacheHelper.addClosedListener(new IndexReader.ClosedListener() {
@Override
- public void onClose(IndexReader reader) {
+ public void onClose(IndexReader.CacheKey cacheKey) {
onReaderClosed();
}
});
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/test-framework/src/java/org/apache/lucene/index/AllDeletedFilterReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AllDeletedFilterReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AllDeletedFilterReader.java
index ff789a0..f11d2e3 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/AllDeletedFilterReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/AllDeletedFilterReader.java
@@ -39,4 +39,14 @@ public class AllDeletedFilterReader extends FilterLeafReader {
public int numDocs() {
return 0;
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/test-framework/src/java/org/apache/lucene/index/AssertingDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingDirectoryReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingDirectoryReader.java
index 712f36d..2326726 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingDirectoryReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingDirectoryReader.java
@@ -41,13 +41,8 @@ public class AssertingDirectoryReader extends FilterDirectoryReader {
}
@Override
- public Object getCoreCacheKey() {
- return in.getCoreCacheKey();
- }
-
- @Override
- public Object getCombinedCoreAndDeletesKey() {
- return in.getCombinedCoreAndDeletesKey();
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
index e837359..b03fa3d 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
@@ -51,11 +51,23 @@ public class AssertingLeafReader extends FilterLeafReader {
assert in.numDeletedDocs() + in.numDocs() == in.maxDoc();
assert !in.hasDeletions() || in.numDeletedDocs() > 0 && in.numDocs() < in.maxDoc();
- addCoreClosedListener(ownerCoreCacheKey -> {
- final Object expectedKey = getCoreCacheKey();
- assert expectedKey == ownerCoreCacheKey
- : "Core closed listener called on a different key " + expectedKey + " <> " + ownerCoreCacheKey;
- });
+ CacheHelper coreCacheHelper = in.getCoreCacheHelper();
+ if (coreCacheHelper != null) {
+ coreCacheHelper.addClosedListener(cacheKey -> {
+ final Object expectedKey = coreCacheHelper.getKey();
+ assert expectedKey == cacheKey
+ : "Core closed listener called on a different key " + expectedKey + " <> " + cacheKey;
+ });
+ }
+
+ CacheHelper readerCacheHelper = in.getReaderCacheHelper();
+ if (readerCacheHelper != null) {
+ readerCacheHelper.addClosedListener(cacheKey -> {
+ final Object expectedKey = readerCacheHelper.getKey();
+ assert expectedKey == cacheKey
+ : "Core closed listener called on a different key " + expectedKey + " <> " + cacheKey;
+ });
+ }
}
@Override
@@ -1137,12 +1149,12 @@ public class AssertingLeafReader extends FilterLeafReader {
// we don't change behavior of the reader: just validate the API.
@Override
- public Object getCoreCacheKey() {
- return in.getCoreCacheKey();
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
}
@Override
- public Object getCombinedCoreAndDeletesKey() {
- return in.getCombinedCoreAndDeletesKey();
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
index 554d908..60e2cca 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
@@ -586,6 +586,16 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
super.document(maxDoc() - 1 - docID, visitor);
}
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
+
}
private static class DummyFilterDirectoryReader extends FilterDirectoryReader {
@@ -603,6 +613,11 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
return new DummyFilterDirectoryReader(in);
}
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java
index a75af54..751833f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java
@@ -174,5 +174,15 @@ public final class FieldFilterLeafReader extends FilterLeafReader {
}
}
-
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedDirectoryReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedDirectoryReader.java
index 76aecef..7fb1581 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedDirectoryReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedDirectoryReader.java
@@ -46,4 +46,9 @@ public class MismatchedDirectoryReader extends FilterDirectoryReader {
protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
return new AssertingDirectoryReader(in);
}
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
index 38c1d7f..7dd6ba8 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
@@ -45,6 +45,16 @@ public class MismatchedLeafReader extends FilterLeafReader {
in.document(docID, new MismatchedVisitor(visitor));
}
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
+
static FieldInfos shuffleInfos(FieldInfos infos, Random random) {
// first, shuffle the order
List<FieldInfo> shuffled = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java b/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java
index b1cc0ee..f9fa601 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java
@@ -156,7 +156,18 @@ public class MockRandomMergePolicy extends MergePolicy {
if (LuceneTestCase.VERBOSE) {
System.out.println("NOTE: MockRandomMergePolicy now swaps in a SlowCodecReaderWrapper for merging reader=" + reader);
}
- return SlowCodecReaderWrapper.wrap(new FilterLeafReader(new MergeReaderWrapper(reader)) {});
+ return SlowCodecReaderWrapper.wrap(new FilterLeafReader(new MergeReaderWrapper(reader)) {
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
+ });
} else if (thingToDo == 1) {
// renumber fields
// NOTE: currently this only "blocks" bulk merges just by
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
index ae4c890..c40f875 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
@@ -132,27 +132,6 @@ public class QueryUtils {
}
}
- /** This is a MultiReader that can be used for randomly wrapping other readers
- * without creating FieldCache insanity.
- * The trick is to use an opaque/fake cache key. */
- public static class FCInvisibleMultiReader extends MultiReader {
- private final Object cacheKey = new Object();
-
- public FCInvisibleMultiReader(IndexReader... readers) throws IOException {
- super(readers);
- }
-
- @Override
- public Object getCoreCacheKey() {
- return cacheKey;
- }
-
- @Override
- public Object getCombinedCoreAndDeletesKey() {
- return cacheKey;
- }
- }
-
/**
* Given an IndexSearcher, returns a new IndexSearcher whose IndexReader
* is a MultiReader containing the Reader of the original IndexSearcher,
@@ -172,17 +151,17 @@ public class QueryUtils {
IndexReader[] readers = new IndexReader[] {
edge < 0 ? r : new MultiReader(),
new MultiReader(),
- new FCInvisibleMultiReader(edge < 0 ? emptyReader(4) : new MultiReader(),
+ new MultiReader(edge < 0 ? emptyReader(4) : new MultiReader(),
new MultiReader(),
0 == edge ? r : new MultiReader()),
0 < edge ? new MultiReader() : emptyReader(7),
new MultiReader(),
- new FCInvisibleMultiReader(0 < edge ? new MultiReader() : emptyReader(5),
+ new MultiReader(0 < edge ? new MultiReader() : emptyReader(5),
new MultiReader(),
0 < edge ? r : new MultiReader())
};
- IndexSearcher out = LuceneTestCase.newSearcher(new FCInvisibleMultiReader(readers));
+ IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers));
out.setSimilarity(s.getSimilarity(true));
return out;
}
@@ -191,12 +170,6 @@ public class QueryUtils {
return new LeafReader() {
@Override
- public void addCoreClosedListener(CoreClosedListener listener) {}
-
- @Override
- public void removeCoreClosedListener(CoreClosedListener listener) {}
-
- @Override
public Fields fields() throws IOException {
return new Fields() {
@Override
@@ -290,6 +263,16 @@ public class QueryUtils {
public Sort getIndexSort() {
return null;
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 90f85b9..591db20 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -74,7 +74,6 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.*;
-import org.apache.lucene.index.IndexReader.ReaderClosedListener;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.mockfile.FilterPath;
@@ -86,7 +85,6 @@ import org.apache.lucene.search.LRUQueryCache;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryCache;
import org.apache.lucene.search.QueryCachingPolicy;
-import org.apache.lucene.search.QueryUtils.FCInvisibleMultiReader;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -1664,7 +1662,7 @@ public abstract class LuceneTestCase extends Assert {
Random random = random();
for (int i = 0, c = random.nextInt(6)+1; i < c; i++) {
- switch(random.nextInt(5)) {
+ switch(random.nextInt(4)) {
case 0:
// will create no FC insanity in atomic case, as ParallelLeafReader has own cache key:
if (VERBOSE) {
@@ -1675,15 +1673,6 @@ public abstract class LuceneTestCase extends Assert {
new ParallelCompositeReader((CompositeReader) r);
break;
case 1:
- // H�ckidy-Hick-Hack: a standard MultiReader will cause FC insanity, so we use
- // QueryUtils' reader with a fake cache key, so insanity checker cannot walk
- // along our reader:
- if (VERBOSE) {
- System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with FCInvisibleMultiReader");
- }
- r = new FCInvisibleMultiReader(r);
- break;
- case 2:
if (r instanceof LeafReader) {
final LeafReader ar = (LeafReader) r;
final List<String> allFields = new ArrayList<>();
@@ -1703,7 +1692,7 @@ public abstract class LuceneTestCase extends Assert {
);
}
break;
- case 3:
+ case 2:
// H�ckidy-Hick-Hack: a standard Reader will cause FC insanity, so we use
// QueryUtils' reader with a fake cache key, so insanity checker cannot walk
// along our reader:
@@ -1716,7 +1705,7 @@ public abstract class LuceneTestCase extends Assert {
r = new AssertingDirectoryReader((DirectoryReader)r);
}
break;
- case 4:
+ case 3:
if (VERBOSE) {
System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with MismatchedLeaf/DirectoryReader");
}
@@ -1731,10 +1720,6 @@ public abstract class LuceneTestCase extends Assert {
}
}
- if ((r instanceof CompositeReader) && !(r instanceof FCInvisibleMultiReader)) {
- // prevent cache insanity caused by e.g. ParallelCompositeReader, to fix we wrap one more time:
- r = new FCInvisibleMultiReader(r);
- }
if (VERBOSE) {
System.out.println("wrapReader wrapped: " +r);
}
@@ -1900,7 +1885,7 @@ public abstract class LuceneTestCase extends Assert {
} else {
int threads = 0;
final ThreadPoolExecutor ex;
- if (random.nextBoolean()) {
+ if (r.getReaderCacheHelper() == null || random.nextBoolean()) {
ex = null;
} else {
threads = TestUtil.nextInt(random, 1, 8);
@@ -1914,12 +1899,7 @@ public abstract class LuceneTestCase extends Assert {
if (VERBOSE) {
System.out.println("NOTE: newSearcher using ExecutorService with " + threads + " threads");
}
- r.addReaderClosedListener(new ReaderClosedListener() {
- @Override
- public void onClose(IndexReader reader) {
- TestUtil.shutdownExecutorService(ex);
- }
- });
+ r.getReaderCacheHelper().addClosedListener(cacheKey -> TestUtil.shutdownExecutorService(ex));
}
IndexSearcher ret;
if (wrapWithAssertions) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index b108b38..1c30e4c 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -63,6 +63,7 @@ import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexDeletionPolicy;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BooleanQuery;
@@ -219,7 +220,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
public Date getStartTimeStamp() { return startTime; }
- private final Map<Object, IndexFingerprint> perSegmentFingerprintCache = new MapMaker().weakKeys().makeMap();
+ private final Map<IndexReader.CacheKey, IndexFingerprint> perSegmentFingerprintCache = new MapMaker().weakKeys().makeMap();
public long getStartNanoTime() {
return startNanoTime;
@@ -1775,8 +1776,14 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
*/
public IndexFingerprint getIndexFingerprint(SolrIndexSearcher searcher, LeafReaderContext ctx, long maxVersion)
throws IOException {
+ IndexReader.CacheHelper cacheHelper = ctx.reader().getReaderCacheHelper();
+ if (cacheHelper == null) {
+ log.debug("Cannot cache IndexFingerprint as reader does not support caching. searcher:{} reader:{} readerHash:{} maxVersion:{}", searcher, ctx.reader(), ctx.reader().hashCode(), maxVersion);
+ return IndexFingerprint.getFingerprint(searcher, ctx, maxVersion);
+ }
+
IndexFingerprint f = null;
- f = perSegmentFingerprintCache.get(ctx.reader().getCombinedCoreAndDeletesKey());
+ f = perSegmentFingerprintCache.get(cacheHelper.getKey());
// fingerprint is either not cached or
// if we want fingerprint only up to a version less than maxVersionEncountered in the segment, or
// documents were deleted from segment for which fingerprint was cached
@@ -1787,7 +1794,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
// cache fingerprint for the segment only if all the versions in the segment are included in the fingerprint
if (f.getMaxVersionEncountered() == f.getMaxInHash()) {
log.info("Caching fingerprint for searcher:{} leafReaderContext:{} mavVersion:{}", searcher, ctx, maxVersion);
- perSegmentFingerprintCache.put(ctx.reader().getCombinedCoreAndDeletesKey(), f);
+ perSegmentFingerprintCache.put(cacheHelper.getKey(), f);
}
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
index c06aab0..8078bdc 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
@@ -761,6 +761,8 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
}
}
+ // this reader alters the content of the given reader so it should not
+ // delegate the caching stuff
private class ReaderWrapper extends FilterLeafReader {
private String field;
@@ -774,10 +776,6 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
return null;
}
- public Object getCoreCacheKey() {
- return in.getCoreCacheKey();
- }
-
public FieldInfos getFieldInfos() {
Iterator<FieldInfo> it = in.getFieldInfos().iterator();
List<FieldInfo> newInfos = new ArrayList<>();
@@ -805,6 +803,21 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
FieldInfos infos = new FieldInfos(newInfos.toArray(new FieldInfo[newInfos.size()]));
return infos;
}
+
+ // NOTE: delegating the caches is wrong here as we are altering the content
+ // of the reader, this should ONLY be used under an uninvertingreader which
+ // will restore doc values back using uninversion, otherwise all sorts of
+ // crazy things could happen.
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
index 33ea575..e4ada59 100644
--- a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
+++ b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
@@ -914,4 +914,14 @@ class TermVectorReusingLeafReader extends FilterLeafReader {
return tvFields;
}
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
index 12f5bd1..c445cdf 100644
--- a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
+++ b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
@@ -47,7 +47,6 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
private final CompositeReader in;
private final Fields fields;
- private final boolean merging;
/** This method is sugar for getting an {@link LeafReader} from
* an {@link IndexReader} of any kind. If the reader is already atomic,
@@ -55,19 +54,18 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
*/
public static LeafReader wrap(IndexReader reader) throws IOException {
if (reader instanceof CompositeReader) {
- return new SlowCompositeReaderWrapper((CompositeReader) reader, false);
+ return new SlowCompositeReaderWrapper((CompositeReader) reader);
} else {
assert reader instanceof LeafReader;
return (LeafReader) reader;
}
}
- SlowCompositeReaderWrapper(CompositeReader reader, boolean merging) throws IOException {
+ SlowCompositeReaderWrapper(CompositeReader reader) throws IOException {
super();
in = reader;
fields = MultiFields.getFields(in);
in.registerParentReader(this);
- this.merging = merging;
}
@Override
@@ -76,13 +74,16 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
}
@Override
- public void addCoreClosedListener(CoreClosedListener listener) {
- addCoreClosedListenerAsReaderClosedListener(in, listener);
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
}
@Override
- public void removeCoreClosedListener(CoreClosedListener listener) {
- removeCoreClosedListenerAsReaderClosedListener(in, listener);
+ public CacheHelper getCoreCacheHelper() {
+ // TODO: this is trappy as the expectation is that core keys live for a long
+ // time, but here we need to bound it to the lifetime of the wrapped
+ // composite reader? Unfortunately some features seem to rely on this...
+ return in.getReaderCacheHelper();
}
@Override
@@ -120,7 +121,8 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
SortedDocValues dv = MultiDocValues.getSortedValues(in, field);
if (dv instanceof MultiSortedDocValues) {
map = ((MultiSortedDocValues)dv).mapping;
- if (map.owner == getCoreCacheKey() && merging == false) {
+ IndexReader.CacheHelper cacheHelper = getReaderCacheHelper();
+ if (cacheHelper != null && map.owner == cacheHelper.getKey()) {
cachedOrdMaps.put(field, map);
}
}
@@ -161,7 +163,8 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
SortedSetDocValues dv = MultiDocValues.getSortedSetValues(in, field);
if (dv instanceof MultiDocValues.MultiSortedSetDocValues) {
map = ((MultiDocValues.MultiSortedSetDocValues)dv).mapping;
- if (map.owner == getCoreCacheKey() && merging == false) {
+ IndexReader.CacheHelper cacheHelper = getReaderCacheHelper();
+ if (cacheHelper != null && map.owner == cacheHelper.getKey()) {
cachedOrdMaps.put(field, map);
}
}
@@ -195,7 +198,7 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
// TODO: this could really be a weak map somewhere else on the coreCacheKey,
// but do we really need to optimize slow-wrapper any more?
- private final Map<String,OrdinalMap> cachedOrdMaps = new HashMap<>();
+ final Map<String,OrdinalMap> cachedOrdMaps = new HashMap<>();
@Override
public NumericDocValues getNormValues(String field) throws IOException {
@@ -246,16 +249,6 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
}
@Override
- public Object getCoreCacheKey() {
- return in.getCoreCacheKey();
- }
-
- @Override
- public Object getCombinedCoreAndDeletesKey() {
- return in.getCombinedCoreAndDeletesKey();
- }
-
- @Override
protected void doClose() throws IOException {
// TODO: as this is a wrapper, should we really close the delegate?
in.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
index f072867..acd2255 100644
--- a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
+++ b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -162,7 +163,11 @@ public class RptWithGeometrySpatialField extends AbstractSpatialFieldType<Compos
}
docId = doc;
//lookup in cache
- PerSegCacheKey key = new PerSegCacheKey(readerContext.reader().getCoreCacheKey(), doc);
+ IndexReader.CacheHelper cacheHelper = readerContext.reader().getCoreCacheHelper();
+ if (cacheHelper == null) {
+ throw new IllegalStateException("Leaf " + readerContext.reader() + " is not suited for caching");
+ }
+ PerSegCacheKey key = new PerSegCacheKey(cacheHelper.getKey(), doc);
shape = cache.get(key);
if (shape == null) {
shape = (Shape) targetFuncValues.objectVal(doc);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index 65d470e..71478aa 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -388,12 +388,23 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.field = field;
}
- public SortedDocValues getSortedDocValues(String field) {
- return null;
+ // NOTE: delegating the caches is wrong here as we are altering the content
+ // of the reader, this should ONLY be used under an uninvertingreader which
+ // will restore doc values back using uninversion, otherwise all sorts of
+ // crazy things could happen.
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
}
- public Object getCoreCacheKey() {
- return in.getCoreCacheKey();
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
+
+ public SortedDocValues getSortedDocValues(String field) {
+ return null;
}
public FieldInfos getFieldInfos() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/search/Insanity.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/Insanity.java b/solr/core/src/java/org/apache/solr/search/Insanity.java
index 7f16797..aa36652 100644
--- a/solr/core/src/java/org/apache/solr/search/Insanity.java
+++ b/solr/core/src/java/org/apache/solr/search/Insanity.java
@@ -118,13 +118,14 @@ public class Insanity {
// important to override these, so fieldcaches are shared on what we wrap
@Override
- public Object getCoreCacheKey() {
- return in.getCoreCacheKey();
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
}
@Override
- public Object getCombinedCoreAndDeletesKey() {
- return in.getCombinedCoreAndDeletesKey();
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
}
+
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java b/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java
index 544800e..89e6f0b 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java
@@ -17,7 +17,6 @@
package org.apache.solr.uninverting;
import java.io.IOException;
-import java.io.PrintStream;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.BinaryDocValues;
@@ -41,7 +40,6 @@ import org.apache.lucene.util.RamUsageEstimator;
* <p>Created: May 19, 2004 11:13:14 AM
*
* @since lucene 1.4
- * @see FieldCacheSanityChecker
*
* @lucene.internal
*/
@@ -357,7 +355,7 @@ public interface FieldCache {
private final Object custom;
private final Accountable value;
- public CacheEntry(Object readerKey, String fieldName,
+ public CacheEntry(IndexReader.CacheKey readerKey, String fieldName,
Class<?> cacheType,
Object custom,
Accountable value) {
@@ -437,21 +435,13 @@ public interface FieldCache {
/**
* Expert: drops all cache entries associated with this
- * reader {@link IndexReader#getCoreCacheKey}. NOTE: this cache key must
+ * reader {@link org.apache.lucene.index.IndexReader.CacheHelper#getKey()}.
+ * NOTE: this cache key must
* precisely match the reader that the cache entry is
* keyed on. If you pass a top-level reader, it usually
* will have no effect as Lucene now caches at the segment
* reader level.
*/
- public void purgeByCacheKey(Object coreCacheKey);
+ public void purgeByCacheKey(IndexReader.CacheKey coreCacheKey);
- /**
- * If non-null, FieldCacheImpl will warn whenever
- * entries are created that are not sane according to
- * {@link FieldCacheSanityChecker}.
- */
- public void setInfoStream(PrintStream stream);
-
- /** counterpart of {@link #setInfoStream(PrintStream)} */
- public PrintStream getInfoStream();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java b/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
index 90be400..e6e8fda 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
@@ -17,7 +17,6 @@
package org.apache.solr.uninverting;
import java.io.IOException;
-import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -31,13 +30,13 @@ import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
@@ -82,7 +81,7 @@ public class FieldCacheImpl implements FieldCache {
}
@Override
- public synchronized void purgeByCacheKey(Object coreCacheKey) {
+ public synchronized void purgeByCacheKey(IndexReader.CacheKey coreCacheKey) {
for(Cache c : caches.values()) {
c.purgeByCacheKey(coreCacheKey);
}
@@ -95,8 +94,8 @@ public class FieldCacheImpl implements FieldCache {
final Cache cache = cacheEntry.getValue();
final Class<?> cacheType = cacheEntry.getKey();
synchronized(cache.readerCache) {
- for (final Map.Entry<Object,Map<CacheKey, Accountable>> readerCacheEntry : cache.readerCache.entrySet()) {
- final Object readerKey = readerCacheEntry.getKey();
+ for (final Map.Entry<IndexReader.CacheKey,Map<CacheKey, Accountable>> readerCacheEntry : cache.readerCache.entrySet()) {
+ final IndexReader.CacheKey readerKey = readerCacheEntry.getKey();
if (readerKey == null) continue;
final Map<CacheKey, Accountable> innerCache = readerCacheEntry.getValue();
for (final Map.Entry<CacheKey, Accountable> mapEntry : innerCache.entrySet()) {
@@ -112,10 +111,14 @@ public class FieldCacheImpl implements FieldCache {
}
// per-segment fieldcaches don't purge until the shared core closes.
- final SegmentReader.CoreClosedListener purgeCore = FieldCacheImpl.this::purgeByCacheKey;
+ final IndexReader.ClosedListener purgeCore = FieldCacheImpl.this::purgeByCacheKey;
private void initReader(LeafReader reader) {
- reader.addCoreClosedListener(purgeCore);
+ IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper();
+ if (cacheHelper == null) {
+ throw new IllegalStateException("Cannot cache on " + reader);
+ }
+ cacheHelper.addClosedListener(purgeCore);
}
/** Expert: Internal cache. */
@@ -127,13 +130,13 @@ public class FieldCacheImpl implements FieldCache {
final FieldCacheImpl wrapper;
- final Map<Object,Map<CacheKey,Accountable>> readerCache = new WeakHashMap<>();
+ final Map<IndexReader.CacheKey,Map<CacheKey,Accountable>> readerCache = new WeakHashMap<>();
protected abstract Accountable createValue(LeafReader reader, CacheKey key)
throws IOException;
/** Remove this reader from the cache, if present. */
- public void purgeByCacheKey(Object coreCacheKey) {
+ public void purgeByCacheKey(IndexReader.CacheKey coreCacheKey) {
synchronized(readerCache) {
readerCache.remove(coreCacheKey);
}
@@ -142,7 +145,11 @@ public class FieldCacheImpl implements FieldCache {
/** Sets the key to the value for the provided reader;
* if the key is already set then this doesn't change it. */
public void put(LeafReader reader, CacheKey key, Accountable value) {
- final Object readerKey = reader.getCoreCacheKey();
+ IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper();
+ if (cacheHelper == null) {
+ throw new IllegalStateException("Cannot cache on " + reader);
+ }
+ final IndexReader.CacheKey readerKey = cacheHelper.getKey();
synchronized (readerCache) {
Map<CacheKey,Accountable> innerCache = readerCache.get(readerKey);
if (innerCache == null) {
@@ -163,7 +170,12 @@ public class FieldCacheImpl implements FieldCache {
public Object get(LeafReader reader, CacheKey key) throws IOException {
Map<CacheKey,Accountable> innerCache;
Accountable value;
- final Object readerKey = reader.getCoreCacheKey();
+ IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper();
+ if (cacheHelper == null) {
+ reader.getCoreCacheHelper();
+ throw new IllegalStateException("Cannot cache on " + reader);
+ }
+ final IndexReader.CacheKey readerKey = cacheHelper.getKey();
synchronized (readerCache) {
innerCache = readerCache.get(readerKey);
if (innerCache == null) {
@@ -188,39 +200,12 @@ public class FieldCacheImpl implements FieldCache {
synchronized (readerCache) {
innerCache.put(key, progress.value);
}
-
- // Only check if key.custom (the parser) is
- // non-null; else, we check twice for a single
- // call to FieldCache.getXXX
- if (key.custom != null && wrapper != null) {
- final PrintStream infoStream = wrapper.getInfoStream();
- if (infoStream != null) {
- printNewInsanity(infoStream, progress.value);
- }
- }
}
return progress.value;
}
}
return value;
}
-
- private void printNewInsanity(PrintStream infoStream, Object value) {
- final FieldCacheSanityChecker.Insanity[] insanities = FieldCacheSanityChecker.checkSanity(wrapper);
- for(int i=0;i<insanities.length;i++) {
- final FieldCacheSanityChecker.Insanity insanity = insanities[i];
- final CacheEntry[] entries = insanity.getCacheEntries();
- for(int j=0;j<entries.length;j++) {
- if (entries[j].getValue() == value) {
- // OK this insanity involves our entry
- infoStream.println("WARNING: new FieldCache insanity created\nDetails: " + insanity.toString());
- infoStream.println("\nStack:\n");
- new Throwable().printStackTrace(infoStream);
- break;
- }
- }
- }
- }
}
/** Expert: Every composite-key in the internal cache is of this type. */
@@ -1265,14 +1250,5 @@ public class FieldCacheImpl implements FieldCache {
}
}
- private volatile PrintStream infoStream;
-
- public void setInfoStream(PrintStream stream) {
- infoStream = stream;
- }
-
- public PrintStream getInfoStream() {
- return infoStream;
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/uninverting/FieldCacheSanityChecker.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/FieldCacheSanityChecker.java b/solr/core/src/java/org/apache/solr/uninverting/FieldCacheSanityChecker.java
deleted file mode 100644
index 3d874ce..0000000
--- a/solr/core/src/java/org/apache/solr/uninverting/FieldCacheSanityChecker.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.uninverting;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReaderContext;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.MapOfSets;
-import org.apache.solr.uninverting.FieldCache.CacheEntry;
-
-/**
- * Provides methods for sanity checking that entries in the FieldCache
- * are not wasteful or inconsistent.
- * </p>
- * <p>
- * Lucene 2.9 Introduced numerous enhancements into how the FieldCache
- * is used by the low levels of Lucene searching (for Sorting and
- * ValueSourceQueries) to improve both the speed for Sorting, as well
- * as reopening of IndexReaders. But these changes have shifted the
- * usage of FieldCache from "top level" IndexReaders (frequently a
- * MultiReader or DirectoryReader) down to the leaf level SegmentReaders.
- * As a result, existing applications that directly access the FieldCache
- * may find RAM usage increase significantly when upgrading to 2.9 or
- * Later. This class provides an API for these applications (or their
- * Unit tests) to check at run time if the FieldCache contains "insane"
- * usages of the FieldCache.
- * </p>
- * @lucene.experimental
- * @see FieldCache
- * @see FieldCacheSanityChecker.Insanity
- * @see FieldCacheSanityChecker.InsanityType
- */
-final class FieldCacheSanityChecker {
-
- public FieldCacheSanityChecker() {
- /* NOOP */
- }
-
- /**
- * Quick and dirty convenience method
- * @see #check
- */
- public static Insanity[] checkSanity(FieldCache cache) {
- return checkSanity(cache.getCacheEntries());
- }
-
- /**
- * Quick and dirty convenience method that instantiates an instance with
- * "good defaults" and uses it to test the CacheEntrys
- * @see #check
- */
- public static Insanity[] checkSanity(CacheEntry... cacheEntries) {
- FieldCacheSanityChecker sanityChecker = new FieldCacheSanityChecker();
- return sanityChecker.check(cacheEntries);
- }
-
-
- /**
- * Tests a CacheEntry[] for indication of "insane" cache usage.
- * <p>
- * <B>NOTE:</b>FieldCache CreationPlaceholder objects are ignored.
- * (:TODO: is this a bad idea? are we masking a real problem?)
- * </p>
- */
- public Insanity[] check(CacheEntry... cacheEntries) {
- if (null == cacheEntries || 0 == cacheEntries.length)
- return new Insanity[0];
-
- // the indirect mapping lets MapOfSet dedup identical valIds for us
- //
- // maps the (valId) identityhashCode of cache values to
- // sets of CacheEntry instances
- final MapOfSets<Integer, CacheEntry> valIdToItems = new MapOfSets<>(new HashMap<Integer, Set<CacheEntry>>(17));
- // maps ReaderField keys to Sets of ValueIds
- final MapOfSets<ReaderField, Integer> readerFieldToValIds = new MapOfSets<>(new HashMap<ReaderField, Set<Integer>>(17));
- //
-
- // any keys that we know result in more then one valId
- final Set<ReaderField> valMismatchKeys = new HashSet<>();
-
- // iterate over all the cacheEntries to get the mappings we'll need
- for (int i = 0; i < cacheEntries.length; i++) {
- final CacheEntry item = cacheEntries[i];
- final Accountable val = item.getValue();
-
- // It's OK to have dup entries, where one is eg
- // float[] and the other is the Bits (from
- // getDocWithField())
- if (val instanceof FieldCacheImpl.BitsEntry) {
- continue;
- }
-
- if (val instanceof FieldCache.CreationPlaceholder)
- continue;
-
- final ReaderField rf = new ReaderField(item.getReaderKey(),
- item.getFieldName());
-
- final Integer valId = Integer.valueOf(System.identityHashCode(val));
-
- // indirect mapping, so the MapOfSet will dedup identical valIds for us
- valIdToItems.put(valId, item);
- if (1 < readerFieldToValIds.put(rf, valId)) {
- valMismatchKeys.add(rf);
- }
- }
-
- final List<Insanity> insanity = new ArrayList<>(valMismatchKeys.size() * 3);
-
- insanity.addAll(checkValueMismatch(valIdToItems,
- readerFieldToValIds,
- valMismatchKeys));
- insanity.addAll(checkSubreaders(valIdToItems,
- readerFieldToValIds));
-
- return insanity.toArray(new Insanity[insanity.size()]);
- }
-
- /**
- * Internal helper method used by check that iterates over
- * valMismatchKeys and generates a Collection of Insanity
- * instances accordingly. The MapOfSets are used to populate
- * the Insanity objects.
- * @see InsanityType#VALUEMISMATCH
- */
- private Collection<Insanity> checkValueMismatch(MapOfSets<Integer, CacheEntry> valIdToItems,
- MapOfSets<ReaderField, Integer> readerFieldToValIds,
- Set<ReaderField> valMismatchKeys) {
-
- final List<Insanity> insanity = new ArrayList<>(valMismatchKeys.size() * 3);
-
- if (! valMismatchKeys.isEmpty() ) {
- // we have multiple values for some ReaderFields
-
- final Map<ReaderField, Set<Integer>> rfMap = readerFieldToValIds.getMap();
- final Map<Integer, Set<CacheEntry>> valMap = valIdToItems.getMap();
- for (final ReaderField rf : valMismatchKeys) {
- final List<CacheEntry> badEntries = new ArrayList<>(valMismatchKeys.size() * 2);
- for(final Integer value: rfMap.get(rf)) {
- for (final CacheEntry cacheEntry : valMap.get(value)) {
- badEntries.add(cacheEntry);
- }
- }
-
- CacheEntry[] badness = new CacheEntry[badEntries.size()];
- badness = badEntries.toArray(badness);
-
- insanity.add(new Insanity(InsanityType.VALUEMISMATCH,
- "Multiple distinct value objects for " +
- rf.toString(), badness));
- }
- }
- return insanity;
- }
-
- /**
- * Internal helper method used by check that iterates over
- * the keys of readerFieldToValIds and generates a Collection
- * of Insanity instances whenever two (or more) ReaderField instances are
- * found that have an ancestry relationships.
- *
- * @see InsanityType#SUBREADER
- */
- private Collection<Insanity> checkSubreaders( MapOfSets<Integer, CacheEntry> valIdToItems,
- MapOfSets<ReaderField, Integer> readerFieldToValIds) {
-
- final List<Insanity> insanity = new ArrayList<>(23);
-
- Map<ReaderField, Set<ReaderField>> badChildren = new HashMap<>(17);
- MapOfSets<ReaderField, ReaderField> badKids = new MapOfSets<>(badChildren); // wrapper
-
- Map<Integer, Set<CacheEntry>> viToItemSets = valIdToItems.getMap();
- Map<ReaderField, Set<Integer>> rfToValIdSets = readerFieldToValIds.getMap();
-
- Set<ReaderField> seen = new HashSet<>(17);
-
- Set<ReaderField> readerFields = rfToValIdSets.keySet();
- for (final ReaderField rf : readerFields) {
-
- if (seen.contains(rf)) continue;
-
- List<Object> kids = getAllDescendantReaderKeys(rf.readerKey);
- for (Object kidKey : kids) {
- ReaderField kid = new ReaderField(kidKey, rf.fieldName);
-
- if (badChildren.containsKey(kid)) {
- // we've already process this kid as RF and found other problems
- // track those problems as our own
- badKids.put(rf, kid);
- badKids.putAll(rf, badChildren.get(kid));
- badChildren.remove(kid);
-
- } else if (rfToValIdSets.containsKey(kid)) {
- // we have cache entries for the kid
- badKids.put(rf, kid);
- }
- seen.add(kid);
- }
- seen.add(rf);
- }
-
- // every mapping in badKids represents an Insanity
- for (final ReaderField parent : badChildren.keySet()) {
- Set<ReaderField> kids = badChildren.get(parent);
-
- List<CacheEntry> badEntries = new ArrayList<>(kids.size() * 2);
-
- // put parent entr(ies) in first
- {
- for (final Integer value : rfToValIdSets.get(parent)) {
- badEntries.addAll(viToItemSets.get(value));
- }
- }
-
- // now the entries for the descendants
- for (final ReaderField kid : kids) {
- for (final Integer value : rfToValIdSets.get(kid)) {
- badEntries.addAll(viToItemSets.get(value));
- }
- }
-
- CacheEntry[] badness = new CacheEntry[badEntries.size()];
- badness = badEntries.toArray(badness);
-
- insanity.add(new Insanity(InsanityType.SUBREADER,
- "Found caches for descendants of " +
- parent.toString(),
- badness));
- }
-
- return insanity;
-
- }
-
- /**
- * Checks if the seed is an IndexReader, and if so will walk
- * the hierarchy of subReaders building up a list of the objects
- * returned by {@code seed.getCoreCacheKey()}
- */
- private List<Object> getAllDescendantReaderKeys(Object seed) {
- List<Object> all = new ArrayList<>(17); // will grow as we iter
- all.add(seed);
- for (int i = 0; i < all.size(); i++) {
- final Object obj = all.get(i);
- // TODO: We don't check closed readers here (as getTopReaderContext
- // throws AlreadyClosedException), what should we do? Reflection?
- if (obj instanceof IndexReader) {
- try {
- final List<IndexReaderContext> childs =
- ((IndexReader) obj).getContext().children();
- if (childs != null) { // it is composite reader
- for (final IndexReaderContext ctx : childs) {
- all.add(ctx.reader().getCoreCacheKey());
- }
- }
- } catch (AlreadyClosedException ace) {
- // ignore this reader
- }
- }
- }
- // need to skip the first, because it was the seed
- return all.subList(1, all.size());
- }
-
- /**
- * Simple pair object for using "readerKey + fieldName" a Map key
- */
- private final static class ReaderField {
- public final Object readerKey;
- public final String fieldName;
- public ReaderField(Object readerKey, String fieldName) {
- this.readerKey = readerKey;
- this.fieldName = fieldName;
- }
- @Override
- public int hashCode() {
- return System.identityHashCode(readerKey) * fieldName.hashCode();
- }
- @Override
- public boolean equals(Object that) {
- if (! (that instanceof ReaderField)) return false;
-
- ReaderField other = (ReaderField) that;
- return (this.readerKey == other.readerKey &&
- this.fieldName.equals(other.fieldName));
- }
- @Override
- public String toString() {
- return readerKey.toString() + "+" + fieldName;
- }
- }
-
- /**
- * Simple container for a collection of related CacheEntry objects that
- * in conjunction with each other represent some "insane" usage of the
- * FieldCache.
- */
- public final static class Insanity {
- private final InsanityType type;
- private final String msg;
- private final CacheEntry[] entries;
- public Insanity(InsanityType type, String msg, CacheEntry... entries) {
- if (null == type) {
- throw new IllegalArgumentException
- ("Insanity requires non-null InsanityType");
- }
- if (null == entries || 0 == entries.length) {
- throw new IllegalArgumentException
- ("Insanity requires non-null/non-empty CacheEntry[]");
- }
- this.type = type;
- this.msg = msg;
- this.entries = entries;
-
- }
- /**
- * Type of insane behavior this object represents
- */
- public InsanityType getType() { return type; }
- /**
- * Description of hte insane behavior
- */
- public String getMsg() { return msg; }
- /**
- * CacheEntry objects which suggest a problem
- */
- public CacheEntry[] getCacheEntries() { return entries; }
- /**
- * Multi-Line representation of this Insanity object, starting with
- * the Type and Msg, followed by each CacheEntry.toString() on its
- * own line prefaced by a tab character
- */
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append(getType()).append(": ");
-
- String m = getMsg();
- if (null != m) buf.append(m);
-
- buf.append('\n');
-
- CacheEntry[] ce = getCacheEntries();
- for (int i = 0; i < ce.length; i++) {
- buf.append('\t').append(ce[i].toString()).append('\n');
- }
-
- return buf.toString();
- }
- }
-
- /**
- * An Enumeration of the different types of "insane" behavior that
- * may be detected in a FieldCache.
- *
- * @see InsanityType#SUBREADER
- * @see InsanityType#VALUEMISMATCH
- * @see InsanityType#EXPECTED
- */
- public final static class InsanityType {
- private final String label;
- private InsanityType(final String label) {
- this.label = label;
- }
- @Override
- public String toString() { return label; }
-
- /**
- * Indicates an overlap in cache usage on a given field
- * in sub/super readers.
- */
- public final static InsanityType SUBREADER
- = new InsanityType("SUBREADER");
-
- /**
- * <p>
- * Indicates entries have the same reader+fieldname but
- * different cached values. This can happen if different datatypes,
- * or parsers are used -- and while it's not necessarily a bug
- * it's typically an indication of a possible problem.
- * </p>
- * <p>
- * <b>NOTE:</b> Only the reader, fieldname, and cached value are actually
- * tested -- if two cache entries have different parsers or datatypes but
- * the cached values are the same Object (== not just equal()) this method
- * does not consider that a red flag. This allows for subtle variations
- * in the way a Parser is specified (null vs DEFAULT_LONG_PARSER, etc...)
- * </p>
- */
- public final static InsanityType VALUEMISMATCH
- = new InsanityType("VALUEMISMATCH");
-
- /**
- * Indicates an expected bit of "insanity". This may be useful for
- * clients that wish to preserve/log information about insane usage
- * but indicate that it was expected.
- */
- public final static InsanityType EXPECTED
- = new InsanityType("EXPECTED");
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
index bc27231..0ba0b81 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
@@ -227,6 +227,15 @@ public class UninvertingReader extends FilterLeafReader {
protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
return new UninvertingDirectoryReader(in, mapping);
}
+
+ // NOTE: delegating the cache helpers is wrong since this wrapper alters the
+ // content of the reader, it is only fine to do that because Solr ALWAYS
+ // consumes index readers through this wrapper
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
+ }
}
final Map<String,Type> mapping;
@@ -391,14 +400,18 @@ public class UninvertingReader extends FilterLeafReader {
return mapping.get(field);
}
+ // NOTE: delegating the cache helpers is wrong since this wrapper alters the
+ // content of the reader, it is only fine to do that because Solr ALWAYS
+ // consumes index readers through this wrapper
+
@Override
- public Object getCoreCacheKey() {
- return in.getCoreCacheKey();
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
}
@Override
- public Object getCombinedCoreAndDeletesKey() {
- return in.getCombinedCoreAndDeletesKey();
+ public CacheHelper getReaderCacheHelper() {
+ return in.getReaderCacheHelper();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
index bd4612e..a147b0f 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
@@ -289,6 +289,16 @@ public class SolrIndexSplitter {
public Bits getLiveDocs() {
return liveDocs;
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/test/org/apache/solr/core/TestNRTOpen.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestNRTOpen.java b/solr/core/src/test/org/apache/solr/core/TestNRTOpen.java
index 1ebd5b6..0970953 100644
--- a/solr/core/src/test/org/apache/solr/core/TestNRTOpen.java
+++ b/solr/core/src/test/org/apache/solr/core/TestNRTOpen.java
@@ -145,7 +145,7 @@ public class TestNRTOpen extends SolrTestCaseJ4 {
try {
DirectoryReader ir = searcher.get().getRawReader();
for (LeafReaderContext context : ir.leaves()) {
- set.add(context.reader().getCoreCacheKey());
+ set.add(context.reader().getCoreCacheHelper().getKey());
}
} finally {
searcher.decref();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java b/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java
index 0685e55..195aae5 100644
--- a/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java
+++ b/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java
@@ -18,15 +18,20 @@ package org.apache.solr.index;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
+import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -48,21 +53,16 @@ public class TestSlowCompositeReaderWrapper extends LuceneTestCase {
final LeafReader leafReader = SlowCompositeReaderWrapper.wrap(reader);
final int numListeners = TestUtil.nextInt(random(), 1, 10);
- final List<LeafReader.CoreClosedListener> listeners = new ArrayList<>();
+ final List<IndexReader.ClosedListener> listeners = new ArrayList<>();
AtomicInteger counter = new AtomicInteger(numListeners);
for (int i = 0; i < numListeners; ++i) {
- CountCoreListener listener = new CountCoreListener(counter, leafReader.getCoreCacheKey());
+ CountCoreListener listener = new CountCoreListener(counter, leafReader.getCoreCacheHelper().getKey());
listeners.add(listener);
- leafReader.addCoreClosedListener(listener);
+ leafReader.getCoreCacheHelper().addClosedListener(listener);
}
for (int i = 0; i < 100; ++i) {
- leafReader.addCoreClosedListener(listeners.get(random().nextInt(listeners.size())));
- }
- final int removed = random().nextInt(numListeners);
- Collections.shuffle(listeners, random());
- for (int i = 0; i < removed; ++i) {
- leafReader.removeCoreClosedListener(listeners.get(i));
+ leafReader.getCoreCacheHelper().addClosedListener(listeners.get(random().nextInt(listeners.size())));
}
assertEquals(numListeners, counter.get());
// make sure listeners are registered on the wrapped reader and that closing any of them has the same effect
@@ -71,11 +71,11 @@ public class TestSlowCompositeReaderWrapper extends LuceneTestCase {
} else {
leafReader.close();
}
- assertEquals(removed, counter.get());
+ assertEquals(0, counter.get());
w.w.getDirectory().close();
}
- private static final class CountCoreListener implements LeafReader.CoreClosedListener {
+ private static final class CountCoreListener implements IndexReader.ClosedListener {
private final AtomicInteger count;
private final Object coreCacheKey;
@@ -86,10 +86,37 @@ public class TestSlowCompositeReaderWrapper extends LuceneTestCase {
}
@Override
- public void onClose(Object coreCacheKey) {
+ public void onClose(IndexReader.CacheKey coreCacheKey) {
assertSame(this.coreCacheKey, coreCacheKey);
count.decrementAndGet();
}
}
+
+ public void testOrdMapsAreCached() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("sorted", new BytesRef("a")));
+ doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("b")));
+ doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("c")));
+ w.addDocument(doc);
+ w.getReader().close();
+ doc = new Document();
+ doc.add(new SortedDocValuesField("sorted", new BytesRef("b")));
+ doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("c")));
+ doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
+ w.addDocument(doc);
+ IndexReader reader = w.getReader();
+ assertTrue(reader.leaves().size() > 1);
+ SlowCompositeReaderWrapper slowWrapper = (SlowCompositeReaderWrapper) SlowCompositeReaderWrapper.wrap(reader);
+ assertEquals(0, slowWrapper.cachedOrdMaps.size());
+ assertEquals(MultiSortedDocValues.class, slowWrapper.getSortedDocValues("sorted").getClass());
+ assertEquals(1, slowWrapper.cachedOrdMaps.size());
+ assertEquals(MultiSortedSetDocValues.class, slowWrapper.getSortedSetDocValues("sorted_set").getClass());
+ assertEquals(2, slowWrapper.cachedOrdMaps.size());
+ reader.close();
+ w.close();
+ dir.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df6f8307/solr/core/src/test/org/apache/solr/search/TestDocSet.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestDocSet.java b/solr/core/src/test/org/apache/solr/search/TestDocSet.java
index 2849f09..db6523e 100644
--- a/solr/core/src/test/org/apache/solr/search/TestDocSet.java
+++ b/solr/core/src/test/org/apache/solr/search/TestDocSet.java
@@ -388,16 +388,6 @@ public class TestDocSet extends LuceneTestCase {
}
@Override
- public void addCoreClosedListener(CoreClosedListener listener) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void removeCoreClosedListener(CoreClosedListener listener) {
- throw new UnsupportedOperationException();
- }
-
- @Override
public FieldInfos getFieldInfos() {
return new FieldInfos(new FieldInfo[0]);
}
@@ -468,6 +458,16 @@ public class TestDocSet extends LuceneTestCase {
public Sort getIndexSort() {
return null;
}
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return null;
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null;
+ }
};
}