You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/12/23 16:08:21 UTC
svn commit: r1647588 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/ lucene/misc/
lucene/misc/src/java/org/apache/lucene/index/
lucene/misc/src/java/org/apache/lucene/index/sorter/ lucene/misc/...
Author: rmuir
Date: Tue Dec 23 15:08:20 2014
New Revision: 1647588
URL: http://svn.apache.org/r1647588
Log:
LUCENE-6131: optimize SortingMergePolicy
Added:
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/MergeReaderWrapper.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/MergeReaderWrapper.java
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/Sorter.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/Sorter.java
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/SortingMergePolicy.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/SortingMergePolicy.java
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/search/BlockJoinComparatorSource.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/search/BlockJoinComparatorSource.java
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/search/EarlyTerminatingSortingCollector.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/search/EarlyTerminatingSortingCollector.java
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/search/package.html
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/search/package.html
lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/IndexSortingTest.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/IndexSortingTest.java
lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/SorterTestBase.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/SorterTestBase.java
lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/SortingLeafReaderTest.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/SortingLeafReaderTest.java
lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestBlockJoinSorter.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestBlockJoinSorter.java
lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestSortingMergePolicy.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestSortingMergePolicy.java
lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/search/TestEarlyTerminatingSortingCollector.java
- copied unchanged from r1647587, lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/search/TestEarlyTerminatingSortingCollector.java
Removed:
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/sorter/
lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/sorter/
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
lucene/dev/branches/branch_5x/lucene/misc/ (props changed)
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/package.html
lucene/dev/branches/branch_5x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1647588&r1=1647587&r2=1647588&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Tue Dec 23 15:08:20 2014
@@ -155,6 +155,8 @@ Optimizations
other OS's will continue to use the previous defaults (tuned for spinning disks).
(Robert Muir, Uwe Schindler, hossman, Mike McCandless)
+* LUCENE-6131: Optimize SortingMergePolicy. (Robert Muir)
+
API Changes
* LUCENE-5900: Deprecated more constructors taking Version in *InfixSuggester and
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1647588&r1=1647587&r2=1647588&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Tue Dec 23 15:08:20 2014
@@ -47,6 +47,7 @@ public final class SlowCompositeReaderWr
private final CompositeReader in;
private final Fields fields;
private final Bits liveDocs;
+ private final boolean merging;
/** This method is sugar for getting an {@link LeafReader} from
* an {@link IndexReader} of any kind. If the reader is already atomic,
@@ -54,19 +55,20 @@ public final class SlowCompositeReaderWr
*/
public static LeafReader wrap(IndexReader reader) throws IOException {
if (reader instanceof CompositeReader) {
- return new SlowCompositeReaderWrapper((CompositeReader) reader);
+ return new SlowCompositeReaderWrapper((CompositeReader) reader, false);
} else {
assert reader instanceof LeafReader;
return (LeafReader) reader;
}
}
- private SlowCompositeReaderWrapper(CompositeReader reader) throws IOException {
+ SlowCompositeReaderWrapper(CompositeReader reader, boolean merging) throws IOException {
super();
in = reader;
fields = MultiFields.getFields(in);
liveDocs = MultiFields.getLiveDocs(in);
in.registerParentReader(this);
+ this.merging = merging;
}
@Override
@@ -125,7 +127,7 @@ public final class SlowCompositeReaderWr
SortedDocValues dv = MultiDocValues.getSortedValues(in, field);
if (dv instanceof MultiSortedDocValues) {
map = ((MultiSortedDocValues)dv).mapping;
- if (map.owner == getCoreCacheKey()) {
+ if (map.owner == getCoreCacheKey() && merging == false) {
cachedOrdMaps.put(field, map);
}
}
@@ -163,7 +165,7 @@ public final class SlowCompositeReaderWr
SortedSetDocValues dv = MultiDocValues.getSortedSetValues(in, field);
if (dv instanceof MultiSortedSetDocValues) {
map = ((MultiSortedSetDocValues)dv).mapping;
- if (map.owner == getCoreCacheKey()) {
+ if (map.owner == getCoreCacheKey() && merging == false) {
cachedOrdMaps.put(field, map);
}
}
Modified: lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/package.html?rev=1647588&r1=1647587&r2=1647588&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/package.html (original)
+++ lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/package.html Tue Dec 23 15:08:20 2014
@@ -17,5 +17,23 @@
<html>
<body>
Misc index tools and index support.
+
+SortingMergePolicy:
+<p>Provides index sorting capablities. The application can use any
+Sort specification, e.g. to sort by fields using DocValues or FieldCache, or to
+reverse the order of the documents (by using SortField.Type.DOC in reverse).
+Multi-level sorts can be specified the same way you would when searching, by
+building Sort from multiple SortFields.
+
+<p>{@link org.apache.lucene.index.SortingMergePolicy} can be used to
+make Lucene sort segments before merging them. This will ensure that every
+segment resulting from a merge will be sorted according to the provided
+{@link org.apache.lucene.search.Sort}. This however makes merging and
+thus indexing slower.
+
+<p>Sorted segments allow for early query termination when the sort order
+matches index order. This makes query execution faster since not all documents
+need to be visited. Please note that this is an expert feature and should not
+be used without a deep understanding of Lucene merging and document collection.
</body>
</html>
Modified: lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1647588&r1=1647587&r2=1647588&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Tue Dec 23 15:08:20 2014
@@ -56,12 +56,12 @@ import org.apache.lucene.index.MultiDocV
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.SortingMergePolicy;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.sorter.EarlyTerminatingSortingCollector;
-import org.apache.lucene.index.sorter.SortingMergePolicy;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.EarlyTerminatingSortingCollector;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PrefixQuery;