You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2020/07/10 13:53:43 UTC
[lucene-solr] branch jira/SOLR-14608-export updated: SOLR-14608:
Size segment level sort queues based on segement maxdoc
This is an automated email from the ASF dual-hosted git repository.
jbernste pushed a commit to branch jira/SOLR-14608-export
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/SOLR-14608-export by this push:
new 95e706a SOLR-14608: Size segment level sort queues based on segement maxdoc
95e706a is described below
commit 95e706abc425003d79a037500b9887f2c8a7798c
Author: Joel Bernstein <jb...@apache.org>
AuthorDate: Fri Jul 10 09:53:22 2020 -0400
SOLR-14608: Size segment level sort queues based on segement maxdoc
---
.../org/apache/solr/handler/export/ExportWriter.java | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
index 5273c7c..d56607d 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -648,17 +648,28 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
}
private MergeIterator getMergeIterator(List<LeafReaderContext> leafs, FixedBitSet[] bits, SortDoc sortDoc) {
-
- int sortQueueSize = Math.min(100000/bits.length, 30000);
+
+ long totalDocs = 0;
+ for(int i=0; i< leafs.size(); i++) {
+ totalDocs += leafs.get(i).reader().maxDoc();
+ }
+
+ int[] sizes = new int[leafs.size()];
+ for(int i=0; i< leafs.size(); i++) {
+ long maxDoc = leafs.get(i).reader().maxDoc();
+ int sortQueueSize = Math.min((int)((maxDoc/totalDocs) * 100000), 30000);
+ sizes[i] = sortQueueSize;
+ }
+
SegmentIterator[] segmentIterators = new SegmentIterator[bits.length];
for(int i=0; i<segmentIterators.length; i++) {
- SortQueue sortQueue = new SortQueue(sortQueueSize, sortDoc);
+ SortQueue sortQueue = new SortQueue(sizes[i], sortDoc);
segmentIterators[i] = new SegmentIterator(bits[i], leafs.get(i), sortQueue, sortDoc);
}
+
return new MergeIterator(segmentIterators);
}
-
private static class SegmentIterator {
private FixedBitSet bits;