You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2020/07/29 11:13:58 UTC
[lucene-solr] branch jira/SOLR-14608-export updated: SOLR-14608:
Fix generics / raw types,
move around the timer metrics so that they make sense.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/SOLR-14608-export
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/SOLR-14608-export by this push:
new 536d962 SOLR-14608: Fix generics / raw types, move around the timer metrics so that they make sense.
536d962 is described below
commit 536d962d6e016573cafd2f420511e4f7083e0468
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Wed Jul 29 13:13:09 2020 +0200
SOLR-14608: Fix generics / raw types, move around the timer metrics so that they make
sense.
---
.../solr/handler/export/DoubleValueSortDoc.java | 2 +-
.../apache/solr/handler/export/ExportBuffers.java | 4 +-
.../apache/solr/handler/export/ExportWriter.java | 65 ++++++++++++++--------
.../solr/handler/export/QuadValueSortDoc.java | 2 +-
.../solr/handler/export/SingleValueSortDoc.java | 2 +-
.../org/apache/solr/handler/export/SortDoc.java | 5 +-
.../apache/solr/handler/export/StringValue.java | 2 +-
.../solr/handler/export/TripleValueSortDoc.java | 4 +-
8 files changed, 51 insertions(+), 35 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
index 117fe6c..8bca043 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
@@ -98,7 +98,7 @@ class DoubleValueSortDoc extends SingleValueSortDoc {
}
}
- public int compareTo(Object o) {
+ public int compareTo(SortDoc o) {
DoubleValueSortDoc sd = (DoubleValueSortDoc) o;
int comp = value1.compareTo(sd.value1);
if (comp == 0) {
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportBuffers.java b/solr/core/src/java/org/apache/solr/handler/export/ExportBuffers.java
index 89c341e..dfeaaaa 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportBuffers.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportBuffers.java
@@ -86,8 +86,8 @@ class ExportBuffers {
this.writeOutputBufferTimer = writeOutputBufferTimer;
this.fillerWaitTimer = fillerWaitTimer;
this.writerWaitTimer = writerWaitTimer;
- this.bufferOne = new Buffer(30000);
- this.bufferTwo = new Buffer(30000);
+ this.bufferOne = new Buffer(queueSize);
+ this.bufferTwo = new Buffer(queueSize);
this.totalHits = totalHits;
fillBuffer = bufferOne;
outputBuffer = bufferTwo;
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
index af1a7d0..3954605 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -116,11 +116,12 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
FixedBitSet[] sets = null;
PushWriter writer;
private String wt;
- final Timer identifyLowestSortingDocTimer;
final Timer transferBatchToBufferTimer;
+ final Timer getMergeIteratorTimer;
final Timer writeOutputBufferTimer;
final Timer writerWaitTimer;
final Timer fillerWaitTimer;
+ final Timer totalTimer;
public ExportWriter(SolrQueryRequest req, SolrQueryResponse res, String wt,
@@ -132,12 +133,15 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
this.initialStreamContext = initialStreamContext;
this.solrMetricsContext = solrMetricsContext;
this.metricsPath = metricsPath;
- this.batchSize = req.getParams().getInt(BATCH_SIZE_PARAM, DEFAULT_BATCH_SIZE);
- identifyLowestSortingDocTimer = solrMetricsContext.timer("identifyLowestSortingDoc", metricsPath);
+ // may be too tricky to get this right? always use default for now
+ //this.batchSize = req.getParams().getInt(BATCH_SIZE_PARAM, DEFAULT_BATCH_SIZE);
+ this.batchSize = DEFAULT_BATCH_SIZE;
+ getMergeIteratorTimer = solrMetricsContext.timer("createMergeIterator", metricsPath);
transferBatchToBufferTimer = solrMetricsContext.timer("transferBatchToBuffer", metricsPath);
writeOutputBufferTimer = solrMetricsContext.timer("writeOutputBuffer", metricsPath);
- writerWaitTimer = solrMetricsContext.timer("writerWaitTimer", metricsPath);
- fillerWaitTimer = solrMetricsContext.timer("fillerWaitTimer", metricsPath);
+ writerWaitTimer = solrMetricsContext.timer("writerWait", metricsPath);
+ fillerWaitTimer = solrMetricsContext.timer("fillerWait", metricsPath);
+ totalTimer = solrMetricsContext.timer("totalTime", metricsPath);
}
@Override
@@ -170,6 +174,15 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
}
public void write(OutputStream os) throws IOException {
+ Timer.Context timerContext = totalTimer.time();
+ try {
+ _write(os);
+ } finally {
+ timerContext.stop();
+ }
+ }
+
+ private void _write(OutputStream os) throws IOException {
QueryResponseWriter rw = req.getCore().getResponseWriters().get(wt);
if (rw instanceof BinaryResponseWriter) {
//todo add support for other writers after testing
@@ -304,13 +317,12 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
return tupleStream;
}
-
private void transferBatchToBufferForOutput(MergeIterator mergeIterator,
ExportBuffers.Buffer destination) throws IOException {
Timer.Context timerContext = transferBatchToBufferTimer.time();
try {
int outDocsIndex = -1;
- for (int i = 0; i < 30000; i++) {
+ for (int i = 0; i < batchSize; i++) {
SortDoc sortDoc = mergeIterator.next();
if (sortDoc != null) {
destination.outDocs[++outDocsIndex].setValues(sortDoc);
@@ -594,7 +606,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
static class MergeIterator {
- private TreeSet<SortDoc> set = new TreeSet();
+ private TreeSet<SortDoc> set = new TreeSet<>();
private SegmentIterator[] segmentIterators;
private SortDoc outDoc;
@@ -639,25 +651,30 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
}
public MergeIterator getMergeIterator(List<LeafReaderContext> leaves, FixedBitSet[] bits, SortDoc sortDoc) throws IOException {
- long totalDocs = 0;
- for (int i = 0; i < leaves.size(); i++) {
- totalDocs += leaves.get(i).reader().maxDoc();
- }
+ Timer.Context timerContext = getMergeIteratorTimer.time();
+ try {
+ long totalDocs = 0;
+ for (int i = 0; i < leaves.size(); i++) {
+ totalDocs += leaves.get(i).reader().maxDoc();
+ }
- int[] sizes = new int[leaves.size()];
- for (int i = 0; i < leaves.size(); i++) {
- long maxDoc = leaves.get(i).reader().maxDoc();
- int sortQueueSize = Math.min((int) (((double) maxDoc / (double) totalDocs) * 200000), 30000);
- sizes[i] = sortQueueSize;
- }
+ int[] sizes = new int[leaves.size()];
+ for (int i = 0; i < leaves.size(); i++) {
+ long maxDoc = leaves.get(i).reader().maxDoc();
+ int sortQueueSize = Math.min((int) (((double) maxDoc / (double) totalDocs) * 200000), batchSize);
+ sizes[i] = sortQueueSize;
+ }
- SegmentIterator[] segmentIterators = new SegmentIterator[leaves.size()];
- for (int i = 0; i < segmentIterators.length; i++) {
- SortQueue sortQueue = new SortQueue(sizes[i], sortDoc.copy());
- segmentIterators[i] = new SegmentIterator(bits[i], leaves.get(i), sortQueue, sortDoc.copy());
- }
+ SegmentIterator[] segmentIterators = new SegmentIterator[leaves.size()];
+ for (int i = 0; i < segmentIterators.length; i++) {
+ SortQueue sortQueue = new SortQueue(sizes[i], sortDoc.copy());
+ segmentIterators[i] = new SegmentIterator(bits[i], leaves.get(i), sortQueue, sortDoc.copy());
+ }
- return new MergeIterator(segmentIterators, sortDoc);
+ return new MergeIterator(segmentIterators, sortDoc);
+ } finally {
+ timerContext.stop();
+ }
}
private static class SegmentIterator {
diff --git a/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
index bf4c6a6..38ecaf9 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
@@ -128,7 +128,7 @@ class QuadValueSortDoc extends TripleValueSortDoc {
}
}
- public int compareTo(Object o) {
+ public int compareTo(SortDoc o) {
QuadValueSortDoc sd = (QuadValueSortDoc) o;
int comp = value1.compareTo(sd.value1);
if (comp == 0) {
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
index 8e59778..61f20b4 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
@@ -83,7 +83,7 @@ class SingleValueSortDoc extends SortDoc {
}
}
- public int compareTo(Object o) {
+ public int compareTo(SortDoc o) {
SingleValueSortDoc sd = (SingleValueSortDoc) o;
int comp = value1.compareTo(sd.value1);
if (comp == 0) {
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
index e21e18b..2bad3c8 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
-class SortDoc implements Comparable {
+class SortDoc implements Comparable<SortDoc> {
protected int docId = -1;
protected int ord = -1;
@@ -112,8 +112,7 @@ class SortDoc implements Comparable {
return docId + docBase > sd.docId + sd.docBase; //index order
}
- public int compareTo(Object o) {
- SortDoc sd = (SortDoc) o;
+ public int compareTo(SortDoc sd) {
for (int i = 0; i < sortValues.length; i++) {
int comp = sortValues[i].compareTo(sd.sortValues[i]);
if (comp != 0) {
diff --git a/solr/core/src/java/org/apache/solr/handler/export/StringValue.java b/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
index d45018a..d485972 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
@@ -124,7 +124,7 @@ class StringValue implements SortValue {
public void toGlobalValue(SortValue previousValue) {
lastOrd = currentOrd;
- StringValue sv = (StringValue)previousValue;
+ StringValue sv = (StringValue) previousValue;
if (sv.lastOrd == currentOrd) {
//Take the global ord from the previousValue
this.currentOrd = sv.currentOrd;
diff --git a/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
index 5176ede..bbcf230 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
@@ -112,8 +112,8 @@ class TripleValueSortDoc extends DoubleValueSortDoc {
}
}
- public int compareTo(Object o) {
- TripleValueSortDoc sd = (TripleValueSortDoc)o;
+ public int compareTo(SortDoc o) {
+ TripleValueSortDoc sd = (TripleValueSortDoc) o;
int comp = value1.compareTo(sd.value1);
if (comp == 0) {
comp = value2.compareTo(sd.value2);