You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2020/07/15 19:56:09 UTC
[lucene-solr] branch jira/SOLR-14608-export updated: SOLR-14608:
Works with one string value sort field.
This is an automated email from the ASF dual-hosted git repository.
jbernste pushed a commit to branch jira/SOLR-14608-export
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/SOLR-14608-export by this push:
new ee9c3d0 SOLR-14608: Works with one string value sort field.
ee9c3d0 is described below
commit ee9c3d083c60850c3e48d301356329cf0f017c86
Author: Joel Bernstein <jb...@apache.org>
AuthorDate: Wed Jul 15 15:55:48 2020 -0400
SOLR-14608: Works with one string value sort field.
---
.../apache/solr/handler/export/ExportBuffers.java | 3 +
.../apache/solr/handler/export/ExportWriter.java | 137 ++++++++++-----------
.../solr/handler/export/ExportWriterStream.java | 4 +-
.../solr/handler/export/SingleValueSortDoc.java | 5 +
.../org/apache/solr/handler/export/SortDoc.java | 2 +-
.../solr/handler/export/StringFieldWriter.java | 41 ++----
.../apache/solr/handler/export/StringValue.java | 8 +-
.../java/org/apache/solr/util/SolrLogPostTool.java | 5 +
8 files changed, 101 insertions(+), 104 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportBuffers.java b/solr/core/src/java/org/apache/solr/handler/export/ExportBuffers.java
index 94eaaa9..9e759a9 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportBuffers.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportBuffers.java
@@ -92,7 +92,9 @@ class ExportBuffers {
fillBuffer = bufferOne;
outputBuffer = bufferTwo;
SortDoc writerSortDoc = exportWriter.getSortDoc(searcher, sort.getSort());
+
MergeIterator mergeIterator = exportWriter.getMergeIterator(leaves, sets, writerSortDoc);
+
bufferOne.initialize(writerSortDoc);
bufferTwo.initialize(writerSortDoc);
barrier = new CyclicBarrier(2, () -> swapBuffers());
@@ -103,6 +105,7 @@ class ExportBuffers {
long lastOutputCounter = 0;
for (int count = 0; count < totalHits; ) {
// log.debug("--- filler fillOutDocs in {}", fillBuffer);
+
exportWriter.fillOutDocs(mergeIterator, buffer);
count += (buffer.outDocsIndex + 1);
// log.debug("--- filler count={}, exchange buffer from {}", count, buffer);
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
index 3fd94f4..e74ea23 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -80,6 +80,7 @@ import org.slf4j.LoggerFactory;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
+import java.util.LinkedList;
import static org.apache.solr.common.util.Utils.makeMap;
/**
@@ -303,47 +304,6 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
return tupleStream;
}
- private void identifyLowestSortingUnexportedDocs(List<LeafReaderContext> leaves, SortDoc sortDoc, SortDoc[] outDocs) throws IOException {
- Timer.Context timerContext = identifyLowestSortingDocTimer.time();
- try {
- MergeIterator mergeIterator = getMergeIterator(leaves, sets, sortDoc);
- for(int i=0; i<outDocs.length; i++) {
- SortDoc sdoc = mergeIterator.next();
- if(sdoc == null) {
- //Null out the rest of the outDocs
- outDocs[i] = null;
- } else {
- outDocs[i].setValues(sdoc);
- }
- }
- } finally {
- timerContext.stop();
- }
- }
-
- private void identifyLowestSortingUnexportedDocs(List<LeafReaderContext> leaves, SortDoc sortDoc, SortQueue queue) throws IOException {
- Timer.Context timerContext = identifyLowestSortingDocTimer.time();
-
- try {
- queue.reset();
- SortDoc top = queue.top();
- for (int i = 0; i < leaves.size(); i++) {
- sortDoc.setNextReader(leaves.get(i));
- DocIdSetIterator it = new BitSetIterator(sets[i], 0); // cost is not useful here
- int docId;
- while ((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- sortDoc.setValues(docId);
- if (top.lessThan(sortDoc)) {
- top.setValues(sortDoc);
- top = queue.updateTop();
- }
- }
- }
- } finally {
- timerContext.stop();
- }
- }
-
private void transferBatchToBufferForOutput(MergeIterator mergeIterator,
ExportBuffers.Buffer destination) throws IOException {
@@ -407,6 +367,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
ExportBuffers buffers = new ExportBuffers(this, leaves, req.getSearcher(), os, writer, sort, queueSize, totalHits,
writeOutputBufferTimer, fillerWaitTimer, writerWaitTimer, sets);
+
if (streamExpression != null) {
streamContext.put(ExportBuffers.EXPORT_BUFFERS_KEY, buffers);
final TupleStream tupleStream;
@@ -459,7 +420,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
}
Timer.Context timerContext = writeOutputBufferTimer.time();
try {
- for (int i = buffer.outDocsIndex; i >= 0; --i) {
+ for (int i = 0; i <= buffer.outDocsIndex; ++i) {
// we're using the raw writer here because there's no potential
// reduction in the number of output items, unlike when using
// streaming expressions
@@ -484,21 +445,14 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
}
}
-
- void fillOutDocs(List<LeafReaderContext> leaves, SortDoc sortDoc,
- SortQueue sortQueue, ExportBuffers.Buffer buffer) throws IOException {
- identifyLowestSortingUnexportedDocs(leaves, sortDoc, sortQueue);
- transferBatchToBufferForOutput(sortQueue, leaves, buffer);
- }
-
void fillOutDocs(MergeIterator mergeIterator,
ExportBuffers.Buffer buffer) throws IOException {
transferBatchToBufferForOutput(mergeIterator, buffer);
}
void writeDoc(SortDoc sortDoc,
- List<LeafReaderContext> leaves,
- EntryWriter ew, FieldWriter[] writers) throws IOException {
+ List<LeafReaderContext> leaves,
+ EntryWriter ew, FieldWriter[] writers) throws IOException {
int ord = sortDoc.ord;
LeafReaderContext context = leaves.get(ord);
int fieldIndex = 0;
@@ -667,18 +621,27 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
return new SortDoc(sortValues);
}
+
public static class MergeIterator {
private TreeSet<SortDoc> set = new TreeSet();
private SegmentIterator[] segmentIterators;
+ private SortDoc outDoc;
- public MergeIterator(SegmentIterator[] segmentIterators) {
+ public MergeIterator(SegmentIterator[] segmentIterators, SortDoc proto) {
+ outDoc = proto.copy();
this.segmentIterators = segmentIterators;
for(int i=0; i<segmentIterators.length; i++) {
- SortDoc sortDoc = segmentIterators[i].next();
- if(sortDoc != null) {
- set.add(sortDoc);
+ try {
+ SortDoc sortDoc = segmentIterators[i].next();
+ if(sortDoc != null) {
+ set.add(sortDoc);
+ }
+ } catch(Exception e) {
+ e.printStackTrace();
+ throw e;
}
+
}
}
@@ -689,11 +652,12 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
public SortDoc next() {
- SortDoc sortDoc = set.pollFirst();
-
+ SortDoc sortDoc = set.pollLast();
//We've exhausted all documents
if(sortDoc == null) {
return null;
+ } else {
+ outDoc.setValues(sortDoc);
}
SortDoc nextDoc = segmentIterators[sortDoc.ord].next();
@@ -701,7 +665,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
//The entire expense of the operation is here
set.add(nextDoc);
}
- return sortDoc;
+ return outDoc;
}
}
@@ -715,17 +679,17 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
int[] sizes = new int[leafs.size()];
for(int i=0; i< leafs.size(); i++) {
long maxDoc = leafs.get(i).reader().maxDoc();
- int sortQueueSize = Math.min((int)((maxDoc/totalDocs) * 100000), 30000);
+ int sortQueueSize = Math.min((int)(((double)maxDoc/(double)totalDocs) * 200000), 30000);
sizes[i] = sortQueueSize;
}
- SegmentIterator[] segmentIterators = new SegmentIterator[bits.length];
+ SegmentIterator[] segmentIterators = new SegmentIterator[leafs.size()];
for(int i=0; i<segmentIterators.length; i++) {
- SortQueue sortQueue = new SortQueue(sizes[i], sortDoc);
+ SortQueue sortQueue = new SortQueue(sizes[i], sortDoc.copy());
segmentIterators[i] = new SegmentIterator(bits[i], leafs.get(i), sortQueue, sortDoc.copy());
}
- return new MergeIterator(segmentIterators);
+ return new MergeIterator(segmentIterators, sortDoc);
}
private static class SegmentIterator {
@@ -734,39 +698,60 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
private SortQueue queue;
private SortDoc sortDoc;
private SortDoc lastDoc;
+ private SortDoc nextDoc;
+ private int ord;
+ private LeafReaderContext context;
+ private SortDoc[] outDocs;
+ private int index;
+
public SegmentIterator(FixedBitSet bits, LeafReaderContext context, SortQueue sortQueue, SortDoc sortDoc) throws IOException {
this.bits = bits;
this.queue = sortQueue;
this.sortDoc = sortDoc;
- this.sortDoc.setNextReader(context);
this.lastDoc = sortDoc.copy();
+ this.nextDoc = sortDoc.copy();
+ this.context = context;
+ this.ord = context.ord;
+ this.outDocs = new SortDoc[sortQueue.maxSize];
topDocs();
}
public SortDoc next() {
- SortDoc sortDoc = queue.pop();
- if(sortDoc == null) {
+ SortDoc sortDoc = null;
+ if(index > -1) {
+ sortDoc = outDocs[index--];
+ } else {
topDocs();
- sortDoc = queue.pop();
+ if(index > -1) {
+ sortDoc = outDocs[index--];
+ }
}
if(sortDoc != null) {
//Clear the bit so it's not loaded again.
bits.clear(sortDoc.docId);
+
//Load the global ordinal (only matters for strings)
sortDoc.setGlobalValues(lastDoc);
+
//Save this doc so we don't have to lookup the global ordinal for the next doc if they have the same segment ordinal.
lastDoc.setValues(sortDoc);
+ nextDoc.setValues(sortDoc);
+ //We are now done with this doc.
+ sortDoc.reset();
+ } else {
+ nextDoc = null;
}
- return sortDoc;
+ return nextDoc;
}
- private void topDocs() {
+ private void topDocs() {
try {
queue.reset();
SortDoc top = queue.top();
+ sortDoc.setNextReader(context);
DocIdSetIterator it = new BitSetIterator(bits, 0); // cost is not useful here
int docId;
while ((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
@@ -776,8 +761,22 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
top = queue.updateTop();
}
}
- } catch(Exception e) {
+ //Pop the queue and load up the array.
+ index = -1;
+
+ while(true) {
+ SortDoc sortDoc = queue.pop();
+ if(sortDoc == null) {
+ break;
+ } else {
+ if(sortDoc.docId > -1) {
+ outDocs[++index] = sortDoc;
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java
index f096810..f76dc42 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java
@@ -56,6 +56,7 @@ public class ExportWriterStream extends TupleStream implements Expressible {
StreamContext context;
StreamComparator streamComparator;
int pos = -1;
+ int index = -1;
ExportBuffers exportBuffers;
ExportBuffers.Buffer buffer;
Timer.Context writeOutputTimerContext;
@@ -196,6 +197,7 @@ public class ExportWriterStream extends TupleStream implements Expressible {
res = Tuple.EOF();
} else {
pos = buffer.outDocsIndex;
+ index = -1; //restart index.
log.debug("--- ews new pos={}", pos);
}
}
@@ -213,7 +215,7 @@ public class ExportWriterStream extends TupleStream implements Expressible {
if (writeOutputTimerContext == null) {
writeOutputTimerContext = exportBuffers.getWriteOutputBufferTimer().time();
}
- SortDoc sortDoc = buffer.outDocs[pos];
+ SortDoc sortDoc = buffer.outDocs[++index];
tupleEntryWriter.tuple = new Tuple();
exportBuffers.exportWriter.writeDoc(sortDoc, exportBuffers.leaves, tupleEntryWriter, exportBuffers.exportWriter.fieldWriters);
pos--;
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
index 913daf9..25e01a5 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
@@ -62,6 +62,11 @@ class SingleValueSortDoc extends SortDoc {
this.value1 = value1;
}
+ public void setGlobalValues(SortDoc previous) {
+ SortValue previousValue = ((SingleValueSortDoc)previous).value1;
+ value1.toGlobalValue(previousValue);
+ }
+
public SortDoc copy() {
return new SingleValueSortDoc(value1.copy());
}
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
index c4bd018..ac0bd7b 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
-class SortDoc {
+class SortDoc implements Comparable {
protected int docId = -1;
protected int ord = -1;
diff --git a/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java
index 846eb08..4a80c07 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java
@@ -55,45 +55,22 @@ class StringFieldWriter extends FieldWriter {
public boolean write(SortDoc sortDoc, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
BytesRef ref;
- SortValue sortValue = sortDoc.getSortValue(this.field);
- if (sortValue != null) {
- if (sortValue.isPresent()) {
- ref = (BytesRef) sortValue.getCurrentValue();
- } else { //empty-value
- return false;
- }
- } else {
- // field is not part of 'sort' param, but part of 'fl' param
- SortedDocValues vals = lastDocValues.get(sortDoc.ord);
- if (vals == null || vals.docID() >= sortDoc.docId) {
- vals = DocValues.getSorted(reader, this.field);
- lastDocValues.put(sortDoc.ord, vals);
- }
- if (vals.advance(sortDoc.docId) != sortDoc.docId) {
- return false;
- }
- int ord = vals.ordValue();
- ref = vals.lookupOrd(ord);
+ SortedDocValues vals = DocValues.getSorted(reader, this.field);
+
+ if (vals.advance(sortDoc.docId) != sortDoc.docId) {
+ return false;
}
+ int ord = vals.ordValue();
+ ref = vals.lookupOrd(ord);
+
if (ew instanceof JavaBinCodec.BinEntryWriter) {
ew.put(this.field, utf8.reset(ref.bytes, ref.offset, ref.length, null));
} else {
String v = null;
- if (sortValue != null) {
- v = ((StringValue) sortValue).getLastString();
- if (v == null) {
- fieldType.indexedToReadable(ref, cref);
- v = cref.toString();
- ((StringValue) sortValue).setLastString(v);
- }
- } else {
- fieldType.indexedToReadable(ref, cref);
- v = cref.toString();
- }
-
+ fieldType.indexedToReadable(ref, cref);
+ v = cref.toString();
ew.put(this.field, v);
-
}
return true;
}
diff --git a/solr/core/src/java/org/apache/solr/handler/export/StringValue.java b/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
index 689f85a..a953a6a 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
@@ -38,7 +38,7 @@ class StringValue implements SortValue {
protected LongValues toGlobal = LongValues.IDENTITY; // this segment to global ordinal. NN;
protected SortedDocValues docValues;
- protected int currentOrd;
+ public int currentOrd;
protected int lastDocID;
private boolean present;
@@ -75,6 +75,7 @@ class StringValue implements SortValue {
}
public void setCurrentValue(int docId) throws IOException {
+ //System.out.println(docId +":"+lastDocID);
if (docId < lastDocID) {
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
}
@@ -84,9 +85,11 @@ class StringValue implements SortValue {
if (docId > docValues.docID()) {
docValues.advance(docId);
}
+
if (docId == docValues.docID()) {
present = true;
currentOrd = docValues.ordValue();
+ //System.out.println("Local:"+currentOrd+", Global:"+toGlobal.get(docValues.ordValue()));
} else {
present = false;
currentOrd = -1;
@@ -104,6 +107,7 @@ class StringValue implements SortValue {
this.present = v.present;
this.leafOrd = v.leafOrd;
this.lastOrd = v.lastOrd;
+ this.toGlobal = v.toGlobal;
}
public Object getCurrentValue() throws IOException {
@@ -120,6 +124,7 @@ class StringValue implements SortValue {
lastOrd = currentOrd;
StringValue sv = (StringValue)previousValue;
if(sv.lastOrd == currentOrd) {
+ //Take the global ord from the previousValue
this.currentOrd = sv.currentOrd;
} else {
this.currentOrd = (int) toGlobal.get(this.currentOrd);
@@ -142,6 +147,7 @@ class StringValue implements SortValue {
public void reset() {
this.currentOrd = comp.resetValue();
this.present = false;
+ lastDocID = 0;
}
public int compareTo(SortValue o) {
diff --git a/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
index 7f3a567..08e1618 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
@@ -487,6 +487,11 @@ public class SolrLogPostTool {
setFieldIfUnset(doc, "rows_i", dr);
}
+ if(parts[0].equals("start")) {
+ String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
+ setFieldIfUnset(doc, "start_i", dr);
+ }
+
if(parts[0].equals("distrib")) {
String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
setFieldIfUnset(doc, "distrib_s", dr);