You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2020/06/18 00:35:42 UTC
[lucene-solr] 02/02: SOLR-14537: Add minimal caching of
SortedDocValues when possible.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/solr-14537
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 27c5a586f417740669d948e156e3a56a23a863e5
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Thu Jun 18 02:28:11 2020 +0200
SOLR-14537: Add minimal caching of SortedDocValues when possible.
---
.../solr/handler/export/DoubleValueSortDoc.java | 1 +
.../org/apache/solr/handler/export/ExportWriter.java | 1 +
.../apache/solr/handler/export/QuadValueSortDoc.java | 1 +
.../solr/handler/export/SingleValueSortDoc.java | 3 ++-
.../java/org/apache/solr/handler/export/SortDoc.java | 19 ++++++++++---------
.../apache/solr/handler/export/StringFieldWriter.java | 9 ++++++++-
.../org/apache/solr/handler/export/StringValue.java | 5 +++--
.../solr/handler/export/TripleValueSortDoc.java | 1 +
8 files changed, 27 insertions(+), 13 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
index 25899f4..8c2a92a 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
@@ -44,6 +44,7 @@ class DoubleValueSortDoc extends SingleValueSortDoc {
public void reset() {
this.docId = -1;
this.docBase = -1;
+ this.ord = -1;
value1.reset();
value2.reset();
}
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
index 7dbeac2..9914ef6 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -24,6 +24,7 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
import java.util.List;
import com.codahale.metrics.Timer;
diff --git a/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
index e4f5a02..b8599c7 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
@@ -50,6 +50,7 @@ class QuadValueSortDoc extends TripleValueSortDoc {
public void reset() {
this.docId = -1;
this.docBase = -1;
+ this.ord = -1;
value1.reset();
value2.reset();
value3.reset();
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
index 963901c..0bd3a4a 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
@@ -41,6 +41,7 @@ class SingleValueSortDoc extends SortDoc {
public void reset() {
this.docId = -1;
this.docBase = -1;
+ this.ord = -1;
this.value1.reset();
}
@@ -83,7 +84,7 @@ class SingleValueSortDoc extends SortDoc {
}
public String toString() {
- return docId+":"+value1.toString();
+ return ord + ":" + docBase + ":" + docId + ":val=" + value1.toString();
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
index 5e2c75d..d893bd1 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
@@ -56,6 +56,7 @@ class SortDoc {
public void reset() {
this.docId = -1;
this.docBase = -1;
+ this.ord = -1;
for (SortValue value : sortValues) {
value.reset();
}
@@ -63,7 +64,7 @@ class SortDoc {
public void setValues(int docId) throws IOException {
this.docId = docId;
- for(SortValue sortValue : sortValues) {
+ for (SortValue sortValue : sortValues) {
sortValue.setCurrentValue(docId);
}
}
@@ -73,14 +74,14 @@ class SortDoc {
this.ord = sortDoc.ord;
this.docBase = sortDoc.docBase;
SortValue[] vals = sortDoc.sortValues;
- for(int i=0; i<vals.length; i++) {
+ for (int i = 0; i < vals.length; i++) {
sortValues[i].setCurrentValue(vals[i]);
}
}
public SortDoc copy() {
SortValue[] svs = new SortValue[sortValues.length];
- for(int i=0; i<sortValues.length; i++) {
+ for (int i = 0; i < sortValues.length; i++) {
svs[i] = sortValues[i].copy();
}
@@ -88,12 +89,12 @@ class SortDoc {
}
public boolean lessThan(Object o) {
- if(docId == -1) {
+ if (docId == -1) {
return true;
}
SortDoc sd = (SortDoc)o;
SortValue[] sortValues1 = sd.sortValues;
- for(int i=0; i<sortValues.length; i++) {
+ for (int i = 0; i < sortValues.length; i++) {
int comp = sortValues[i].compareTo(sortValues1[i]);
if (comp < 0) {
return true;
@@ -101,12 +102,12 @@ class SortDoc {
return false;
}
}
- return docId+docBase > sd.docId+sd.docBase; //index order
+ return docId + docBase > sd.docId + sd.docBase; //index order
}
public int compareTo(Object o) {
SortDoc sd = (SortDoc)o;
- for (int i=0; i<sortValues.length; i++) {
+ for (int i = 0; i < sortValues.length; i++) {
int comp = sortValues[i].compareTo(sd.sortValues[i]);
if (comp != 0) {
return comp;
@@ -118,8 +119,8 @@ class SortDoc {
public String toString() {
StringBuilder builder = new StringBuilder();
- builder.append("docId: ").append(docId).append("; ");
- for (int i=0; i < sortValues.length; i++) {
+ builder.append(ord).append(':').append(docBase).append(':').append(docId).append("; ");
+ for (int i = 0; i < sortValues.length; i++) {
builder.append("value").append(i).append(": ").append(sortValues[i]).append(", ");
}
return builder.toString();
diff --git a/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java
index 7fb24ae..846eb08 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java
@@ -18,6 +18,8 @@
package org.apache.solr.handler.export;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReader;
@@ -32,6 +34,7 @@ import org.apache.solr.schema.FieldType;
class StringFieldWriter extends FieldWriter {
private String field;
private FieldType fieldType;
+ private Map<Integer, SortedDocValues> lastDocValues = new HashMap<>();
private CharsRefBuilder cref = new CharsRefBuilder();
final ByteArrayUtf8CharSequence utf8 = new ByteArrayUtf8CharSequence(new byte[0], 0, 0) {
@Override
@@ -61,7 +64,11 @@ class StringFieldWriter extends FieldWriter {
}
} else {
// field is not part of 'sort' param, but part of 'fl' param
- SortedDocValues vals = DocValues.getSorted(reader, this.field);
+ SortedDocValues vals = lastDocValues.get(sortDoc.ord);
+ if (vals == null || vals.docID() >= sortDoc.docId) {
+ vals = DocValues.getSorted(reader, this.field);
+ lastDocValues.put(sortDoc.ord, vals);
+ }
if (vals.advance(sortDoc.docId) != sortDoc.docId) {
return false;
}
diff --git a/solr/core/src/java/org/apache/solr/handler/export/StringValue.java b/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
index 2ea25d5..c04ca2b 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
@@ -32,12 +32,13 @@ class StringValue implements SortValue {
protected SortedDocValues globalDocValues;
final protected OrdinalMap ordinalMap;
+ final protected String field;
+ final protected IntComp comp;
+
protected LongValues toGlobal = LongValues.IDENTITY; // this segment to global ordinal. NN;
protected SortedDocValues docValues;
- final protected String field;
protected int currentOrd;
- final protected IntComp comp;
protected int lastDocID;
private boolean present;
diff --git a/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
index 3e8bfd8..c68c1a8 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
@@ -47,6 +47,7 @@ class TripleValueSortDoc extends DoubleValueSortDoc {
public void reset() {
this.docId = -1;
this.docBase = -1;
+ this.ord = -1;
value1.reset();
value2.reset();
value3.reset();