You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ds...@apache.org on 2017/03/17 01:34:04 UTC
lucene-solr:branch_6x: SOLR-10273: DocumentBuilder move longest field
to last position
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 3a3acfda0 -> 993003b33
SOLR-10273: DocumentBuilder move longest field to last position
(cherry picked from commit 8fbd9f1)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/993003b3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/993003b3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/993003b3
Branch: refs/heads/branch_6x
Commit: 993003b33e33ba78c66ffda41acb12b8239c359a
Parents: 3a3acfd
Author: David Smiley <ds...@apache.org>
Authored: Thu Mar 16 21:22:08 2017 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Thu Mar 16 21:33:58 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 4 +
.../org/apache/solr/update/DocumentBuilder.java | 60 ++++++++++-
.../apache/solr/update/DocumentBuilderTest.java | 108 +++++++++++++++----
3 files changed, 149 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/993003b3/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b023422..abddf96 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -234,6 +234,10 @@ Optimizations
artificial random queries over random indexes showed performance increases of 5% to 68%, with common values
in the teens. (yonik)
+* SOLR-10273: The field with the longest value (if it exceeds 4K) is moved to be last in the Lucene Document in order
+ to benefit from stored field optimizations in Lucene that can avoid reading it when it's not needed. If the field is
+ multi-valued, they all move together to the end to retain order. (David Smiley)
+
Other Changes
----------------------
* SOLR-9980: Expose configVersion in core admin status (Jessica Cheng Mallet via Tom�s Fern�ndez L�bbe)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/993003b3/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
index ab82c9d..00c6171 100644
--- a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
+++ b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.update;
+import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Set;
@@ -23,6 +25,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
@@ -34,10 +37,13 @@ import org.apache.solr.schema.SchemaField;
import com.google.common.collect.Sets;
/**
- *
+ * Builds a Lucene {@link Document} from a {@link SolrInputDocument}.
*/
public class DocumentBuilder {
+ // accessible only for tests
+ static int MIN_LENGTH_TO_MOVE_LAST = Integer.getInteger("solr.docBuilder.minLengthToMoveLast", 4*1024); // internal setting
+
/**
* Add a field value to a given document.
* @param doc Document that the field needs to be added to
@@ -261,6 +267,58 @@ public class DocumentBuilder {
}
}
}
+
+ if (!forInPlaceUpdate) {
+ moveLargestFieldLast(out);
+ }
+
return out;
}
+
+ /** Move the largest stored field last, because Lucene can avoid loading that one if it's not needed. */
+ private static void moveLargestFieldLast(Document doc) {
+ String largestField = null;
+ int largestFieldLen = -1;
+ boolean largestIsLast = true;
+ for (IndexableField field : doc) {
+ if (!field.fieldType().stored()) {
+ continue;
+ }
+ if (largestIsLast && !field.name().equals(largestField)) {
+ largestIsLast = false;
+ }
+ if (field.numericValue() != null) { // just ignore these as non-competitive (avoid toString'ing their number)
+ continue;
+ }
+ String strVal = field.stringValue();
+ if (strVal != null) {
+ if (strVal.length() > largestFieldLen) {
+ largestField = field.name();
+ largestFieldLen = strVal.length();
+ largestIsLast = true;
+ }
+ } else {
+ BytesRef bytesRef = field.binaryValue();
+ if (bytesRef != null && bytesRef.length > largestFieldLen) {
+ largestField = field.name();
+ largestFieldLen = bytesRef.length;
+ largestIsLast = true;
+ }
+ }
+ }
+ if (!largestIsLast && largestField != null && largestFieldLen > MIN_LENGTH_TO_MOVE_LAST) { // only bother if the value isn't tiny
+ LinkedList<IndexableField> addToEnd = new LinkedList<>();
+ Iterator<IndexableField> iterator = doc.iterator();
+ while (iterator.hasNext()) {
+ IndexableField field = iterator.next();
+ if (field.name().equals(largestField)) {
+ addToEnd.add(field);
+ iterator.remove(); // Document may not have "remove" but it's iterator allows mutation
+ }
+ }
+ for (IndexableField field : addToEnd) {
+ doc.add(field);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/993003b3/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java b/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
index 6e6b30e..9a75fd2 100644
--- a/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
+++ b/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
@@ -16,8 +16,12 @@
*/
package org.apache.solr.update;
+import java.util.Iterator;
import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
@@ -40,6 +44,8 @@ import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
+import org.junit.After;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -48,12 +54,23 @@ import org.junit.Test;
*
*/
public class DocumentBuilderTest extends SolrTestCaseJ4 {
+ static final int save_min_len = DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST;
@BeforeClass
public static void beforeClass() throws Exception {
initCore("solrconfig.xml", "schema.xml");
}
+ @AfterClass
+ public static void afterClass() {
+ DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST = save_min_len;
+ }
+
+ @After
+ public void afterTest() {
+ DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST = save_min_len;
+ }
+
@Test
public void testBuildDocument() throws Exception
{
@@ -144,8 +161,8 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
assertTrue("title_stringNoNorms has the omitNorms attribute set to true, if the boost is different than 1.0, it will fail",1.0f == out.getField( "title_stringNoNorms" ).boost() );
assertTrue("It is OK that title has a boost of 3",3.0f == out.getField( "title" ).boost() );
}
-
-
+
+
@Test
public void testCopyFieldWithFieldBoost() {
SolrCore core = h.getCore();
@@ -159,7 +176,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
assertTrue("title_stringNoNorms has the omitNorms attribute set to true, if the boost is different than 1.0, it will fail",1.0f == out.getField( "title_stringNoNorms" ).boost() );
assertTrue("It is OK that title has a boost of 3",3.0f == out.getField( "title" ).boost() );
}
-
+
@Test
public void testWithPolyFieldsAndFieldBoost() {
SolrCore core = h.getCore();
@@ -170,7 +187,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
assertFalse(schema.getField("amount").omitNorms());
assertTrue(schema.getField("amount" + FieldType.POLY_FIELD_SEPARATOR + "_currency").omitNorms());
assertTrue(schema.getField("amount" + FieldType.POLY_FIELD_SEPARATOR + "_amount_raw").omitNorms());
-
+
SolrInputDocument doc = new SolrInputDocument();
doc.addField( "store", "40.7143,-74.006", 3.0f );
doc.addField( "amount", "10.5", 3.0f );
@@ -184,7 +201,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
assertTrue(1f == out.getField("amount" + FieldType.POLY_FIELD_SEPARATOR + "_currency").boost());
assertTrue(1f == out.getField("amount" + FieldType.POLY_FIELD_SEPARATOR + "_amount_raw").boost());
}
-
+
@Test
public void testWithPolyFieldsAndDocumentBoost() {
SolrCore core = h.getCore();
@@ -195,7 +212,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
assertFalse(schema.getField("amount").omitNorms());
assertTrue(schema.getField("amount" + FieldType.POLY_FIELD_SEPARATOR + "_currency").omitNorms());
assertTrue(schema.getField("amount" + FieldType.POLY_FIELD_SEPARATOR + "_amount_raw").omitNorms());
-
+
SolrInputDocument doc = new SolrInputDocument();
doc.setDocumentBoost(3.0f);
doc.addField( "store", "40.7143,-74.006");
@@ -209,7 +226,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
assertTrue(1f == out.getField("amount" + FieldType.POLY_FIELD_SEPARATOR + "_currency").boost());
assertTrue(1f == out.getField("amount" + FieldType.POLY_FIELD_SEPARATOR + "_amount_raw").boost());
}
-
+
/**
* It's ok to boost a field if it has norms
*/
@@ -244,7 +261,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
assertEquals(15.0f, outF[0].boost(), 0.0f);
assertEquals(1.0f, outF[1].boost(), 0.0f);
assertEquals(1.0f, outF[2].boost(), 0.0f);
-
+
}
public void testMultiValuedFieldAndDocBoostsWithCopy() throws Exception {
@@ -252,12 +269,12 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
IndexSchema schema = core.getLatestSchema();
SolrInputField field = new SolrInputField( "foo_t" );
List<CopyField> copyFields = schema.getCopyFieldsList(field.getName());
-
+
assertNotNull( copyFields );
assertFalse( copyFields.isEmpty() );
assertMultiValuedFieldAndDocBoosts( field );
}
-
+
public void testMultiValuedFieldAndDocBoostsNoCopy() throws Exception {
SolrCore core = h.getCore();
IndexSchema schema = core.getLatestSchema();
@@ -278,15 +295,15 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
doc.addField("id", "42");
SolrInputField inTitle = new SolrInputField( "title" );
- inTitle.addValue( "titleA" , 2.0F );
- inTitle.addValue( "titleB" , 7.0F );
+ inTitle.addValue( "titleA" , 2.0F );
+ inTitle.addValue( "titleB" , 7.0F );
final float TITLE_BOOST = 2.0F * 7.0F;
assertEquals(TITLE_BOOST, inTitle.getBoost(), 0.0F);
doc.put( inTitle.getName(), inTitle );
-
+
SolrInputField inFoo = new SolrInputField( "foo_t" );
inFoo.addValue( "summer time" , 1.0F );
- inFoo.addValue( "in the city" , 5.0F );
+ inFoo.addValue( "in the city" , 5.0F );
inFoo.addValue( "living is easy" , 11.0F );
final float FOO_BOOST = 1.0F * 5.0F * 11.0F;
assertEquals(FOO_BOOST, inFoo.getBoost(), 0.0F);
@@ -326,11 +343,11 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
assertEquals(FOO_BOOST, outText[2].boost(), 0.0F);
assertEquals(1.0F, outText[3].boost(), 0.0F);
assertEquals(1.0F, outText[4].boost(), 0.0F);
-
+
// copyField dest with no norms should not have received any boost
assertEquals(1.0F, outNoNorms[0].boost(), 0.0F);
assertEquals(1.0F, outNoNorms[1].boost(), 0.0F);
-
+
// now index that SolrInputDocument to check the computed norms
assertU(adoc(doc));
@@ -350,12 +367,12 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
SolrIndexSearcher searcher = req.getSearcher();
LeafReader reader = SlowCompositeReaderWrapper.wrap(searcher.getTopReaderContext().reader());
- assertTrue("similarity doesn't extend ClassicSimilarity, " +
+ assertTrue("similarity doesn't extend ClassicSimilarity, " +
"config or defaults have changed since test was written",
searcher.getSimilarity(true) instanceof ClassicSimilarity);
ClassicSimilarity sim = (ClassicSimilarity) searcher.getSimilarity(true);
-
+
NumericDocValues titleNorms = reader.getNormValues("title");
NumericDocValues fooNorms = reader.getNormValues("foo_t");
NumericDocValues textNorms = reader.getNormValues("text");
@@ -366,7 +383,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
assertEquals(expectedNorm(sim, 8-3, FOO_BOOST * DOC_BOOST),
fooNorms.get(docid));
- assertEquals(expectedNorm(sim, 2 + 8-3,
+ assertEquals(expectedNorm(sim, 2 + 8-3,
TITLE_BOOST * FOO_BOOST * DOC_BOOST),
textNorms.get(docid));
@@ -376,13 +393,13 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
}
/**
- * Given a length, and boost returns the expected encoded norm
+ * Given a length, and boost returns the expected encoded norm
*/
private static byte expectedNorm(final ClassicSimilarity sim,
final int length, final float boost) {
return (byte) sim.encodeNormValue(boost / ((float) Math.sqrt(length)));
}
-
+
public void testBoostOmitNorms() throws Exception {
XmlDoc xml = new XmlDoc();
@@ -404,7 +421,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
+ "</doc>";
assertNull(h.validateUpdate(add(xml, new String[0])));
}
-
+
/**
* It's ok to supply a document boost even if a field omits norms
*/
@@ -516,7 +533,54 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
sif2.setName("foo");
assertFalse(assertSolrInputFieldEquals(sif1, sif2));
+ }
+ public void testMoveLargestLast() {
+ SolrInputDocument inDoc = new SolrInputDocument();
+ String TEXT_FLD = "text"; // not stored. It won't be moved. This value is the longest, however.
+ inDoc.addField(TEXT_FLD,
+ "NOT STORED|" + RandomStrings.randomAsciiOfLength(random(), 4 * DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));
+
+ String CAT_FLD = "cat"; // stored, multiValued
+ inDoc.addField(CAT_FLD,
+ "STORED V1|");
+ // pretty long value
+ inDoc.addField(CAT_FLD,
+ "STORED V2|" + RandomStrings.randomAsciiOfLength(random(), 2 * DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));
+ inDoc.addField(CAT_FLD,
+ "STORED V3|" + RandomStrings.randomAsciiOfLength(random(), DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));
+
+ String SUBJECT_FLD = "subject"; // stored. This value is long, but not long enough.
+ inDoc.addField(SUBJECT_FLD,
+ "2ndplace|" + RandomStrings.randomAsciiOfLength(random(), DocumentBuilder.MIN_LENGTH_TO_MOVE_LAST));
+
+ Document outDoc = DocumentBuilder.toDocument(inDoc, h.getCore().getLatestSchema());
+
+ // filter outDoc by stored fields; convert to list.
+ List<IndexableField> storedFields = StreamSupport.stream(outDoc.spliterator(), false)
+ .filter(f -> f.fieldType().stored()).collect(Collectors.toList());
+ // clip to last 3. We expect these to be for CAT_FLD
+ storedFields = storedFields.subList(storedFields.size() - 3, storedFields.size());
+
+ Iterator<IndexableField> fieldIterator = storedFields.iterator();
+ IndexableField field;
+
+ // Test that we retained the particular value ordering, even though though the 2nd of three was longest
+
+ assertTrue(fieldIterator.hasNext());
+ field = fieldIterator.next();
+ assertEquals(CAT_FLD, field.name());
+ assertTrue(field.stringValue().startsWith("STORED V1|"));
+
+ assertTrue(fieldIterator.hasNext());
+ field = fieldIterator.next();
+ assertEquals(CAT_FLD, field.name());
+ assertTrue(field.stringValue().startsWith("STORED V2|"));
+
+ assertTrue(fieldIterator.hasNext());
+ field = fieldIterator.next();
+ assertEquals(CAT_FLD, field.name());
+ assertTrue(field.stringValue().startsWith("STORED V3|"));
}
}