You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2022/12/08 09:56:31 UTC
[lucene] branch main updated: Add support for stored fields to MemoryIndex (#11999)
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 66127f6e696 Add support for stored fields to MemoryIndex (#11999)
66127f6e696 is described below
commit 66127f6e69637b081cbc8a2bbbdfac6d0b5dd81f
Author: Alan Woodward <ro...@apache.org>
AuthorDate: Thu Dec 8 09:56:24 2022 +0000
Add support for stored fields to MemoryIndex (#11999)
---
lucene/CHANGES.txt | 2 +
.../apache/lucene/index/memory/MemoryIndex.java | 57 ++++++++++-
.../lucene/index/memory/TestMemoryIndex.java | 106 +++++++++++++++++++++
3 files changed, 163 insertions(+), 2 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ff85eeb8fd7..2b4836a8c41 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -138,6 +138,8 @@ New Features
* GITHUB#11929: MMapDirectory gives more granular control on which files to
preload. (Adrien Grand, Uwe Schindler)
+* GITHUB#11999: MemoryIndex now supports stored fields. (Alan Woodward)
+
Improvements
---------------------
* GITHUB#11778: Detailed part-of-speech information for particle(조사) and ending(어미) on Nori
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index 75d6217fa2e..4925dfb8ac0 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -17,10 +17,12 @@
package org.apache.lucene.index.memory;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.SortedMap;
@@ -416,6 +418,10 @@ public class MemoryIndex {
if (field.fieldType().pointDimensionCount() > 0) {
storePointValues(info, field.binaryValue());
}
+
+ if (field.fieldType().stored()) {
+ storeValues(info, field);
+ }
}
/**
@@ -527,6 +533,26 @@ public class MemoryIndex {
info.pointValues[info.pointValuesCount++] = BytesRef.deepCopyOf(pointValue);
}
+ private void storeValues(Info info, IndexableField field) {
+ if (info.storedValues == null) {
+ info.storedValues = new ArrayList<>();
+ }
+ BytesRef binaryValue = field.binaryValue();
+ if (binaryValue != null) {
+ info.storedValues.add(binaryValue);
+ return;
+ }
+ Number numberValue = field.numericValue();
+ if (numberValue != null) {
+ info.storedValues.add(numberValue);
+ return;
+ }
+ String stringValue = field.stringValue();
+ if (stringValue != null) {
+ info.storedValues.add(stringValue);
+ }
+ }
+
private void storeDocValues(Info info, DocValuesType docValuesType, Object docValuesValue) {
String fieldName = info.fieldInfo.name;
DocValuesType existingDocValuesType = info.fieldInfo.getDocValuesType();
@@ -875,6 +901,8 @@ public class MemoryIndex {
private boolean preparedDocValuesAndPointValues;
+ private List<Object> storedValues;
+
private BytesRef[] pointValues;
private byte[] minPackedValue;
@@ -1797,9 +1825,34 @@ public class MemoryIndex {
}
@Override
- public void document(int docID, StoredFieldVisitor visitor) {
+ public void document(int docID, StoredFieldVisitor visitor) throws IOException {
if (DEBUG) System.err.println("MemoryIndexReader.document");
- // no-op: there are no stored fields
+ for (Info info : fields.values()) {
+ StoredFieldVisitor.Status status = visitor.needsField(info.fieldInfo);
+ if (status == StoredFieldVisitor.Status.STOP) {
+ return;
+ }
+ if (status == StoredFieldVisitor.Status.NO) {
+ continue;
+ }
+ if (info.storedValues != null) {
+ for (Object value : info.storedValues) {
+ if (value instanceof BytesRef bytes) {
+ visitor.binaryField(info.fieldInfo, BytesRef.deepCopyOf(bytes).bytes);
+ } else if (value instanceof Double d) {
+ visitor.doubleField(info.fieldInfo, d);
+ } else if (value instanceof Float f) {
+ visitor.floatField(info.fieldInfo, f);
+ } else if (value instanceof Long l) {
+ visitor.longField(info.fieldInfo, l);
+ } else if (value instanceof Integer i) {
+ visitor.intField(info.fieldInfo, i);
+ } else if (value instanceof String s) {
+ visitor.stringField(info.fieldInfo, s);
+ }
+ }
+ }
+ }
}
@Override
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index 8fcb80f343a..4913e3f7cba 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -22,12 +22,16 @@ import static org.hamcrest.core.StringContains.containsString;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
+import java.util.Objects;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.LongStream;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
@@ -687,4 +691,106 @@ public class TestMemoryIndex extends LuceneTestCase {
+ "fields=2, terms=2, positions=3",
mi.toStringDebug());
}
+
+ public void testStoredFields() throws IOException {
+ List<IndexableField> fields = new ArrayList<>();
+ fields.add(new StoredField("float", 1.5f));
+ fields.add(new StoredField("multifloat", 2.5f));
+ fields.add(new StoredField("multifloat", 3.5f));
+ fields.add(new StoredField("long", 10L));
+ fields.add(new StoredField("multilong", 15L));
+ fields.add(new StoredField("multilong", 20L));
+ fields.add(new StoredField("int", 1.7));
+ fields.add(new StoredField("multiint", 2.7));
+ fields.add(new StoredField("multiint", 2.8));
+ fields.add(new StoredField("multiint", 2.9));
+ fields.add(new StoredField("double", 3.7d));
+ fields.add(new StoredField("multidouble", 4.5d));
+ fields.add(new StoredField("multidouble", 4.6d));
+ fields.add(new StoredField("multidouble", 4.7d));
+ fields.add(new StoredField("string", "foo"));
+ fields.add(new StoredField("multistring", "bar"));
+ fields.add(new StoredField("multistring", "baz"));
+ fields.add(new StoredField("binary", "bfoo".getBytes(StandardCharsets.UTF_8)));
+ fields.add(new StoredField("multibinary", "bbar".getBytes(StandardCharsets.UTF_8)));
+ fields.add(new StoredField("multibinary", "bbaz".getBytes(StandardCharsets.UTF_8)));
+
+ Collections.shuffle(fields, random());
+ Document doc = new Document();
+ for (IndexableField f : fields) {
+ doc.add(f);
+ }
+
+ MemoryIndex mi = MemoryIndex.fromDocument(doc, new StandardAnalyzer());
+ Document d = mi.createSearcher().doc(0);
+
+ assertContains(d, "long", 10L, IndexableField::numericValue);
+ assertContains(d, "int", 1.7, IndexableField::numericValue);
+ assertContains(d, "double", 3.7d, IndexableField::numericValue);
+ assertContains(d, "float", 1.5f, IndexableField::numericValue);
+ assertContains(d, "string", "foo", IndexableField::stringValue);
+ assertBinaryContains(d, "binary", new BytesRef("bfoo"));
+
+ assertMultiContains(d, "multilong", new Object[] {15L, 20L}, IndexableField::numericValue);
+ assertMultiContains(d, "multiint", new Object[] {2.7, 2.8, 2.9}, IndexableField::numericValue);
+ assertMultiContains(
+ d, "multidouble", new Object[] {4.5d, 4.6d, 4.7d}, IndexableField::numericValue);
+ assertMultiContains(d, "multifloat", new Object[] {2.5f, 3.5f}, IndexableField::numericValue);
+ assertMultiContains(d, "multistring", new Object[] {"bar", "baz"}, IndexableField::stringValue);
+ assertBinaryMultiContains(
+ d, "multibinary", new BytesRef[] {new BytesRef("bbar"), new BytesRef("bbaz")});
+ }
+
+ private static void assertContains(
+ Document d, String field, Object expected, Function<IndexableField, Object> value) {
+ assertNotNull(d.getField(field));
+ IndexableField[] fields = d.getFields(field);
+ assertEquals(1, fields.length);
+ assertEquals(expected, value.apply(fields[0]));
+ }
+
+ private static void assertBinaryContains(Document d, String field, BytesRef expected) {
+ assertNotNull(d.getField(field));
+ IndexableField[] fields = d.getFields(field);
+ assertEquals(1, fields.length);
+ assertEquals(0, expected.compareTo(fields[0].binaryValue()));
+ }
+
+ private static void assertMultiContains(
+ Document d, String field, Object[] expected, Function<IndexableField, Object> value) {
+ assertNotNull(d.get(field));
+ IndexableField[] fields = d.getFields(field);
+ assertEquals(expected.length, fields.length);
+ for (IndexableField f : fields) {
+ Object actual = value.apply(f);
+ assertTrue(arrayContains(expected, actual));
+ }
+ }
+
+ private static void assertBinaryMultiContains(Document d, String field, BytesRef[] expected) {
+ IndexableField[] fields = d.getFields(field);
+ assertEquals(expected.length, fields.length);
+ for (IndexableField f : fields) {
+ BytesRef actual = f.binaryValue();
+ assertTrue(arrayBinaryContains(expected, actual));
+ }
+ }
+
+ private static boolean arrayContains(Object[] array, Object value) {
+ for (Object o : array) {
+ if (Objects.equals(o, value)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean arrayBinaryContains(BytesRef[] array, BytesRef value) {
+ for (BytesRef b : array) {
+ if (b.compareTo(value) == 0) {
+ return true;
+ }
+ }
+ return false;
+ }
}