You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2017/11/09 14:03:16 UTC
[3/4] lucene-solr:master: LUCENE-8045: ParallelLeafReader should
correctly report FieldInfo.dvGen
LUCENE-8045: ParallelLeafReader should correctly report FieldInfo.dvGen
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/77ea2110
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/77ea2110
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/77ea2110
Branch: refs/heads/master
Commit: 77ea2110965a05e91cbd04257ccd394e1fd3fad3
Parents: b03c724
Author: Alan Woodward <ro...@apache.org>
Authored: Thu Nov 9 13:31:20 2017 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Thu Nov 9 14:02:57 2017 +0000
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../org/apache/lucene/index/FieldInfos.java | 12 ++++++---
.../apache/lucene/index/ParallelLeafReader.java | 2 +-
.../lucene/index/TestParallelLeafReader.java | 26 ++++++++++++++++++++
4 files changed, 39 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77ea2110/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 867843e..9477a79 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -58,6 +58,9 @@ Bug Fixes
Previously bogus numbers were used, and many similarities would
completely degrade. (Robert Muir, Adrien Grand)
+* LUCENE-8045: ParallelLeafReader did not correctly report FieldInfo.dvGen
+ (Alan Woodward)
+
Optimizations
* LUCENE-8018: Smaller FieldInfos memory footprint by not retaining unnecessary
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77ea2110/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
index a7dc0b8..3e68693 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
@@ -403,7 +403,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber,
boolean storeTermVector,
- boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues,
+ boolean omitNorms, boolean storePayloads, IndexOptions indexOptions,
+ DocValuesType docValues, long dvGen,
int dimensionCount, int dimensionNumBytes) {
assert assertNotFinished();
if (docValues == null) {
@@ -417,7 +418,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
// before then we'll get the same name and number,
// else we'll allocate a new one:
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues, dimensionCount, dimensionNumBytes);
- fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, new HashMap<>(), dimensionCount, dimensionNumBytes);
+ fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes);
assert !byName.containsKey(fi.name);
globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
byName.put(fi.name, fi);
@@ -435,16 +436,21 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
fi.setDocValuesType(docValues); // this will also perform the consistency check.
+ fi.setDocValuesGen(dvGen);
}
}
return fi;
}
public FieldInfo add(FieldInfo fi) {
+ return add(fi, -1);
+ }
+
+ public FieldInfo add(FieldInfo fi, long dvGen) {
// IMPORTANT - reuse the field number if possible for consistent field numbers across segments
return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(),
fi.omitsNorms(), fi.hasPayloads(),
- fi.getIndexOptions(), fi.getDocValuesType(),
+ fi.getIndexOptions(), fi.getDocValuesType(), dvGen,
fi.getPointDimensionCount(), fi.getPointNumBytes());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77ea2110/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
index 74b3962..492b6e7 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
@@ -129,7 +129,7 @@ public class ParallelLeafReader extends LeafReader {
for (FieldInfo fieldInfo : readerFieldInfos) {
// NOTE: first reader having a given field "wins":
if (!fieldToReader.containsKey(fieldInfo.name)) {
- builder.add(fieldInfo);
+ builder.add(fieldInfo, fieldInfo.getDocValuesGen());
fieldToReader.put(fieldInfo.name, reader);
// only add these if the reader responsible for that field name is the current:
// TODO consider populating 1st leaf with vectors even if the field name has been seen on a previous leaf
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77ea2110/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java
index 35523f3..71110d3 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java
@@ -23,6 +23,7 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.AlreadyClosedException;
@@ -371,4 +372,29 @@ public class TestParallelLeafReader extends LuceneTestCase {
new ParallelLeafReader(false, getOnlyLeafReader(r2), getOnlyLeafReader(r1)).close();
IOUtils.close(r1, dir1, r2, dir2);
}
+
+ public void testWithDocValuesUpdates() throws Exception {
+ Directory dir1 = newDirectory();
+ IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
+ IndexWriter w1 = new IndexWriter(dir1, iwc1);
+ Document d = new Document();
+ d.add(newTextField("name", "billy", Field.Store.NO));
+ d.add(new NumericDocValuesField("age", 21));
+ w1.addDocument(d);
+ w1.commit();
+ w1.updateNumericDocValue(new Term("name", "billy"), "age", 22);
+ w1.close();
+
+ IndexReader r1 = DirectoryReader.open(dir1);
+ LeafReader lr = new ParallelLeafReader(false, getOnlyLeafReader(r1));
+
+ NumericDocValues dv = lr.getNumericDocValues("age");
+ assertEquals(0, dv.nextDoc());
+ assertEquals(22, dv.longValue());
+
+ assertEquals(1, lr.getFieldInfos().fieldInfo("age").getDocValuesGen());
+
+ IOUtils.close(lr, r1, dir1);
+
+ }
}