You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2018/06/04 06:40:41 UTC
[2/2] lucene-solr:branch_7x: LUCENE-8335: Enforce soft-deletes field
up-front
LUCENE-8335: Enforce soft-deletes field up-front
Soft deletes field must be marked as such once it's introduced
and can't be changed after the fact.
Co-authored-by: Nhat Nguyen <nh...@elastic.co>
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/67b6593e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/67b6593e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/67b6593e
Branch: refs/heads/branch_7x
Commit: 67b6593e7adbce76532e285cef42118e6cc3448f
Parents: 6f6b7f5
Author: Simon Willnauer <si...@apache.org>
Authored: Mon Jun 4 08:23:31 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon Jun 4 08:40:20 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
.../simpletext/SimpleTextFieldInfosFormat.java | 11 +-
.../lucene50/Lucene50FieldInfosFormat.java | 2 +-
.../lucene60/Lucene60FieldInfosFormat.java | 8 +-
.../java/org/apache/lucene/index/FieldInfo.java | 18 +++-
.../org/apache/lucene/index/FieldInfos.java | 51 ++++++---
.../org/apache/lucene/index/IndexWriter.java | 8 +-
.../org/apache/lucene/index/MultiFields.java | 6 +-
.../apache/lucene/index/ParallelLeafReader.java | 7 +-
.../org/apache/lucene/index/TestCodecs.java | 4 +-
.../test/org/apache/lucene/index/TestDoc.java | 2 +-
.../apache/lucene/index/TestFieldsReader.java | 2 +-
.../apache/lucene/index/TestIndexWriter.java | 106 +++++++++++++++++++
.../lucene/index/TestPendingSoftDeletes.java | 10 +-
.../apache/lucene/index/TestSegmentMerger.java | 2 +-
.../search/highlight/TermVectorLeafReader.java | 2 +-
.../apache/lucene/index/memory/MemoryIndex.java | 4 +-
.../index/BaseFieldInfoFormatTestCase.java | 14 +--
.../index/BaseIndexFileFormatTestCase.java | 2 +-
.../lucene/index/MismatchedLeafReader.java | 3 +-
.../lucene/index/RandomPostingsTester.java | 4 +-
.../solr/handler/component/ExpandComponent.java | 3 +-
.../solr/search/CollapsingQParserPlugin.java | 2 +-
.../java/org/apache/solr/search/Insanity.java | 2 +-
.../solr/uninverting/UninvertingReader.java | 2 +-
25 files changed, 224 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 54c874d..44829fa 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -99,6 +99,10 @@ New Features
now use to also take pending deletes into account which ensures that all file
generations per segment always go forward. (Simon Willnauer)
+* LUCENE-8335: Enforce soft-deletes field up-front. Soft deletes field must be marked
+ as such once it's introduced and can't be changed after the fact.
+ (Nhat Nguyen via Simon Willnauer)
+
Bug Fixes
* LUCENE-8221: MoreLikeThis.setMaxDocFreqPct can easily int-overflow on larger
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
index 0ace153..1c40cbd 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
@@ -66,6 +66,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
static final BytesRef ATT_VALUE = new BytesRef(" value ");
static final BytesRef DIM_COUNT = new BytesRef(" dimensional count ");
static final BytesRef DIM_NUM_BYTES = new BytesRef(" dimensional num bytes ");
+ static final BytesRef SOFT_DELETES = new BytesRef(" soft-deletes ");
@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
@@ -140,9 +141,13 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
assert StringHelper.startsWith(scratch.get(), DIM_NUM_BYTES);
int dimensionalNumBytes = Integer.parseInt(readString(DIM_NUM_BYTES.length, scratch));
+ SimpleTextUtil.readLine(input, scratch);
+ assert StringHelper.startsWith(scratch.get(), SOFT_DELETES);
+ boolean isSoftDeletesField = Boolean.parseBoolean(readString(SOFT_DELETES.length, scratch));
+
infos[i] = new FieldInfo(name, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts),
- dimensionalCount, dimensionalNumBytes);
+ dimensionalCount, dimensionalNumBytes, isSoftDeletesField);
}
SimpleTextUtil.checkFooter(input);
@@ -238,6 +243,10 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
SimpleTextUtil.write(out, DIM_NUM_BYTES);
SimpleTextUtil.write(out, Integer.toString(fi.getPointNumBytes()), scratch);
SimpleTextUtil.writeNewline(out);
+
+ SimpleTextUtil.write(out, SOFT_DELETES);
+ SimpleTextUtil.write(out, Boolean.toString(fi.isSoftDeletesField()), scratch);
+ SimpleTextUtil.writeNewline(out);
}
SimpleTextUtil.writeChecksum(out, scratch);
success = true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
index a76bfeb..30dca70 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
@@ -148,7 +148,7 @@ public final class Lucene50FieldInfosFormat extends FieldInfosFormat {
lastAttributes = attributes;
try {
infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads,
- indexOptions, docValuesType, dvGen, attributes, 0, 0);
+ indexOptions, docValuesType, dvGen, attributes, 0, 0, false);
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java
index a35461e..522a73f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java
@@ -136,6 +136,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
boolean omitNorms = (bits & OMIT_NORMS) != 0;
boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
+ boolean isSoftDeletesField = (bits & SOFT_DELETES_FIELD) != 0;
final IndexOptions indexOptions = getIndexOptions(input, input.readByte());
@@ -159,7 +160,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
try {
infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads,
indexOptions, docValuesType, dvGen, attributes,
- pointDimensionCount, pointNumBytes);
+ pointDimensionCount, pointNumBytes, isSoftDeletesField);
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
@@ -277,6 +278,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
if (fi.omitsNorms()) bits |= OMIT_NORMS;
if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
+ if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;
output.writeByte(bits);
output.writeByte(indexOptionsByte(fi.getIndexOptions()));
@@ -301,10 +303,12 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
// Codec header
static final String CODEC_NAME = "Lucene60FieldInfos";
static final int FORMAT_START = 0;
- static final int FORMAT_CURRENT = FORMAT_START;
+ static final int FORMAT_SOFT_DELETES = 1;
+ static final int FORMAT_CURRENT = FORMAT_SOFT_DELETES;
// Field flags
static final byte STORE_TERMVECTOR = 0x1;
static final byte OMIT_NORMS = 0x2;
static final byte STORE_PAYLOADS = 0x4;
+ static final byte SOFT_DELETES_FIELD = 0x8;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
index 422292b..250c94b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
@@ -53,14 +53,17 @@ public final class FieldInfo {
private int pointDimensionCount;
private int pointNumBytes;
+ // whether this field is used as the soft-deletes field
+ private final boolean softDeletesField;
+
/**
* Sole constructor.
*
* @lucene.experimental
*/
- public FieldInfo(String name, int number, boolean storeTermVector, boolean omitNorms,
- boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues,
- long dvGen, Map<String,String> attributes, int pointDimensionCount, int pointNumBytes) {
+ public FieldInfo(String name, int number, boolean storeTermVector, boolean omitNorms, boolean storePayloads,
+ IndexOptions indexOptions, DocValuesType docValues, long dvGen, Map<String,String> attributes,
+ int pointDimensionCount, int pointNumBytes, boolean softDeletesField) {
this.name = Objects.requireNonNull(name);
this.number = number;
this.docValuesType = Objects.requireNonNull(docValues, "DocValuesType must not be null (field: \"" + name + "\")");
@@ -78,6 +81,7 @@ public final class FieldInfo {
this.attributes = Objects.requireNonNull(attributes);
this.pointDimensionCount = pointDimensionCount;
this.pointNumBytes = pointNumBytes;
+ this.softDeletesField = softDeletesField;
assert checkConsistency();
}
@@ -334,4 +338,12 @@ public final class FieldInfo {
public Map<String,String> attributes() {
return attributes;
}
+
+ /**
+ * Returns true if this field is configured and used as the soft-deletes field.
+ * See {@link IndexWriterConfig#softDeletesField}
+ */
+ public boolean isSoftDeletesField() {
+ return softDeletesField;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
index f47e15c..22ca9de7 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
@@ -43,6 +43,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
private final boolean hasNorms;
private final boolean hasDocValues;
private final boolean hasPointValues;
+ private final String softDeletesField;
// used only by fieldInfo(int)
private final FieldInfo[] byNumber;
@@ -62,6 +63,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
boolean hasNorms = false;
boolean hasDocValues = false;
boolean hasPointValues = false;
+ String softDeletesField = null;
int size = 0; // number of elements in byNumberTemp, number of used array slots
FieldInfo[] byNumberTemp = new FieldInfo[10]; // initial array capacity of 10
@@ -92,6 +94,12 @@ public class FieldInfos implements Iterable<FieldInfo> {
hasDocValues |= info.getDocValuesType() != DocValuesType.NONE;
hasPayloads |= info.hasPayloads();
hasPointValues |= (info.getPointDimensionCount() != 0);
+ if (info.isSoftDeletesField()) {
+ if (softDeletesField != null && softDeletesField.equals(info.name) == false) {
+ throw new IllegalArgumentException("multiple soft-deletes fields [" + info.name + ", " + softDeletesField + "]");
+ }
+ softDeletesField = info.name;
+ }
}
this.hasVectors = hasVectors;
@@ -102,6 +110,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
this.hasNorms = hasNorms;
this.hasDocValues = hasDocValues;
this.hasPointValues = hasPointValues;
+ this.softDeletesField = softDeletesField;
List<FieldInfo> valuesTemp = new ArrayList<>();
byNumber = new FieldInfo[size];
@@ -153,6 +162,11 @@ public class FieldInfos implements Iterable<FieldInfo> {
public boolean hasPointValues() {
return hasPointValues;
}
+
+ /** Returns the soft-deletes field name if exists; otherwise returns null */
+ public String getSoftDeletesField() {
+ return softDeletesField;
+ }
/** Returns the number of fields */
public int size() {
@@ -220,12 +234,16 @@ public class FieldInfos implements Iterable<FieldInfo> {
// norms back on after they were already ommitted; today
// we silently discard the norm but this is badly trappy
private int lowestUnassignedFieldNumber = -1;
+
+ // The soft-deletes field from IWC to enforce a single soft-deletes field
+ private final String softDeletesFieldName;
- FieldNumbers() {
+ FieldNumbers(String softDeletesFieldName) {
this.nameToNumber = new HashMap<>();
this.numberToName = new HashMap<>();
this.docValuesType = new HashMap<>();
this.dimensions = new HashMap<>();
+ this.softDeletesFieldName = softDeletesFieldName;
}
/**
@@ -234,7 +252,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
* number assigned if possible otherwise the first unassigned field number
* is used as the field number.
*/
- synchronized int addOrGet(String fieldName, int preferredFieldNumber, DocValuesType dvType, int dimensionCount, int dimensionNumBytes) {
+ synchronized int addOrGet(String fieldName, int preferredFieldNumber, DocValuesType dvType, int dimensionCount, int dimensionNumBytes, boolean isSoftDeletesField) {
if (dvType != DocValuesType.NONE) {
DocValuesType currentDVType = docValuesType.get(fieldName);
if (currentDVType == null) {
@@ -274,6 +292,16 @@ public class FieldInfos implements Iterable<FieldInfo> {
nameToNumber.put(fieldName, fieldNumber);
}
+ if (isSoftDeletesField) {
+ if (softDeletesFieldName == null) {
+ throw new IllegalArgumentException("this index has [" + fieldName + "] as soft-deletes already but soft-deletes field is not configured in IWC");
+ } else if (fieldName.equals(softDeletesFieldName) == false) {
+ throw new IllegalArgumentException("cannot configure [" + softDeletesFieldName + "] as soft-deletes; this index uses [" + fieldName + "] as soft-deletes already");
+ }
+ } else if (fieldName.equals(softDeletesFieldName)) {
+ throw new IllegalArgumentException("cannot configure [" + softDeletesFieldName + "] as soft-deletes; this index uses [" + fieldName + "] as non-soft-deletes already");
+ }
+
return fieldNumber.intValue();
}
@@ -354,11 +382,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
private final HashMap<String,FieldInfo> byName = new HashMap<>();
final FieldNumbers globalFieldNumbers;
private boolean finished;
-
- Builder() {
- this(new FieldNumbers());
- }
-
+
/**
* Creates a new instance with the given {@link FieldNumbers}.
*/
@@ -384,8 +408,9 @@ public class FieldInfos implements Iterable<FieldInfo> {
// number for this field. If the field was seen
// before then we'll get the same name and number,
// else we'll allocate a new one:
- final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, DocValuesType.NONE, 0, 0);
- fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, new HashMap<>(), 0, 0);
+ final boolean isSoftDeletesField = name.equals(globalFieldNumbers.softDeletesFieldName);
+ final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, DocValuesType.NONE, 0, 0, isSoftDeletesField);
+ fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, new HashMap<>(), 0, 0, isSoftDeletesField);
assert !byName.containsKey(fi.name);
globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, DocValuesType.NONE);
byName.put(fi.name, fi);
@@ -398,7 +423,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions,
DocValuesType docValues, long dvGen,
- int dimensionCount, int dimensionNumBytes) {
+ int dimensionCount, int dimensionNumBytes, boolean isSoftDeletesField) {
assert assertNotFinished();
if (docValues == null) {
throw new NullPointerException("DocValuesType must not be null");
@@ -410,8 +435,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
// number for this field. If the field was seen
// before then we'll get the same name and number,
// else we'll allocate a new one:
- final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues, dimensionCount, dimensionNumBytes);
- fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes);
+ final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues, dimensionCount, dimensionNumBytes, isSoftDeletesField);
+ fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes, isSoftDeletesField);
assert !byName.containsKey(fi.name);
globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
byName.put(fi.name, fi);
@@ -444,7 +469,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(),
fi.omitsNorms(), fi.hasPayloads(),
fi.getIndexOptions(), fi.getDocValuesType(), dvGen,
- fi.getPointDimensionCount(), fi.getPointNumBytes());
+ fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
}
public FieldInfo fieldInfo(String fieldName) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index ba30df0..144d2c0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -962,12 +962,12 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
* If this {@link SegmentInfos} has no global field number map the returned instance is empty
*/
private FieldNumbers getFieldNumberMap() throws IOException {
- final FieldNumbers map = new FieldNumbers();
+ final FieldNumbers map = new FieldNumbers(config.softDeletesField);
for(SegmentCommitInfo info : segmentInfos) {
FieldInfos fis = readFieldInfos(info);
for(FieldInfo fi : fis) {
- map.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes());
+ map.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
}
}
@@ -1789,7 +1789,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
if (globalFieldNumberMap.contains(f.name(), dvType) == false) {
// if this field doesn't exists we try to add it. if it exists and the DV type doesn't match we
// get a consistent error message as if you try to do that during an indexing operation.
- globalFieldNumberMap.addOrGet(f.name(), -1, dvType, 0, 0);
+ globalFieldNumberMap.addOrGet(f.name(), -1, dvType, 0, 0, f.name().equals(config.softDeletesField));
assert globalFieldNumberMap.contains(f.name(), dvType);
}
if (config.getIndexSortFields().contains(f.name())) {
@@ -2826,7 +2826,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
FieldInfos fis = readFieldInfos(info);
for(FieldInfo fi : fis) {
// This will throw exceptions if any of the incoming fields have an illegal schema change:
- globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes());
+ globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
}
infos.add(copySegmentAsIs(info, newSegName, context));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
index 1a7b15b..19078a8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.util.Bits;
@@ -263,7 +264,10 @@ public final class MultiFields extends Fields {
* will be unavailable.
*/
public static FieldInfos getMergedFieldInfos(IndexReader reader) {
- final FieldInfos.Builder builder = new FieldInfos.Builder();
+ final String softDeletesField = reader.leaves().stream()
+ .map(l -> l.reader().getFieldInfos().getSoftDeletesField())
+ .filter(Objects::nonNull).findAny().orElse(null);
+ final FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(softDeletesField));
for(final LeafReaderContext ctx : reader.leaves()) {
builder.add(ctx.reader().getFieldInfos());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
index 492b6e7..25f200a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
@@ -101,9 +102,11 @@ public class ParallelLeafReader extends LeafReader {
throw new IllegalArgumentException("All readers must have same maxDoc: "+maxDoc+"!="+reader.maxDoc());
}
}
-
+ final String softDeletesField = completeReaderSet.stream()
+ .map(r -> r.getFieldInfos().getSoftDeletesField())
+ .filter(Objects::nonNull).findAny().orElse(null);
// TODO: make this read-only in a cleaner way?
- FieldInfos.Builder builder = new FieldInfos.Builder();
+ FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(softDeletesField));
Sort indexSort = null;
int createdVersionMajor = -1;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
index 4625f73..9eb2541 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
@@ -210,7 +210,7 @@ public class TestCodecs extends LuceneTestCase {
terms[i] = new TermData(text, docs, null);
}
- final FieldInfos.Builder builder = new FieldInfos.Builder();
+ final FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));
final FieldData field = new FieldData("field", builder, terms, true, false);
final FieldData[] fields = new FieldData[] {field};
@@ -257,7 +257,7 @@ public class TestCodecs extends LuceneTestCase {
}
public void testRandomPostings() throws Throwable {
- final FieldInfos.Builder builder = new FieldInfos.Builder();
+ final FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));
final FieldData[] fields = new FieldData[NUM_FIELDS];
for(int i=0;i<NUM_FIELDS;i++) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
index 37761d3..b326258 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
@@ -222,7 +222,7 @@ public class TestDoc extends LuceneTestCase {
SegmentMerger merger = new SegmentMerger(Arrays.<CodecReader>asList(r1, r2),
si, InfoStream.getDefault(), trackingDir,
- new FieldInfos.FieldNumbers(), context);
+ new FieldInfos.FieldNumbers(null), context);
MergeState mergeState = merger.merge();
r1.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
index 48d69ec..ce24b7f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
@@ -44,7 +44,7 @@ public class TestFieldsReader extends LuceneTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
testDoc = new Document();
- fieldInfos = new FieldInfos.Builder();
+ fieldInfos = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));
DocHelper.setupDoc(testDoc);
for (IndexableField field : testDoc.getFields()) {
FieldInfo fieldInfo = fieldInfos.getOrAdd(field.name());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 6db7e86..98da021 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -3399,4 +3399,110 @@ public class TestIndexWriter extends LuceneTestCase {
IOUtils.close(reader, writer, dir);
}
+ public void testPreventChangingSoftDeletesField() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig().setSoftDeletesField("my_deletes"));
+ Document v1 = new Document();
+ v1.add(new StringField("id", "1", Field.Store.YES));
+ v1.add(new StringField("version", "1", Field.Store.YES));
+ writer.addDocument(v1);
+ Document v2 = new Document();
+ v2.add(new StringField("id", "1", Field.Store.YES));
+ v2.add(new StringField("version", "2", Field.Store.YES));
+ writer.softUpdateDocument(new Term("id", "1"), v2, new NumericDocValuesField("my_deletes", 1));
+ writer.commit();
+ writer.close();
+ for (SegmentCommitInfo si : SegmentInfos.readLatestCommit(dir)) {
+ FieldInfos fieldInfos = IndexWriter.readFieldInfos(si);
+ assertEquals("my_deletes", fieldInfos.getSoftDeletesField());
+ assertTrue(fieldInfos.fieldInfo("my_deletes").isSoftDeletesField());
+ }
+
+ IllegalArgumentException illegalError = expectThrows(IllegalArgumentException.class, () -> {
+ new IndexWriter(dir, newIndexWriterConfig().setSoftDeletesField("your_deletes"));
+ });
+ assertEquals("cannot configure [your_deletes] as soft-deletes; " +
+ "this index uses [my_deletes] as soft-deletes already", illegalError.getMessage());
+
+ IndexWriterConfig softDeleteConfig = newIndexWriterConfig().setSoftDeletesField("my_deletes")
+ .setMergePolicy(new SoftDeletesRetentionMergePolicy("my_deletes", () -> new MatchAllDocsQuery(), newMergePolicy()));
+ writer = new IndexWriter(dir, softDeleteConfig);
+ Document tombstone = new Document();
+ tombstone.add(new StringField("id", "tombstone", Field.Store.YES));
+ tombstone.add(new NumericDocValuesField("my_deletes", 1));
+ writer.addDocument(tombstone);
+ writer.flush();
+ for (SegmentCommitInfo si : writer.segmentInfos) {
+ FieldInfos fieldInfos = IndexWriter.readFieldInfos(si);
+ assertEquals("my_deletes", fieldInfos.getSoftDeletesField());
+ assertTrue(fieldInfos.fieldInfo("my_deletes").isSoftDeletesField());
+ }
+ writer.close();
+ // reopen writer without soft-deletes field should be prevented
+ IllegalArgumentException reopenError = expectThrows(IllegalArgumentException.class, () -> {
+ new IndexWriter(dir, newIndexWriterConfig());
+ });
+ assertEquals("this index has [my_deletes] as soft-deletes already" +
+ " but soft-deletes field is not configured in IWC", reopenError.getMessage());
+ dir.close();
+ }
+
+ public void testPreventAddingIndexesWithDifferentSoftDeletesField() throws Exception {
+ Directory dir1 = newDirectory();
+ IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig().setSoftDeletesField("soft_deletes_1"));
+ for (int i = 0; i < 2; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "1", Field.Store.YES));
+ d.add(new StringField("version", Integer.toString(i), Field.Store.YES));
+ w1.softUpdateDocument(new Term("id", "1"), d, new NumericDocValuesField("soft_deletes_1", 1));
+ }
+ w1.commit();
+ w1.close();
+
+ Directory dir2 = newDirectory();
+ IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig().setSoftDeletesField("soft_deletes_2"));
+ IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> w2.addIndexes(dir1));
+ assertEquals("cannot configure [soft_deletes_2] as soft-deletes; this index uses [soft_deletes_1] as soft-deletes already",
+ error.getMessage());
+ w2.close();
+
+ Directory dir3 = newDirectory();
+ IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField("soft_deletes_1");
+ IndexWriter w3 = new IndexWriter(dir3, config);
+ w3.addIndexes(dir1);
+ for (SegmentCommitInfo si : w3.segmentInfos) {
+ FieldInfo softDeleteField = IndexWriter.readFieldInfos(si).fieldInfo("soft_deletes_1");
+ assertTrue(softDeleteField.isSoftDeletesField());
+ }
+ w3.close();
+ IOUtils.close(dir1, dir2, dir3);
+ }
+
+ public void testNotAllowUsingExistingFieldAsSoftDeletes() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+ for (int i = 0; i < 2; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "1", Field.Store.YES));
+ if (random().nextBoolean()) {
+ d.add(new NumericDocValuesField("dv_field", 1));
+ w.updateDocument(new Term("id", "1"), d);
+ } else {
+ w.softUpdateDocument(new Term("id", "1"), d, new NumericDocValuesField("dv_field", 1));
+ }
+ }
+ w.commit();
+ w.close();
+ String softDeletesField = random().nextBoolean() ? "id" : "dv_field";
+ IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> {
+ IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField(softDeletesField);
+ new IndexWriter(dir, config);
+ });
+ assertEquals("cannot configure [" + softDeletesField + "] as soft-deletes;" +
+ " this index uses [" + softDeletesField + "] as non-soft-deletes already", error.getMessage());
+ IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField("non-existing-field");
+ w = new IndexWriter(dir, config);
+ w.close();
+ dir.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
index 5fadd3f..3047364 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
@@ -120,7 +120,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
deletes.onNewReader(segmentReader, commitInfo);
reader.close();
writer.close();
- FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 0, Collections.emptyMap(), 0, 0);
+ FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 0, Collections.emptyMap(), 0, 0, true);
List<Integer> docsDeleted = Arrays.asList(1, 3, 7, 8, DocIdSetIterator.NO_MORE_DOCS);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(docsDeleted, 10, true));
for (DocValuesFieldUpdates update : updates) {
@@ -140,7 +140,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
docsDeleted = Arrays.asList(1, 2, DocIdSetIterator.NO_MORE_DOCS);
updates = Arrays.asList(singleUpdate(docsDeleted, 10, true));
- fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 1, Collections.emptyMap(), 0, 0);
+ fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 1, Collections.emptyMap(), 0, 0, true);
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());
}
@@ -182,7 +182,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
SegmentCommitInfo segmentInfo = segmentReader.getSegmentInfo();
PendingDeletes deletes = newPendingDeletes(segmentInfo);
deletes.onNewReader(segmentReader, segmentInfo);
- FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0);
+ FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true);
List<Integer> docsDeleted = Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(docsDeleted, 3, true));
for (DocValuesFieldUpdates update : updates) {
@@ -228,7 +228,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
SegmentCommitInfo segmentInfo = segmentReader.getSegmentInfo();
PendingDeletes deletes = newPendingDeletes(segmentInfo);
deletes.onNewReader(segmentReader, segmentInfo);
- FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0);
+ FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(Arrays.asList(0, 1, DocIdSetIterator.NO_MORE_DOCS), 3, false));
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());
@@ -247,7 +247,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
assertEquals(0, deletes.numPendingDeletes());
segmentInfo.advanceDocValuesGen();
- fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0);
+ fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true);
updates = Arrays.asList(singleUpdate(Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS), 3, true));
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
index 6d0e04b..1171b90 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
@@ -88,7 +88,7 @@ public class TestSegmentMerger extends LuceneTestCase {
SegmentMerger merger = new SegmentMerger(Arrays.<CodecReader>asList(reader1, reader2),
si, InfoStream.getDefault(), mergedDir,
- new FieldInfos.FieldNumbers(),
+ new FieldInfos.FieldNumbers(null),
newIOContext(random(), new IOContext(new MergeInfo(-1, -1, false, -1))));
MergeState mergeState = merger.merge();
int docsMerged = mergeState.segmentInfo.maxDoc();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
index 144209d..1eef95f 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
@@ -81,7 +81,7 @@ public class TermVectorLeafReader extends LeafReader {
}
FieldInfo fieldInfo = new FieldInfo(field, 0,
true, true, terms.hasPayloads(),
- indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0);
+ indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, false);
fieldInfos = new FieldInfos(new FieldInfo[]{fieldInfo});
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index f732cf3..48e2bd2 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -500,7 +500,7 @@ public class MemoryIndex {
IndexOptions indexOptions = storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
return new FieldInfo(fieldName, ord, fieldType.storeTermVectors(), fieldType.omitNorms(), storePayloads,
indexOptions, fieldType.docValuesType(), -1, Collections.emptyMap(),
- fieldType.pointDimensionCount(), fieldType.pointNumBytes());
+ fieldType.pointDimensionCount(), fieldType.pointNumBytes(), false);
}
private void storePointValues(Info info, BytesRef pointValue) {
@@ -519,7 +519,7 @@ public class MemoryIndex {
info.fieldInfo = new FieldInfo(
info.fieldInfo.name, info.fieldInfo.number, info.fieldInfo.hasVectors(), info.fieldInfo.hasPayloads(),
info.fieldInfo.hasPayloads(), info.fieldInfo.getIndexOptions(), docValuesType, -1, info.fieldInfo.attributes(),
- info.fieldInfo.getPointDimensionCount(), info.fieldInfo.getPointNumBytes()
+ info.fieldInfo.getPointDimensionCount(), info.fieldInfo.getPointNumBytes(), info.fieldInfo.isSoftDeletesField()
);
} else if (existingDocValuesType != docValuesType) {
throw new IllegalArgumentException("Can't add [" + docValuesType + "] doc values field [" + fieldName + "], because [" + existingDocValuesType + "] doc values field already exists");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
index 9363ce6..3515b9a 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
@@ -53,7 +53,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
Directory dir = newDirectory();
Codec codec = getCodec();
SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
- FieldInfos.Builder builder = new FieldInfos.Builder();
+ FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));
FieldInfo fi = builder.getOrAdd("field");
fi.setIndexOptions(TextField.TYPE_STORED.indexOptions());
addAttributes(fi);
@@ -75,7 +75,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
Directory dir = newDirectory();
Codec codec = getCodec();
SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
- FieldInfos.Builder builder = new FieldInfos.Builder();
+ FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));
FieldInfo fi = builder.getOrAdd("field");
fi.setIndexOptions(TextField.TYPE_STORED.indexOptions());
addAttributes(fi);
@@ -115,7 +115,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
dir.failOn(fail);
Codec codec = getCodec();
SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
- FieldInfos.Builder builder = new FieldInfos.Builder();
+ FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));
FieldInfo fi = builder.getOrAdd("field");
fi.setIndexOptions(TextField.TYPE_STORED.indexOptions());
addAttributes(fi);
@@ -150,7 +150,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
dir.failOn(fail);
Codec codec = getCodec();
SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
- FieldInfos.Builder builder = new FieldInfos.Builder();
+ FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));
FieldInfo fi = builder.getOrAdd("field");
fi.setIndexOptions(TextField.TYPE_STORED.indexOptions());
addAttributes(fi);
@@ -185,7 +185,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
dir.failOn(fail);
Codec codec = getCodec();
SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
- FieldInfos.Builder builder = new FieldInfos.Builder();
+ FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));
FieldInfo fi = builder.getOrAdd("field");
fi.setIndexOptions(TextField.TYPE_STORED.indexOptions());
addAttributes(fi);
@@ -221,7 +221,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
dir.failOn(fail);
Codec codec = getCodec();
SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
- FieldInfos.Builder builder = new FieldInfos.Builder();
+ FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));
FieldInfo fi = builder.getOrAdd("field");
fi.setIndexOptions(TextField.TYPE_STORED.indexOptions());
addAttributes(fi);
@@ -251,7 +251,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
for (int i = 0; i < numFields; i++) {
fieldNames.add(TestUtil.randomUnicodeString(random()));
}
- FieldInfos.Builder builder = new FieldInfos.Builder();
+ FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));
for (String field : fieldNames) {
IndexableFieldType fieldType = randomFieldType(random());
FieldInfo fi = builder.getOrAdd(field);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
index ab92946..f449644 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
@@ -323,7 +323,7 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
FieldInfo proto = oneDocReader.getFieldInfos().fieldInfo("field");
FieldInfo field = new FieldInfo(proto.name, proto.number, proto.hasVectors(), proto.omitsNorms(), proto.hasPayloads(),
proto.getIndexOptions(), proto.getDocValuesType(), proto.getDocValuesGen(), new HashMap<>(),
- proto.getPointDimensionCount(), proto.getPointNumBytes());
+ proto.getPointDimensionCount(), proto.getPointNumBytes(), proto.isSoftDeletesField());
FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { field } );
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
index 7dd6ba8..2c74677 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
@@ -77,7 +77,8 @@ public class MismatchedLeafReader extends FilterLeafReader {
oldInfo.getDocValuesGen(), // dvGen
oldInfo.attributes(), // attributes
oldInfo.getPointDimensionCount(), // dimension count
- oldInfo.getPointNumBytes()); // dimension numBytes
+ oldInfo.getPointNumBytes(), // dimension numBytes
+ oldInfo.isSoftDeletesField()); // used as soft-deletes field
shuffled.set(i, newInfo);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
index d5eb105..0afab1c 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
@@ -122,7 +122,7 @@ public class RandomPostingsTester {
fieldInfoArray[fieldUpto] = new FieldInfo(field, fieldUpto, false, false, true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
DocValuesType.NONE, -1, new HashMap<>(),
- 0, 0);
+ 0, 0, false);
fieldUpto++;
SortedMap<BytesRef,SeedAndOrd> postings = new TreeMap<>();
@@ -638,7 +638,7 @@ public class RandomPostingsTester {
DocValuesType.NONE,
-1,
new HashMap<>(),
- 0, 0);
+ 0, 0, false);
}
FieldInfos newFieldInfos = new FieldInfos(newFieldInfoArray);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
index b3f792f..1dd5818 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
@@ -796,7 +796,8 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
fieldInfo.getDocValuesGen(),
fieldInfo.attributes(),
fieldInfo.getPointDimensionCount(),
- fieldInfo.getPointNumBytes());
+ fieldInfo.getPointNumBytes(),
+ fieldInfo.isSoftDeletesField());
newInfos.add(f);
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index 559ee8b..07bd2be 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -424,7 +424,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
DocValuesType.NONE,
fieldInfo.getDocValuesGen(),
fieldInfo.attributes(),
- 0, 0);
+ 0, 0, fieldInfo.isSoftDeletesField());
newInfos.add(f);
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/solr/core/src/java/org/apache/solr/search/Insanity.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/Insanity.java b/solr/core/src/java/org/apache/solr/search/Insanity.java
index aa36652..8fe081f 100644
--- a/solr/core/src/java/org/apache/solr/search/Insanity.java
+++ b/solr/core/src/java/org/apache/solr/search/Insanity.java
@@ -66,7 +66,7 @@ public class Insanity {
if (fi.name.equals(insaneField)) {
filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(),
fi.hasPayloads(), fi.getIndexOptions(), DocValuesType.NONE, -1, Collections.emptyMap(),
- fi.getPointDimensionCount(), fi.getPointNumBytes()));
+ fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
} else {
filteredInfos.add(fi);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67b6593e/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
index 967db54..9f0f527 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
@@ -282,7 +282,7 @@ public class UninvertingReader extends FilterLeafReader {
}
filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(),
fi.hasPayloads(), fi.getIndexOptions(), type, fi.getDocValuesGen(), fi.attributes(),
- fi.getPointDimensionCount(), fi.getPointNumBytes()));
+ fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
}
fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
}