You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2020/04/18 12:25:07 UTC
[lucene-solr] branch master updated: LUCENE-9324: Add an ID to
SegmentCommitInfo (#1434)
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 113043b LUCENE-9324: Add an ID to SegmentCommitInfo (#1434)
113043b is described below
commit 113043b1ed2ac95de17f6bdd203f6050ff6ca1f7
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Sat Apr 18 14:24:57 2020 +0200
LUCENE-9324: Add an ID to SegmentCommitInfo (#1434)
We already have IDs in SegmentInfo, as well as on SegmentInfos which are useful to uniquely identify segments and entire commits. Having IDs on SegmentCommitInfo is be useful too in
order to compare commits for equality and make snapshots incremental on generational files.
This change adds a unique ID to SegmentCommitInfo starting from Lucene 8.6. Older segments won't have an ID until the segment receives an update or a delete even if they have been opened and / or committed by Lucene 8.6 or above.
---
lucene/CHANGES.txt | 3 +
.../lucene/index/TestBackwardsCompatibility.java | 26 ++++++-
.../lucene/index/DocumentsWriterPerThread.java | 2 +-
.../java/org/apache/lucene/index/IndexWriter.java | 6 +-
.../org/apache/lucene/index/SegmentCommitInfo.java | 40 +++++++++--
.../java/org/apache/lucene/index/SegmentInfos.java | 38 ++++++++--
.../src/java/org/apache/lucene/util/Version.java | 7 ++
.../src/test/org/apache/lucene/index/TestDoc.java | 2 +-
.../org/apache/lucene/index/TestIndexWriter.java | 80 ++++++++++++++++++++++
.../index/TestIndexWriterThreadsToSegments.java | 2 +-
.../index/TestOneMergeWrappingMergePolicy.java | 2 +-
.../apache/lucene/index/TestPendingDeletes.java | 6 +-
.../lucene/index/TestPendingSoftDeletes.java | 2 +-
.../org/apache/lucene/index/TestSegmentInfos.java | 39 ++++++++++-
.../org/apache/lucene/index/TestSegmentMerger.java | 2 +-
.../apache/lucene/luke/models/util/IndexUtils.java | 6 +-
.../luke/models/overview/OverviewImplTest.java | 2 +-
.../org/apache/lucene/index/IndexSplitter.java | 2 +-
.../apache/lucene/replicator/nrt/PrimaryNode.java | 2 +-
.../lucene/index/BaseLiveDocsFormatTestCase.java | 4 +-
.../lucene/index/BaseMergePolicyTestCase.java | 6 +-
21 files changed, 240 insertions(+), 39 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 6cf4eb4..8171b3b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -143,6 +143,9 @@ Improvements
* LUCENE-9304: Removed ThreadState abstraction from DocumentsWriter which allows pooling of DWPT directly and
improves the approachability of the IndexWriter code. (Simon Willnauer)
+* LUCENE-9324: Add an ID to SegmentCommitInfo in order to compare commits for equality and make
+ snapshots incremental on generational files. (Simon Willnauer, Mike Mccandless, Adrien Grant)
+
Optimizations
---------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 26d7ee0..245cef1 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -768,7 +768,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
writer.close();
}
}
- writer = null;
}
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
@@ -830,8 +829,12 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));
w.addIndexes(oldDir);
w.close();
- targetDir.close();
+ SegmentInfos si = SegmentInfos.readLatestCommit(targetDir);
+ assertNull("none of the segments should have been upgraded",
+ si.asList().stream().filter( // depending on the MergePolicy we might see these segments merged away
+ sci -> sci.getId() != null && sci.info.getVersion().onOrAfter(Version.LUCENE_8_6_0) == false
+ ).findAny().orElse(null));
if (VERBOSE) {
System.out.println("\nTEST: done adding indices; now close");
}
@@ -862,7 +865,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
TestUtil.addIndexesSlowly(w, reader);
w.close();
reader.close();
-
+ SegmentInfos si = SegmentInfos.readLatestCommit(targetDir);
+ assertNull("all SCIs should have an id now",
+ si.asList().stream().filter(sci -> sci.getId() == null).findAny().orElse(null));
targetDir.close();
}
}
@@ -1367,6 +1372,20 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
}
+ public void testSegmentCommitInfoId() throws IOException {
+ for (String name : oldNames) {
+ Directory dir = oldIndexDirs.get(name);
+ SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
+ for (SegmentCommitInfo info : infos) {
+ if (info.info.getVersion().onOrAfter(Version.LUCENE_8_6_0)) {
+ assertNotNull(info.toString(), info.getId());
+ } else {
+ assertNull(info.toString(), info.getId());
+ }
+ }
+ }
+ }
+
public void verifyUsesDefaultCodec(Directory dir, String name) throws Exception {
DirectoryReader r = DirectoryReader.open(dir);
for (LeafReaderContext context : r.leaves()) {
@@ -1392,6 +1411,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
for (SegmentCommitInfo si : infos) {
assertEquals(Version.LATEST, si.info.getVersion());
+ assertNotNull(si.getId());
}
assertEquals(Version.LATEST, infos.getCommitLuceneVersion());
assertEquals(indexCreatedVersion, infos.getIndexCreatedVersionMajor());
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
index fdcd7ee..1286c76 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
@@ -419,7 +419,7 @@ final class DocumentsWriterPerThread {
pendingUpdates.clearDeleteTerms();
segmentInfo.setFiles(new HashSet<>(directory.getCreatedFiles()));
- final SegmentCommitInfo segmentInfoPerCommit = new SegmentCommitInfo(segmentInfo, 0, flushState.softDelCountOnFlush, -1L, -1L, -1L);
+ final SegmentCommitInfo segmentInfoPerCommit = new SegmentCommitInfo(segmentInfo, 0, flushState.softDelCountOnFlush, -1L, -1L, -1L, StringHelper.randomId());
if (infoStream.isEnabled("DWPT")) {
infoStream.message("DWPT", "new segment has " + (flushState.liveDocs == null ? 0 : flushState.delCountOnFlush) + " deleted docs");
infoStream.message("DWPT", "new segment has " + flushState.softDelCountOnFlush + " soft-deleted docs");
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 99ee5cb..64fe468 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -3003,7 +3003,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
notifyAll();
}
}
- SegmentCommitInfo infoPerCommit = new SegmentCommitInfo(info, 0, numSoftDeleted, -1L, -1L, -1L);
+ SegmentCommitInfo infoPerCommit = new SegmentCommitInfo(info, 0, numSoftDeleted, -1L, -1L, -1L, StringHelper.randomId());
info.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));
trackingDir.clearCreatedFiles();
@@ -3081,7 +3081,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
info.info.getUseCompoundFile(), info.info.getCodec(),
info.info.getDiagnostics(), info.info.getId(), info.info.getAttributes(), info.info.getIndexSort());
SegmentCommitInfo newInfoPerCommit = new SegmentCommitInfo(newInfo, info.getDelCount(), info.getSoftDelCount(), info.getDelGen(),
- info.getFieldInfosGen(), info.getDocValuesGen());
+ info.getFieldInfosGen(), info.getDocValuesGen(), info.getId());
newInfo.setFiles(info.info.files());
newInfoPerCommit.setFieldInfosFiles(info.getFieldInfosFiles());
@@ -4273,7 +4273,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
details.put("mergeMaxNumSegments", "" + merge.maxNumSegments);
details.put("mergeFactor", Integer.toString(merge.segments.size()));
setDiagnostics(si, SOURCE_MERGE, details);
- merge.setMergeInfo(new SegmentCommitInfo(si, 0, 0, -1L, -1L, -1L));
+ merge.setMergeInfo(new SegmentCommitInfo(si, 0, 0, -1L, -1L, -1L, StringHelper.randomId()));
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "merge seg=" + merge.info.info.name + " " + segString(merge.segments));
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
index 6911757..954894b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -26,6 +27,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import org.apache.lucene.util.StringHelper;
+
/** Embeds a [read-only] SegmentInfo and adds per-commit
* fields.
*
@@ -35,6 +38,9 @@ public class SegmentCommitInfo {
/** The {@link SegmentInfo} that we wrap. */
public final SegmentInfo info;
+ /** Id that uniquely identifies this segment commit. */
+ private byte[] id;
+
// How many deleted docs in the segment:
private int delCount;
@@ -79,7 +85,6 @@ public class SegmentCommitInfo {
/**
* Sole constructor.
- *
* @param info
* {@link SegmentInfo} that we wrap
* @param delCount
@@ -90,8 +95,9 @@ public class SegmentCommitInfo {
* FieldInfos generation number (used to name field-infos files)
* @param docValuesGen
* DocValues generation number (used to name doc-values updates files)
+ * @param id Id that uniquely identifies this segment commit. This id must be 16 bytes long. See {@link StringHelper#randomId()}
*/
- public SegmentCommitInfo(SegmentInfo info, int delCount, int softDelCount, long delGen, long fieldInfosGen, long docValuesGen) {
+ public SegmentCommitInfo(SegmentInfo info, int delCount, int softDelCount, long delGen, long fieldInfosGen, long docValuesGen, byte[] id) {
this.info = info;
this.delCount = delCount;
this.softDelCount = softDelCount;
@@ -101,6 +107,10 @@ public class SegmentCommitInfo {
this.nextWriteFieldInfosGen = fieldInfosGen == -1 ? 1 : fieldInfosGen + 1;
this.docValuesGen = docValuesGen;
this.nextWriteDocValuesGen = docValuesGen == -1 ? 1 : docValuesGen + 1;
+ this.id = id;
+ if (id != null && id.length != StringHelper.ID_LENGTH) {
+ throw new IllegalArgumentException("invalid id: " + Arrays.toString(id));
+ }
}
/** Returns the per-field DocValues updates files. */
@@ -138,7 +148,7 @@ public class SegmentCommitInfo {
void advanceDelGen() {
delGen = nextWriteDelGen;
nextWriteDelGen = delGen+1;
- sizeInBytes = -1;
+ generationAdvanced();
}
/** Called if there was an exception while writing
@@ -162,7 +172,7 @@ public class SegmentCommitInfo {
void advanceFieldInfosGen() {
fieldInfosGen = nextWriteFieldInfosGen;
nextWriteFieldInfosGen = fieldInfosGen + 1;
- sizeInBytes = -1;
+ generationAdvanced();
}
/**
@@ -187,7 +197,7 @@ public class SegmentCommitInfo {
void advanceDocValuesGen() {
docValuesGen = nextWriteDocValuesGen;
nextWriteDocValuesGen = docValuesGen + 1;
- sizeInBytes = -1;
+ generationAdvanced();
}
/**
@@ -251,7 +261,7 @@ public class SegmentCommitInfo {
void setBufferedDeletesGen(long v) {
if (bufferedDeletesGen == -1) {
bufferedDeletesGen = v;
- sizeInBytes = -1;
+ generationAdvanced();
} else {
throw new IllegalStateException("buffered deletes gen should only be set once");
}
@@ -355,6 +365,9 @@ public class SegmentCommitInfo {
if (softDelCount > 0) {
s += " :softDel=" + softDelCount;
}
+ if (this.id != null) {
+ s += " :id=" + StringHelper.idToString(id);
+ }
return s;
}
@@ -366,7 +379,7 @@ public class SegmentCommitInfo {
@Override
public SegmentCommitInfo clone() {
- SegmentCommitInfo other = new SegmentCommitInfo(info, delCount, softDelCount, delGen, fieldInfosGen, docValuesGen);
+ SegmentCommitInfo other = new SegmentCommitInfo(info, delCount, softDelCount, delGen, fieldInfosGen, docValuesGen, getId());
// Not clear that we need to carry over nextWriteDelGen
// (i.e. do we ever clone after a failed write and
// before the next successful write?), but just do it to
@@ -388,4 +401,17 @@ public class SegmentCommitInfo {
final int getDelCount(boolean includeSoftDeletes) {
return includeSoftDeletes ? getDelCount() + getSoftDelCount() : getDelCount();
}
+
+ private void generationAdvanced() {
+ sizeInBytes = -1;
+ id = StringHelper.randomId();
+ }
+
+ /**
+ * Returns and Id that uniquely identifies this segment commit or <code>null</code> if there is no ID assigned.
+ * This ID changes each time the the segment changes due to a delete, doc-value or field update.
+ */
+ public byte[] getId() {
+ return id == null ? null : id.clone();
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
index 116c2e1..f9edccd 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
@@ -124,7 +124,9 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
public static final int VERSION_72 = 8;
/** The version that recorded softDelCount */
public static final int VERSION_74 = 9;
- static final int VERSION_CURRENT = VERSION_74;
+ /** The version that recorded SegmentCommitInfo IDs */
+ public static final int VERSION_86 = 10;
+ static final int VERSION_CURRENT = VERSION_86;
/** Name of the generation reference file name */
private static final String OLD_SEGMENTS_GEN = "segments.gen";
@@ -374,7 +376,24 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
if (softDelCount + delCount > info.maxDoc()) {
throw new CorruptIndexException("invalid deletion count: " + softDelCount + delCount + " vs maxDoc=" + info.maxDoc(), input);
}
- SegmentCommitInfo siPerCommit = new SegmentCommitInfo(info, delCount, softDelCount, delGen, fieldInfosGen, dvGen);
+ final byte[] sciId;
+ if (format > VERSION_74) {
+ byte marker = input.readByte();
+ switch (marker) {
+ case 1:
+ sciId = new byte[StringHelper.ID_LENGTH];
+ input.readBytes(sciId, 0, sciId.length);
+ break;
+ case 0:
+ sciId = null;
+ break;
+ default:
+ throw new CorruptIndexException("invalid SegmentCommitInfo ID marker: " + marker, input);
+ }
+ } else {
+ sciId = null;
+ }
+ SegmentCommitInfo siPerCommit = new SegmentCommitInfo(info, delCount, softDelCount, delGen, fieldInfosGen, dvGen, sciId);
siPerCommit.setFieldInfosFiles(input.readSetOfStrings());
final Map<Integer,Set<String>> dvUpdateFiles;
final int numDVFields = input.readInt();
@@ -460,7 +479,7 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
try {
segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
- write(directory, segnOutput);
+ write(segnOutput);
segnOutput.close();
directory.sync(Collections.singleton(segmentFileName));
success = true;
@@ -479,7 +498,7 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
}
/** Write ourselves to the provided {@link IndexOutput} */
- public void write(Directory directory, IndexOutput out) throws IOException {
+ public void write(IndexOutput out) throws IOException {
CodecUtil.writeIndexHeader(out, "segments", VERSION_CURRENT,
StringHelper.randomId(), Long.toString(generation, Character.MAX_RADIX));
out.writeVInt(Version.LATEST.major);
@@ -537,6 +556,17 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
throw new IllegalStateException("cannot write segment: invalid maxDoc segment=" + si.name + " maxDoc=" + si.maxDoc() + " softDelCount=" + softDelCount);
}
out.writeInt(softDelCount);
+ // we ensure that there is a valid ID for this SCI just in case
+ // this is manually upgraded outside of IW
+ byte[] sciId = siPerCommit.getId();
+ if (sciId != null) {
+ out.writeByte((byte)1);
+ assert sciId.length == StringHelper.ID_LENGTH : "invalid SegmentCommitInfo#id: " + Arrays.toString(sciId);
+ out.writeBytes(sciId, 0, sciId.length);
+ } else {
+ out.writeByte((byte)0);
+ }
+
out.writeSetOfStrings(siPerCommit.getFieldInfosFiles());
final Map<Integer,Set<String>> dvUpdatesFiles = siPerCommit.getDocValuesUpdatesFiles();
out.writeInt(dvUpdatesFiles.size());
diff --git a/lucene/core/src/java/org/apache/lucene/util/Version.java b/lucene/core/src/java/org/apache/lucene/util/Version.java
index aa1f348..5ed1a95 100644
--- a/lucene/core/src/java/org/apache/lucene/util/Version.java
+++ b/lucene/core/src/java/org/apache/lucene/util/Version.java
@@ -103,6 +103,13 @@ public final class Version {
public static final Version LUCENE_8_5_1 = new Version(8, 5, 1);
/**
+ * Match settings and bugs in Lucene's 8.6.0 release.
+ * @deprecated Use latest
+ */
+ @Deprecated
+ public static final Version LUCENE_8_6_0 = new Version(8, 6, 0);
+
+ /**
* Match settings and bugs in Lucene's 9.0.0 release.
* <p>
* Use this to get the latest & greatest settings, bug
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
index d7eea7a..f776070 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
@@ -238,7 +238,7 @@ public class TestDoc extends LuceneTestCase {
}
}
- return new SegmentCommitInfo(si, 0, 0, -1L, -1L, -1L);
+ return new SegmentCommitInfo(si, 0, 0, -1L, -1L, -1L, StringHelper.randomId());
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 42a85d0..cb02f794 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -2484,8 +2484,11 @@ public class TestIndexWriter extends LuceneTestCase {
assertEquals(StringHelper.ID_LENGTH, id1.length);
byte[] id2 = sis.info(0).info.getId();
+ byte[] sciId2 = sis.info(0).getId();
assertNotNull(id2);
+ assertNotNull(sciId2);
assertEquals(StringHelper.ID_LENGTH, id2.length);
+ assertEquals(StringHelper.ID_LENGTH, sciId2.length);
// Make sure CheckIndex includes id output:
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
@@ -4085,4 +4088,81 @@ public class TestIndexWriter extends LuceneTestCase {
assertEquals(maxCompletedSequenceNumber+2, writer.getMaxCompletedSequenceNumber());
}
}
+
+ public void testSegmentCommitInfoId() throws IOException {
+ try (Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir,
+ new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("num", 1));
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("num", 1));
+ doc.add(new StringField("id", "2", Field.Store.NO));
+ writer.addDocument(doc);
+ writer.commit();
+ SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(dir);
+ byte[] id = segmentCommitInfos.info(0).getId();
+ byte[] segInfoId = segmentCommitInfos.info(0).info.getId();
+
+ writer.updateNumericDocValue(new Term("id", "1"), "num", 2);
+ writer.commit();
+ segmentCommitInfos = SegmentInfos.readLatestCommit(dir);
+ assertEquals(1, segmentCommitInfos.size());
+ assertNotEquals(StringHelper.idToString(id), StringHelper.idToString(segmentCommitInfos.info(0).getId()));
+ assertEquals(StringHelper.idToString(segInfoId), StringHelper.idToString(segmentCommitInfos.info(0).info.getId()));
+ id = segmentCommitInfos.info(0).getId();
+ writer.addDocument(new Document()); // second segment
+ writer.commit();
+ segmentCommitInfos = SegmentInfos.readLatestCommit(dir);
+ assertEquals(2, segmentCommitInfos.size());
+ assertEquals(StringHelper.idToString(id), StringHelper.idToString(segmentCommitInfos.info(0).getId()));
+ assertEquals(StringHelper.idToString(segInfoId), StringHelper.idToString(segmentCommitInfos.info(0).info.getId()));
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("num", 5));
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ writer.updateDocument(new Term("id", "1"), doc);
+ writer.commit();
+ segmentCommitInfos = SegmentInfos.readLatestCommit(dir);
+ assertEquals(3, segmentCommitInfos.size());
+ assertNotEquals(StringHelper.idToString(id), StringHelper.idToString(segmentCommitInfos.info(0).getId()));
+ assertEquals(StringHelper.idToString(segInfoId), StringHelper.idToString(segmentCommitInfos.info(0).info.getId()));
+ writer.close();
+ try (Directory dir2 = newDirectory();
+ IndexWriter writer2 = new IndexWriter(dir2,
+ new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))) {
+ writer2.addIndexes(dir);
+ writer2.commit();
+ SegmentInfos infos2 = SegmentInfos.readLatestCommit(dir2);
+ assertEquals(infos2.size(), segmentCommitInfos.size());
+ for (int i = 0; i < infos2.size(); i++) {
+ assertEquals(StringHelper.idToString(infos2.info(i).getId()), StringHelper.idToString(segmentCommitInfos.info(i).getId()));
+ assertEquals(StringHelper.idToString(infos2.info(i).info.getId()), StringHelper.idToString(segmentCommitInfos.info(i).info.getId()));
+ }
+ }
+ }
+
+ Set<String> ids = new HashSet<>();
+ for (int i = 0; i < 2; i++) {
+ try (Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir,
+ new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("num", 1));
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ writer.addDocument(doc);
+ writer.commit();
+ SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(dir);
+ String id = StringHelper.idToString(segmentCommitInfos.info(0).getId());
+ assertTrue(ids.add(id));
+ writer.updateNumericDocValue(new Term("id", "1"), "num", 2);
+ writer.commit();
+ segmentCommitInfos = SegmentInfos.readLatestCommit(dir);
+ id = StringHelper.idToString(segmentCommitInfos.info(0).getId());
+ assertTrue(ids.add(id));
+ }
+ }
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
index 347cb7d..b14a887 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
@@ -332,7 +332,7 @@ public class TestIndexWriterThreadsToSegments extends LuceneTestCase {
byte id[] = readSegmentInfoID(dir, fileName);
SegmentInfo si = TestUtil.getDefaultCodec().segmentInfoFormat().read(dir, segName, id, IOContext.DEFAULT);
si.setCodec(codec);
- SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, 0, -1, -1, -1);
+ SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, 0, -1, -1, -1, StringHelper.randomId());
SegmentReader sr = new SegmentReader(sci, Version.LATEST.major, IOContext.DEFAULT);
try {
thread0Count += sr.docFreq(new Term("field", "threadID0"));
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOneMergeWrappingMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestOneMergeWrappingMergePolicy.java
index e240f54..e0206cc 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestOneMergeWrappingMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestOneMergeWrappingMergePolicy.java
@@ -137,7 +137,7 @@ public class TestOneMergeWrappingMergePolicy extends LuceneTestCase {
Collections.emptyMap(), // attributes
null /* indexSort */);
final List<SegmentCommitInfo> segments = new LinkedList<SegmentCommitInfo>();
- segments.add(new SegmentCommitInfo(si, 0, 0, 0, 0, 0));
+ segments.add(new SegmentCommitInfo(si, 0, 0, 0, 0, 0, StringHelper.randomId()));
ms.add(new MergePolicy.OneMerge(segments));
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java
index 143f671..841ebe1 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java
@@ -41,7 +41,7 @@ public class TestPendingDeletes extends LuceneTestCase {
Directory dir = new ByteBuffersDirectory();
SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 10, false, Codec.getDefault(),
Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
- SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, -1, -1, -1);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, -1, -1, -1, StringHelper.randomId());
PendingDeletes deletes = newPendingDeletes(commitInfo);
assertNull(deletes.getLiveDocs());
int docToDelete = TestUtil.nextInt(random(), 0, 7);
@@ -75,7 +75,7 @@ public class TestPendingDeletes extends LuceneTestCase {
Directory dir = new ByteBuffersDirectory();
SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 6, false, Codec.getDefault(),
Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
- SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, -1, -1, -1);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, -1, -1, -1, StringHelper.randomId());
PendingDeletes deletes = newPendingDeletes(commitInfo);
assertFalse(deletes.writeLiveDocs(dir));
assertEquals(0, dir.listAll().length);
@@ -132,7 +132,7 @@ public class TestPendingDeletes extends LuceneTestCase {
Directory dir = new ByteBuffersDirectory();
SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 3, false, Codec.getDefault(),
Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
- SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, -1, -1, -1);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, -1, -1, -1, StringHelper.randomId());
FieldInfos fieldInfos = FieldInfos.EMPTY;
si.getCodec().fieldInfosFormat().write(dir, si, "", fieldInfos, IOContext.DEFAULT);
PendingDeletes deletes = newPendingDeletes(commitInfo);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
index 666b3c4..a7c6811 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
@@ -150,7 +150,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
Directory dir = new ByteBuffersDirectory();
SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 10, false, Codec.getDefault(),
Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
- SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, -1, -1, -1);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, -1, -1, -1, StringHelper.randomId());
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
for (int i = 0; i < si.maxDoc(); i++) {
writer.addDocument(new Document());
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
index f5029b8..19d8214 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
@@ -64,7 +64,7 @@ public class TestSegmentInfos extends LuceneTestCase {
Collections.<String,String>emptyMap(), id, Collections.<String,String>emptyMap(), null);
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
- SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1, StringHelper.randomId());
sis.add(commitInfo);
sis.commit(dir);
@@ -86,20 +86,24 @@ public class TestSegmentInfos extends LuceneTestCase {
Collections.<String,String>emptyMap(), id, Collections.<String,String>emptyMap(), null);
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
- SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1, StringHelper.randomId());
sis.add(commitInfo);
info = new SegmentInfo(dir, Version.LUCENE_9_0_0, Version.LUCENE_9_0_0, "_1", 1, false, Codec.getDefault(),
Collections.<String,String>emptyMap(), id, Collections.<String,String>emptyMap(), null);
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
- commitInfo = new SegmentCommitInfo(info, 0, 0,-1, -1, -1);
+ commitInfo = new SegmentCommitInfo(info, 0, 0,-1, -1, -1, StringHelper.randomId());
sis.add(commitInfo);
sis.commit(dir);
+ byte[] commitInfoId0 = sis.info(0).getId();
+ byte[] commitInfoId1 = sis.info(1).getId();
sis = SegmentInfos.readLatestCommit(dir);
assertEquals(Version.LUCENE_9_0_0, sis.getMinSegmentLuceneVersion());
assertEquals(Version.LATEST, sis.getCommitLuceneVersion());
+ assertEquals(StringHelper.idToString(commitInfoId0), StringHelper.idToString(sis.info(0).getId()));
+ assertEquals(StringHelper.idToString(commitInfoId1), StringHelper.idToString(sis.info(1).getId()));
dir.close();
}
@@ -145,5 +149,34 @@ public class TestSegmentInfos extends LuceneTestCase {
dir.close();
}
+
+ public void testIDChangesOnAdvance() throws IOException {
+ try (BaseDirectoryWrapper dir = newDirectory()) {
+ dir.setCheckIndexOnClose(false);
+ byte id[] = StringHelper.randomId();
+ SegmentInfo info = new SegmentInfo(dir, Version.LUCENE_9_0_0, Version.LUCENE_9_0_0, "_0", 1, false, Codec.getDefault(),
+ Collections.<String, String>emptyMap(), StringHelper.randomId(), Collections.<String, String>emptyMap(), null);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1, id);
+ assertEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
+ commitInfo.advanceDelGen();
+ assertNotEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
+
+ id = commitInfo.getId();
+ commitInfo.advanceDocValuesGen();
+ assertNotEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
+
+ id = commitInfo.getId();
+ commitInfo.advanceFieldInfosGen();
+ assertNotEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
+ SegmentCommitInfo clone = commitInfo.clone();
+ id = commitInfo.getId();
+ assertEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
+ assertEquals(StringHelper.idToString(id), StringHelper.idToString(clone.getId()));
+
+ commitInfo.advanceFieldInfosGen();
+ assertNotEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
+ assertEquals("clone changed but shouldn't", StringHelper.idToString(id), StringHelper.idToString(clone.getId()));
+ }
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
index 34f62c6..04f357e 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
@@ -96,7 +96,7 @@ public class TestSegmentMerger extends LuceneTestCase {
//Should be able to open a new SegmentReader against the new directory
SegmentReader mergedReader = new SegmentReader(new SegmentCommitInfo(
mergeState.segmentInfo,
- 0, 0, -1L, -1L, -1L),
+ 0, 0, -1L, -1L, -1L, StringHelper.randomId()),
Version.LATEST.major,
newIOContext(random()));
assertTrue(mergedReader != null);
diff --git a/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java b/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java
index e59689a..71e8070 100644
--- a/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java
+++ b/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java
@@ -303,8 +303,10 @@ public final class IndexUtils {
format = "Lucene 7.2 or later";
} else if (actualVersion == SegmentInfos.VERSION_74) {
format = "Lucene 7.4 or later";
- } else if (actualVersion > SegmentInfos.VERSION_74) {
- format = "Lucene 7.4 or later (UNSUPPORTED)";
+ } else if (actualVersion == SegmentInfos.VERSION_86) {
+ format = "Lucene 8.6 or later";
+ } else if (actualVersion > SegmentInfos.VERSION_86) {
+ format = "Lucene 8.6 or later (UNSUPPORTED)";
}
} else {
format = "Lucene 6.x or prior (UNSUPPORTED)";
diff --git a/lucene/luke/src/test/org/apache/lucene/luke/models/overview/OverviewImplTest.java b/lucene/luke/src/test/org/apache/lucene/luke/models/overview/OverviewImplTest.java
index 6e4522b..5eb15ef9 100644
--- a/lucene/luke/src/test/org/apache/lucene/luke/models/overview/OverviewImplTest.java
+++ b/lucene/luke/src/test/org/apache/lucene/luke/models/overview/OverviewImplTest.java
@@ -87,7 +87,7 @@ public class OverviewImplTest extends OverviewTestBase {
@Test
public void testGetIndexFormat() {
OverviewImpl overview = new OverviewImpl(reader, indexDir.toString());
- assertEquals("Lucene 7.4 or later", overview.getIndexFormat().get());
+ assertEquals("Lucene 8.6 or later", overview.getIndexFormat().get());
}
@Test
diff --git a/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java b/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java
index afe45cc..efc6ba3 100644
--- a/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java
+++ b/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java
@@ -143,7 +143,7 @@ public class IndexSplitter {
info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics(), info.getId(), Collections.emptyMap(), null);
destInfos.add(new SegmentCommitInfo(newInfo, infoPerCommit.getDelCount(), infoPerCommit.getSoftDelCount(),
infoPerCommit.getDelGen(), infoPerCommit.getFieldInfosGen(),
- infoPerCommit.getDocValuesGen()));
+ infoPerCommit.getDocValuesGen(), infoPerCommit.getId()));
// now copy files over
Collection<String> files = infoPerCommit.files();
for (final String srcName : files) {
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/PrimaryNode.java b/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/PrimaryNode.java
index f96f6d2..2d24f9b 100644
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/PrimaryNode.java
+++ b/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/PrimaryNode.java
@@ -245,7 +245,7 @@ public abstract class PrimaryNode extends Node {
// Serialize the SegmentInfos.
ByteBuffersDataOutput buffer = new ByteBuffersDataOutput();
try (ByteBuffersIndexOutput tmpIndexOutput = new ByteBuffersIndexOutput(buffer, "temporary", "temporary")) {
- infos.write(dir, tmpIndexOutput);
+ infos.write(tmpIndexOutput);
}
byte[] infosBytes = buffer.toArrayCopy();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseLiveDocsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseLiveDocsFormatTestCase.java
index 9c01990..4f15bef 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseLiveDocsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseLiveDocsFormatTestCase.java
@@ -125,10 +125,10 @@ public abstract class BaseLiveDocsFormatTestCase extends LuceneTestCase {
final Directory dir = newDirectory();
final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "foo", maxDoc, random().nextBoolean(),
codec, Collections.emptyMap(), StringHelper.randomId(), Collections.emptyMap(), null);
- SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, 0, 0, -1, -1);
+ SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, 0, 0, -1, -1, StringHelper.randomId());
format.writeLiveDocs(bits, dir, sci, maxDoc - numLiveDocs, IOContext.DEFAULT);
- sci = new SegmentCommitInfo(si, maxDoc - numLiveDocs, 0, 1, -1, -1);
+ sci = new SegmentCommitInfo(si, maxDoc - numLiveDocs, 0, 1, -1, -1, StringHelper.randomId());
final Bits bits2 = format.readLiveDocs(dir, sci, IOContext.READONCE);
assertEquals(maxDoc, bits2.length());
for (int i = 0; i < maxDoc; ++i) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseMergePolicyTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseMergePolicyTestCase.java
index 9928c84..94a85df 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseMergePolicyTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseMergePolicyTestCase.java
@@ -140,7 +140,7 @@ public abstract class BaseMergePolicyTestCase extends LuceneTestCase {
Collections.emptyMap(), // attributes
null /* indexSort */);
info.setFiles(Collections.emptyList());
- infos.add(new SegmentCommitInfo(info, random().nextInt(1), 0, -1, -1, -1));
+ infos.add(new SegmentCommitInfo(info, random().nextInt(1), 0, -1, -1, -1, StringHelper.randomId()));
}
MergePolicy.MergeSpecification forcedDeletesMerges = mp.findForcedDeletesMerges(infos, context);
if (forcedDeletesMerges != null) {
@@ -208,7 +208,7 @@ public abstract class BaseMergePolicyTestCase extends LuceneTestCase {
name, maxDoc, false, TestUtil.getDefaultCodec(), Collections.emptyMap(), id,
Collections.singletonMap(IndexWriter.SOURCE, source), null);
info.setFiles(Collections.singleton(name + "_size=" + Long.toString((long) (sizeMB * 1024 * 1024)) + ".fake"));
- return new SegmentCommitInfo(info, numDeletedDocs, 0, 0, 0, 0);
+ return new SegmentCommitInfo(info, numDeletedDocs, 0, 0, 0, 0, StringHelper.randomId());
}
/** A directory that computes the length of a file based on its name. */
@@ -331,7 +331,7 @@ public abstract class BaseMergePolicyTestCase extends LuceneTestCase {
int newDelCount = sci.getDelCount() + segDeletes;
assert newDelCount <= sci.info.maxDoc();
if (newDelCount < sci.info.maxDoc()) { // drop fully deleted segments
- SegmentCommitInfo newInfo = new SegmentCommitInfo(sci.info, sci.getDelCount() + segDeletes, 0, sci.getDelGen() + 1, sci.getFieldInfosGen(), sci.getDocValuesGen());
+ SegmentCommitInfo newInfo = new SegmentCommitInfo(sci.info, sci.getDelCount() + segDeletes, 0, sci.getDelGen() + 1, sci.getFieldInfosGen(), sci.getDocValuesGen(), StringHelper.randomId());
newInfoList.add(newInfo);
}
numDeletes -= segDeletes;