You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2017/04/04 15:33:14 UTC
[32/36] lucene-solr:jira/solr-6203: LUCENE-7756: Only record the
major Lucene version that created the index,
and record the minimum Lucene version that contributed to segments.
LUCENE-7756: Only record the major Lucene version that created the index, and record the minimum Lucene version that contributed to segments.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/23b002a0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/23b002a0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/23b002a0
Branch: refs/heads/jira/solr-6203
Commit: 23b002a0fdf2f6025f1eb026c0afca247fb21ed0
Parents: 3f172a0
Author: Adrien Grand <jp...@gmail.com>
Authored: Thu Mar 30 09:12:45 2017 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Tue Apr 4 09:57:16 2017 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 7 +-
.../lucene50/Lucene50SegmentInfoFormat.java | 2 +-
.../lucene/codecs/lucene62/Lucene62Codec.java | 2 +-
.../apache/lucene/index/FixBrokenOffsets.java | 3 +
.../lucene50/Lucene50RWSegmentInfoFormat.java | 2 +-
.../lucene50/TestLucene50SegmentInfoFormat.java | 10 +
.../lucene53/TestLucene53NormsFormat.java | 6 +
.../lucene/codecs/lucene62/Lucene62RWCodec.java | 12 +
.../lucene62/Lucene62RWSegmentInfoFormat.java | 193 ++++++++
.../lucene62/TestLucene62SegmentInfoFormat.java | 48 ++
.../index/TestBackwardsCompatibility.java | 69 ++-
.../lucene/index/TestFixBrokenOffsets.java | 10 +-
.../lucene/index/TestIndexWriterOnOldIndex.java | 6 +-
.../simpletext/SimpleTextSegmentInfoFormat.java | 29 +-
.../lucene62/Lucene62SegmentInfoFormat.java | 152 +------
.../lucene/codecs/lucene70/Lucene70Codec.java | 3 +-
.../lucene70/Lucene70SegmentInfoFormat.java | 439 +++++++++++++++++++
.../org/apache/lucene/index/CheckIndex.java | 2 +-
.../lucene/index/DocumentsWriterPerThread.java | 2 +-
.../apache/lucene/index/FilterCodecReader.java | 5 +-
.../apache/lucene/index/FilterLeafReader.java | 5 +-
.../org/apache/lucene/index/IndexWriter.java | 54 ++-
.../org/apache/lucene/index/LeafMetaData.java | 74 ++++
.../org/apache/lucene/index/LeafReader.java | 7 +-
.../apache/lucene/index/MergeReaderWrapper.java | 5 +-
.../org/apache/lucene/index/MergeState.java | 2 +-
.../apache/lucene/index/ParallelLeafReader.java | 36 +-
.../apache/lucene/index/ReadersAndUpdates.java | 4 +-
.../org/apache/lucene/index/SegmentInfo.java | 19 +-
.../org/apache/lucene/index/SegmentInfos.java | 77 ++--
.../org/apache/lucene/index/SegmentMerger.java | 14 +
.../org/apache/lucene/index/SegmentReader.java | 10 +-
.../lucene/index/SlowCodecReaderWrapper.java | 5 +-
.../lucene/index/StandardDirectoryReader.java | 4 +-
.../EarlyTerminatingSortingCollector.java | 2 +-
.../lucene62/TestLucene62SegmentInfoFormat.java | 39 --
.../lucene70/TestLucene70SegmentInfoFormat.java | 35 ++
.../org/apache/lucene/index/TestCodecs.java | 4 +-
.../index/TestDemoParallelLeafReader.java | 3 +-
.../test/org/apache/lucene/index/TestDoc.java | 8 +-
.../apache/lucene/index/TestDocumentWriter.java | 9 +-
.../apache/lucene/index/TestIndexSorting.java | 2 +-
.../apache/lucene/index/TestIndexWriter.java | 2 +-
.../index/TestIndexWriterThreadsToSegments.java | 3 +-
.../index/TestOneMergeWrappingMergePolicy.java | 1 +
.../apache/lucene/index/TestSegmentInfos.java | 22 +-
.../apache/lucene/index/TestSegmentMerger.java | 7 +-
.../apache/lucene/index/TestSegmentReader.java | 3 +-
.../lucene/index/TestSegmentTermDocs.java | 7 +-
.../search/highlight/TermVectorLeafReader.java | 7 +-
.../apache/lucene/index/memory/MemoryIndex.java | 6 +-
.../org/apache/lucene/index/IndexSplitter.java | 4 +-
.../lucene/replicator/nrt/ReplicaNode.java | 2 +-
.../index/BaseCompoundFormatTestCase.java | 3 +-
.../index/BaseFieldInfoFormatTestCase.java | 3 +-
.../index/BaseIndexFileFormatTestCase.java | 28 +-
.../lucene/index/BaseNormsFormatTestCase.java | 6 +-
.../index/BaseSegmentInfoFormatTestCase.java | 54 ++-
.../lucene/index/RandomPostingsTester.java | 2 +-
.../org/apache/lucene/search/QueryUtils.java | 6 +-
.../solr/index/SlowCompositeReaderWrapper.java | 19 +-
.../test/org/apache/solr/search/TestDocSet.java | 7 +-
62 files changed, 1208 insertions(+), 404 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 83113a8..1f3f30c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -7,9 +7,12 @@ http://s.apache.org/luceneversions
New Features
-* LUCENE-7703: SegmentInfos now record the Lucene version at index creation
- time. (Adrien Grand)
+* LUCENE-7703: SegmentInfos now record the major Lucene version at index
+ creation time. (Adrien Grand)
+* LUCENE-7756: LeafReader.getMetaData now exposes the index created version as
+ well as the oldest Lucene version that contributed to the segment.
+ (Adrien Grand)
API Changes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
index 69cda34..d2a384e 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
@@ -65,7 +65,7 @@ public class Lucene50SegmentInfoFormat extends SegmentInfoFormat {
final Set<String> files = input.readSetOfStrings();
final Map<String,String> attributes = input.readMapOfStrings();
- si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, null);
+ si = new SegmentInfo(dir, version, null, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, null);
si.setFiles(files);
} catch (Throwable exception) {
priorE = exception;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene62/Lucene62Codec.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene62/Lucene62Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene62/Lucene62Codec.java
index 58b07eb..3dd7daa 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene62/Lucene62Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene62/Lucene62Codec.java
@@ -114,7 +114,7 @@ public class Lucene62Codec extends Codec {
}
@Override
- public final SegmentInfoFormat segmentInfoFormat() {
+ public SegmentInfoFormat segmentInfoFormat() {
return segmentInfosFormat;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/java/org/apache/lucene/index/FixBrokenOffsets.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/index/FixBrokenOffsets.java b/lucene/backward-codecs/src/java/org/apache/lucene/index/FixBrokenOffsets.java
index e775a28..9b3615e 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/index/FixBrokenOffsets.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/index/FixBrokenOffsets.java
@@ -128,6 +128,9 @@ public class FixBrokenOffsets {
}
Directory destDir = FSDirectory.open(destPath);
+ // We need to maintain the same major version
+ int createdMajor = SegmentInfos.readLatestCommit(srcDir).getIndexCreatedVersionMajor();
+ new SegmentInfos(createdMajor).commit(destDir);
IndexWriter writer = new IndexWriter(destDir, new IndexWriterConfig());
writer.addIndexes(filtered);
IOUtils.close(writer, reader, srcDir, destDir);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/Lucene50RWSegmentInfoFormat.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/Lucene50RWSegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/Lucene50RWSegmentInfoFormat.java
index 965ee96..4bed311 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/Lucene50RWSegmentInfoFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/Lucene50RWSegmentInfoFormat.java
@@ -65,7 +65,7 @@ public class Lucene50RWSegmentInfoFormat extends Lucene50SegmentInfoFormat {
final Set<String> files = input.readSetOfStrings();
final Map<String,String> attributes = input.readMapOfStrings();
- si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, null);
+ si = new SegmentInfo(dir, version, null, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, null);
si.setFiles(files);
} catch (Throwable exception) {
priorE = exception;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50SegmentInfoFormat.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50SegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50SegmentInfoFormat.java
index 688afed..0a9bf79 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50SegmentInfoFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50SegmentInfoFormat.java
@@ -29,6 +29,11 @@ public class TestLucene50SegmentInfoFormat extends BaseSegmentInfoFormatTestCase
}
@Override
+ protected int getCreatedVersionMajor() {
+ return Version.LUCENE_6_0_0.major;
+ }
+
+ @Override
protected Version[] getVersions() {
return new Version[] { Version.LUCENE_6_0_0 };
}
@@ -37,4 +42,9 @@ public class TestLucene50SegmentInfoFormat extends BaseSegmentInfoFormatTestCase
protected boolean supportsIndexSort() {
return false;
}
+
+ @Override
+ protected boolean supportsMinVersion() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene53/TestLucene53NormsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene53/TestLucene53NormsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene53/TestLucene53NormsFormat.java
index 80a8eee..7d37b45 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene53/TestLucene53NormsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene53/TestLucene53NormsFormat.java
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene53;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene62.Lucene62RWCodec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
+import org.apache.lucene.util.Version;
/**
* Tests Lucene53NormsFormat
@@ -27,6 +28,11 @@ public class TestLucene53NormsFormat extends BaseNormsFormatTestCase {
private final Codec codec = new Lucene62RWCodec();
@Override
+ protected int getCreatedVersionMajor() {
+ return Version.LUCENE_6_2_0.major;
+ }
+
+ @Override
protected Codec getCodec() {
return codec;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/Lucene62RWCodec.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/Lucene62RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/Lucene62RWCodec.java
index fcb414d..34d3a7f 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/Lucene62RWCodec.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/Lucene62RWCodec.java
@@ -17,14 +17,26 @@
package org.apache.lucene.codecs.lucene62;
import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.lucene53.Lucene53RWNormsFormat;
import org.apache.lucene.codecs.lucene62.Lucene62Codec;
+/**
+ * Read-write version of 6.2 codec for testing
+ * @deprecated for test purposes only
+ */
+@Deprecated
public class Lucene62RWCodec extends Lucene62Codec {
+ private final SegmentInfoFormat segmentInfoFormat = new Lucene62RWSegmentInfoFormat();
private final NormsFormat normsFormat = new Lucene53RWNormsFormat();
@Override
+ public SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfoFormat;
+ }
+
+ @Override
public NormsFormat normsFormat() {
return normsFormat;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/Lucene62RWSegmentInfoFormat.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/Lucene62RWSegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/Lucene62RWSegmentInfoFormat.java
new file mode 100644
index 0000000..f2fbe9d
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/Lucene62RWSegmentInfoFormat.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene62;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSelector;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSelector;
+import org.apache.lucene.search.SortedSetSortField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.Version;
+
+/**
+ * Read-write version of 6.2 SegmentInfoFormat for testing
+ * @deprecated for test purposes only
+ */
+@Deprecated
+public class Lucene62RWSegmentInfoFormat extends Lucene62SegmentInfoFormat {
+
+ @Override
+ public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {
+ final String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene62SegmentInfoFormat.SI_EXTENSION);
+
+ try (IndexOutput output = dir.createOutput(fileName, ioContext)) {
+ // Only add the file once we've successfully created it, else IFD assert can trip:
+ si.addFile(fileName);
+ CodecUtil.writeIndexHeader(output,
+ Lucene62SegmentInfoFormat.CODEC_NAME,
+ Lucene62SegmentInfoFormat.VERSION_CURRENT,
+ si.getId(),
+ "");
+ Version version = si.getVersion();
+ if (version.major < 5) {
+ throw new IllegalArgumentException("invalid major version: should be >= 5 but got: " + version.major + " segment=" + si);
+ }
+ // Write the Lucene version that created this segment, since 3.1
+ output.writeInt(version.major);
+ output.writeInt(version.minor);
+ output.writeInt(version.bugfix);
+ assert version.prerelease == 0;
+ output.writeInt(si.maxDoc());
+
+ output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
+ output.writeMapOfStrings(si.getDiagnostics());
+ Set<String> files = si.files();
+ for (String file : files) {
+ if (!IndexFileNames.parseSegmentName(file).equals(si.name)) {
+ throw new IllegalArgumentException("invalid files: expected segment=" + si.name + ", got=" + files);
+ }
+ }
+ output.writeSetOfStrings(files);
+ output.writeMapOfStrings(si.getAttributes());
+
+ Sort indexSort = si.getIndexSort();
+ int numSortFields = indexSort == null ? 0 : indexSort.getSort().length;
+ output.writeVInt(numSortFields);
+ for (int i = 0; i < numSortFields; ++i) {
+ SortField sortField = indexSort.getSort()[i];
+ SortField.Type sortType = sortField.getType();
+ output.writeString(sortField.getField());
+ int sortTypeID;
+ switch (sortField.getType()) {
+ case STRING:
+ sortTypeID = 0;
+ break;
+ case LONG:
+ sortTypeID = 1;
+ break;
+ case INT:
+ sortTypeID = 2;
+ break;
+ case DOUBLE:
+ sortTypeID = 3;
+ break;
+ case FLOAT:
+ sortTypeID = 4;
+ break;
+ case CUSTOM:
+ if (sortField instanceof SortedSetSortField) {
+ sortTypeID = 5;
+ sortType = SortField.Type.STRING;
+ } else if (sortField instanceof SortedNumericSortField) {
+ sortTypeID = 6;
+ sortType = ((SortedNumericSortField) sortField).getNumericType();
+ } else {
+ throw new IllegalStateException("Unexpected SortedNumericSortField " + sortField);
+ }
+ break;
+ default:
+ throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
+ }
+ output.writeVInt(sortTypeID);
+ if (sortTypeID == 5) {
+ SortedSetSortField ssf = (SortedSetSortField) sortField;
+ if (ssf.getSelector() == SortedSetSelector.Type.MIN) {
+ output.writeByte((byte) 0);
+ } else if (ssf.getSelector() == SortedSetSelector.Type.MAX) {
+ output.writeByte((byte) 1);
+ } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MIN) {
+ output.writeByte((byte) 2);
+ } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MAX) {
+ output.writeByte((byte) 3);
+ } else {
+ throw new IllegalStateException("Unexpected SortedSetSelector type: " + ssf.getSelector());
+ }
+ } else if (sortTypeID == 6) {
+ SortedNumericSortField snsf = (SortedNumericSortField) sortField;
+ if (snsf.getNumericType() == SortField.Type.LONG) {
+ output.writeByte((byte) 0);
+ } else if (snsf.getNumericType() == SortField.Type.INT) {
+ output.writeByte((byte) 1);
+ } else if (snsf.getNumericType() == SortField.Type.DOUBLE) {
+ output.writeByte((byte) 2);
+ } else if (snsf.getNumericType() == SortField.Type.FLOAT) {
+ output.writeByte((byte) 3);
+ } else {
+ throw new IllegalStateException("Unexpected SortedNumericSelector type: " + snsf.getNumericType());
+ }
+ if (snsf.getSelector() == SortedNumericSelector.Type.MIN) {
+ output.writeByte((byte) 0);
+ } else if (snsf.getSelector() == SortedNumericSelector.Type.MAX) {
+ output.writeByte((byte) 1);
+ } else {
+ throw new IllegalStateException("Unexpected sorted numeric selector type: " + snsf.getSelector());
+ }
+ }
+ output.writeByte((byte) (sortField.getReverse() ? 0 : 1));
+
+ // write missing value
+ Object missingValue = sortField.getMissingValue();
+ if (missingValue == null) {
+ output.writeByte((byte) 0);
+ } else {
+ switch(sortType) {
+ case STRING:
+ if (missingValue == SortField.STRING_LAST) {
+ output.writeByte((byte) 1);
+ } else if (missingValue == SortField.STRING_FIRST) {
+ output.writeByte((byte) 2);
+ } else {
+ throw new AssertionError("unrecognized missing value for STRING field \"" + sortField.getField() + "\": " + missingValue);
+ }
+ break;
+ case LONG:
+ output.writeByte((byte) 1);
+ output.writeLong(((Long) missingValue).longValue());
+ break;
+ case INT:
+ output.writeByte((byte) 1);
+ output.writeInt(((Integer) missingValue).intValue());
+ break;
+ case DOUBLE:
+ output.writeByte((byte) 1);
+ output.writeLong(Double.doubleToLongBits(((Double) missingValue).doubleValue()));
+ break;
+ case FLOAT:
+ output.writeByte((byte) 1);
+ output.writeInt(Float.floatToIntBits(((Float) missingValue).floatValue()));
+ break;
+ default:
+ throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
+ }
+ }
+ }
+
+ CodecUtil.writeFooter(output);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/TestLucene62SegmentInfoFormat.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/TestLucene62SegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/TestLucene62SegmentInfoFormat.java
new file mode 100644
index 0000000..e0efa95
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene62/TestLucene62SegmentInfoFormat.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.codecs.lucene62;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.BaseSegmentInfoFormatTestCase;
+import org.apache.lucene.util.Version;
+
+/**
+ * Tests Lucene62SegmentInfoFormat
+ */
+public class TestLucene62SegmentInfoFormat extends BaseSegmentInfoFormatTestCase {
+
+ @Override
+ protected int getCreatedVersionMajor() {
+ return Version.LUCENE_6_2_0.major;
+ }
+
+ @Override
+ protected Version[] getVersions() {
+ return new Version[] { Version.LUCENE_6_2_0 };
+ }
+
+ @Override
+ protected Codec getCodec() {
+ return new Lucene62RWCodec();
+ }
+
+ @Override
+ protected boolean supportsMinVersion() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 8e87dcc..f180b47 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -693,10 +693,18 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
System.out.println("\nTEST: index=" + name);
}
Directory dir = newDirectory(oldIndexDirs.get(name));
+
+ final SegmentInfos oldSegInfos = SegmentInfos.readLatestCommit(dir);
+
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
w.forceMerge(1);
w.close();
-
+
+ final SegmentInfos segInfos = SegmentInfos.readLatestCommit(dir);
+ assertEquals(oldSegInfos.getIndexCreatedVersionMajor(), segInfos.getIndexCreatedVersionMajor());
+ assertEquals(Version.LATEST, segInfos.asList().get(0).info.getVersion());
+ assertEquals(oldSegInfos.asList().get(0).info.getMinVersion(), segInfos.asList().get(0).info.getMinVersion());
+
dir.close();
}
}
@@ -707,26 +715,30 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
System.out.println("\nTEST: old index " + name);
}
Directory oldDir = oldIndexDirs.get(name);
- Version indexCreatedVersion = SegmentInfos.readLatestCommit(oldDir).getIndexCreatedVersion();
+ SegmentInfos infos = SegmentInfos.readLatestCommit(oldDir);
Directory targetDir = newDirectory();
- // Simulate writing into an index that was created on the same version
- new SegmentInfos(indexCreatedVersion).commit(targetDir);
+ if (infos.getCommitLuceneVersion().major != Version.LATEST.major) {
+ // both indexes are not compatible
+ Directory targetDir2 = newDirectory();
+ IndexWriter w = new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));
+ IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> w.addIndexes(oldDir));
+ assertTrue(e.getMessage(), e.getMessage().startsWith("Cannot use addIndexes(Directory) with indexes that have been created by a different Lucene version."));
+ w.close();
+ targetDir2.close();
+
+ // for the next test, we simulate writing to an index that was created on the same major version
+ new SegmentInfos(infos.getIndexCreatedVersionMajor()).commit(targetDir);
+ }
+
IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));
w.addIndexes(oldDir);
w.close();
targetDir.close();
- // Now check that we forbid calling addIndexes with a different version
- targetDir = newDirectory();
- IndexWriter oldWriter = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> oldWriter.addIndexes(oldDir));
- assertTrue(e.getMessage(), e.getMessage().startsWith("Cannot use addIndexes(Directory) with indexes that have been created by a different Lucene version."));
-
if (VERBOSE) {
System.out.println("\nTEST: done adding indices; now close");
}
- oldWriter.close();
targetDir.close();
}
@@ -734,9 +746,22 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
public void testAddOldIndexesReader() throws IOException {
for (String name : oldNames) {
- DirectoryReader reader = DirectoryReader.open(oldIndexDirs.get(name));
+ Directory oldDir = oldIndexDirs.get(name);
+ SegmentInfos infos = SegmentInfos.readLatestCommit(oldDir);
+ DirectoryReader reader = DirectoryReader.open(oldDir);
Directory targetDir = newDirectory();
+ if (infos.getCommitLuceneVersion().major != Version.LATEST.major) {
+ Directory targetDir2 = newDirectory();
+ IndexWriter w = new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));
+ IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TestUtil.addIndexesSlowly(w, reader));
+ assertEquals(e.getMessage(), "Cannot merge a segment that has been created with major version 6 into this index which has been created by major version 7");
+ w.close();
+ targetDir2.close();
+
+ // for the next test, we simulate writing to an index that was created on the same major version
+ new SegmentInfos(infos.getIndexCreatedVersionMajor()).commit(targetDir);
+ }
IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));
TestUtil.addIndexesSlowly(w, reader);
w.close();
@@ -1245,11 +1270,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
// those indexes are created by a single version so we can
// compare the commit version with the created version
- if (infos.getCommitLuceneVersion().onOrAfter(Version.LUCENE_7_0_0)) {
- assertEquals(infos.getCommitLuceneVersion(), infos.getIndexCreatedVersion());
- } else {
- assertNull(infos.getIndexCreatedVersion());
- }
+ assertEquals(infos.getCommitLuceneVersion().major, infos.getIndexCreatedVersionMajor());
}
}
@@ -1316,7 +1337,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
}
- private int checkAllSegmentsUpgraded(Directory dir, Version indexCreatedVersion) throws IOException {
+ private int checkAllSegmentsUpgraded(Directory dir, int indexCreatedVersion) throws IOException {
final SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
if (VERBOSE) {
System.out.println("checkAllSegmentsUpgraded: " + infos);
@@ -1325,7 +1346,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
assertEquals(Version.LATEST, si.info.getVersion());
}
assertEquals(Version.LATEST, infos.getCommitLuceneVersion());
- assertEquals(indexCreatedVersion, infos.getIndexCreatedVersion());
+ assertEquals(indexCreatedVersion, infos.getIndexCreatedVersionMajor());
return infos.size();
}
@@ -1343,7 +1364,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
System.out.println("testUpgradeOldIndex: index=" +name);
}
Directory dir = newDirectory(oldIndexDirs.get(name));
- Version indexCreatedVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersion();
+ int indexCreatedVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor();
newIndexUpgrader(dir).upgrade();
@@ -1360,7 +1381,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
try {
for (Map.Entry<String,Directory> entry : oldIndexDirs.entrySet()) {
String name = entry.getKey();
- Version indexCreatedVersion = SegmentInfos.readLatestCommit(entry.getValue()).getIndexCreatedVersion();
+ int indexCreatedVersion = SegmentInfos.readLatestCommit(entry.getValue()).getIndexCreatedVersionMajor();
Path dir = createTempDir(name);
TestUtil.unzip(getDataInputStream("index." + name + ".zip"), dir);
@@ -1413,7 +1434,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
Directory dir = newDirectory(oldIndexDirs.get(name));
assertEquals("Original index must be single segment", 1, getNumberOfSegments(dir));
- Version indexCreatedVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersion();
+ int indexCreatedVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor();
// create a bunch of dummy segments
int id = 40;
@@ -1472,7 +1493,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
newIndexUpgrader(dir).upgrade();
- checkAllSegmentsUpgraded(dir, null);
+ checkAllSegmentsUpgraded(dir, 6);
dir.close();
}
@@ -1598,7 +1619,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
- Sort sort = reader.leaves().get(0).reader().getIndexSort();
+ Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
assertNotNull(sort);
assertEquals("<long: \"dateDV\">!", sort.toString());
reader.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/test/org/apache/lucene/index/TestFixBrokenOffsets.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestFixBrokenOffsets.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestFixBrokenOffsets.java
index 917785e..46b30d3 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestFixBrokenOffsets.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestFixBrokenOffsets.java
@@ -16,7 +16,6 @@
*/
package org.apache.lucene.index;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
@@ -94,14 +93,11 @@ public class TestFixBrokenOffsets extends LuceneTestCase {
for(int i=0;i<leaves.size();i++) {
codecReaders[i] = (CodecReader) leaves.get(i).reader();
}
- w.addIndexes(codecReaders);
+ IndexWriter finalW2 = w;
+ e = expectThrows(IllegalArgumentException.class, () -> finalW2.addIndexes(codecReaders));
+ assertEquals("Cannot merge a segment that has been created with major version 6 into this index which has been created by major version 7", e.getMessage());
reader.close();
w.close();
-
- // NOT OK: broken offsets were copied into a 7.0 segment:
- ByteArrayOutputStream output = new ByteArrayOutputStream(1024);
- RuntimeException re = expectThrows(RuntimeException.class, () -> {TestUtil.checkIndex(tmpDir2, false, true, output);});
- assertEquals("term [66 6f 6f]: doc 0: pos 1: startOffset 7 < lastStartOffset 10; consider using the FixBrokenOffsets tool in Lucene's backward-codecs module to correct your index", re.getMessage());
tmpDir2.close();
// Now run the tool and confirm the broken offsets are fixed:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/backward-codecs/src/test/org/apache/lucene/index/TestIndexWriterOnOldIndex.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestIndexWriterOnOldIndex.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestIndexWriterOnOldIndex.java
index 73d933a..c77b926 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestIndexWriterOnOldIndex.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestIndexWriterOnOldIndex.java
@@ -36,16 +36,16 @@ public class TestIndexWriterOnOldIndex extends LuceneTestCase {
Directory dir = newFSDirectory(path);
for (OpenMode openMode : OpenMode.values()) {
Directory tmpDir = newDirectory(dir);
- assertEquals(null /** 6.3.0 */, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersion());
+ assertEquals(6 /** 6.3.0 */, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());
IndexWriter w = new IndexWriter(tmpDir, newIndexWriterConfig().setOpenMode(openMode));
w.commit();
w.close();
switch (openMode) {
case CREATE:
- assertEquals(Version.LATEST, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersion());
+ assertEquals(Version.LATEST.major, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());
break;
default:
- assertEquals(null /** 6.3.0 */, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersion());
+ assertEquals(6 /** 6.3.0 */, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());
}
tmpDir.close();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
index 3d38d72..8a71c6d 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
@@ -55,6 +55,7 @@ import org.apache.lucene.util.Version;
*/
public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
final static BytesRef SI_VERSION = new BytesRef(" version ");
+ final static BytesRef SI_MIN_VERSION = new BytesRef(" min version ");
final static BytesRef SI_DOCCOUNT = new BytesRef(" number of documents ");
final static BytesRef SI_USECOMPOUND = new BytesRef(" uses compound file ");
final static BytesRef SI_NUM_DIAG = new BytesRef(" diagnostics ");
@@ -88,7 +89,21 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
} catch (ParseException pe) {
throw new CorruptIndexException("unable to parse version string: " + pe.getMessage(), input, pe);
}
-
+
+ SimpleTextUtil.readLine(input, scratch);
+ assert StringHelper.startsWith(scratch.get(), SI_MIN_VERSION);
+ Version minVersion;
+ try {
+ String versionString = readString(SI_MIN_VERSION.length, scratch);
+ if (versionString.equals("null")) {
+ minVersion = null;
+ } else {
+ minVersion = Version.parse(versionString);
+ }
+ } catch (ParseException pe) {
+ throw new CorruptIndexException("unable to parse version string: " + pe.getMessage(), input, pe);
+ }
+
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), SI_DOCCOUNT);
final int docCount = Integer.parseInt(readString(SI_DOCCOUNT.length, scratch));
@@ -288,7 +303,7 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
SimpleTextUtil.checkFooter(input);
- SegmentInfo info = new SegmentInfo(directory, version, segmentName, docCount,
+ SegmentInfo info = new SegmentInfo(directory, version, minVersion, segmentName, docCount,
isCompoundFile, null, Collections.unmodifiableMap(diagnostics),
id, Collections.unmodifiableMap(attributes), indexSort);
info.setFiles(files);
@@ -345,7 +360,15 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
SimpleTextUtil.write(output, SI_VERSION);
SimpleTextUtil.write(output, si.getVersion().toString(), scratch);
SimpleTextUtil.writeNewline(output);
-
+
+ SimpleTextUtil.write(output, SI_MIN_VERSION);
+ if (si.getMinVersion() == null) {
+ SimpleTextUtil.write(output, "null", scratch);
+ } else {
+ SimpleTextUtil.write(output, si.getMinVersion().toString(), scratch);
+ }
+ SimpleTextUtil.writeNewline(output);
+
SimpleTextUtil.write(output, SI_DOCCOUNT);
SimpleTextUtil.write(output, Integer.toString(si.maxDoc()), scratch);
SimpleTextUtil.writeNewline(output);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/codecs/lucene62/Lucene62SegmentInfoFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene62/Lucene62SegmentInfoFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene62/Lucene62SegmentInfoFormat.java
index da6e395..e91da3b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene62/Lucene62SegmentInfoFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene62/Lucene62SegmentInfoFormat.java
@@ -37,7 +37,6 @@ import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataOutput; // javadocs
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Version;
/**
@@ -244,7 +243,7 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
indexSort = null;
}
- si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, indexSort);
+ si = new SegmentInfo(dir, version, null, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, indexSort);
si.setFiles(files);
} catch (Throwable exception) {
priorE = exception;
@@ -256,153 +255,8 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
}
@Override
- public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene62SegmentInfoFormat.SI_EXTENSION);
-
- try (IndexOutput output = dir.createOutput(fileName, ioContext)) {
- // Only add the file once we've successfully created it, else IFD assert can trip:
- si.addFile(fileName);
- CodecUtil.writeIndexHeader(output,
- Lucene62SegmentInfoFormat.CODEC_NAME,
- Lucene62SegmentInfoFormat.VERSION_CURRENT,
- si.getId(),
- "");
- Version version = si.getVersion();
- if (version.major < 5) {
- throw new IllegalArgumentException("invalid major version: should be >= 5 but got: " + version.major + " segment=" + si);
- }
- // Write the Lucene version that created this segment, since 3.1
- output.writeInt(version.major);
- output.writeInt(version.minor);
- output.writeInt(version.bugfix);
- assert version.prerelease == 0;
- output.writeInt(si.maxDoc());
-
- output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
- output.writeMapOfStrings(si.getDiagnostics());
- Set<String> files = si.files();
- for (String file : files) {
- if (!IndexFileNames.parseSegmentName(file).equals(si.name)) {
- throw new IllegalArgumentException("invalid files: expected segment=" + si.name + ", got=" + files);
- }
- }
- output.writeSetOfStrings(files);
- output.writeMapOfStrings(si.getAttributes());
-
- Sort indexSort = si.getIndexSort();
- int numSortFields = indexSort == null ? 0 : indexSort.getSort().length;
- output.writeVInt(numSortFields);
- for (int i = 0; i < numSortFields; ++i) {
- SortField sortField = indexSort.getSort()[i];
- SortField.Type sortType = sortField.getType();
- output.writeString(sortField.getField());
- int sortTypeID;
- switch (sortField.getType()) {
- case STRING:
- sortTypeID = 0;
- break;
- case LONG:
- sortTypeID = 1;
- break;
- case INT:
- sortTypeID = 2;
- break;
- case DOUBLE:
- sortTypeID = 3;
- break;
- case FLOAT:
- sortTypeID = 4;
- break;
- case CUSTOM:
- if (sortField instanceof SortedSetSortField) {
- sortTypeID = 5;
- sortType = SortField.Type.STRING;
- } else if (sortField instanceof SortedNumericSortField) {
- sortTypeID = 6;
- sortType = ((SortedNumericSortField) sortField).getNumericType();
- } else {
- throw new IllegalStateException("Unexpected SortedNumericSortField " + sortField);
- }
- break;
- default:
- throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
- }
- output.writeVInt(sortTypeID);
- if (sortTypeID == 5) {
- SortedSetSortField ssf = (SortedSetSortField) sortField;
- if (ssf.getSelector() == SortedSetSelector.Type.MIN) {
- output.writeByte((byte) 0);
- } else if (ssf.getSelector() == SortedSetSelector.Type.MAX) {
- output.writeByte((byte) 1);
- } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MIN) {
- output.writeByte((byte) 2);
- } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MAX) {
- output.writeByte((byte) 3);
- } else {
- throw new IllegalStateException("Unexpected SortedSetSelector type: " + ssf.getSelector());
- }
- } else if (sortTypeID == 6) {
- SortedNumericSortField snsf = (SortedNumericSortField) sortField;
- if (snsf.getNumericType() == SortField.Type.LONG) {
- output.writeByte((byte) 0);
- } else if (snsf.getNumericType() == SortField.Type.INT) {
- output.writeByte((byte) 1);
- } else if (snsf.getNumericType() == SortField.Type.DOUBLE) {
- output.writeByte((byte) 2);
- } else if (snsf.getNumericType() == SortField.Type.FLOAT) {
- output.writeByte((byte) 3);
- } else {
- throw new IllegalStateException("Unexpected SortedNumericSelector type: " + snsf.getNumericType());
- }
- if (snsf.getSelector() == SortedNumericSelector.Type.MIN) {
- output.writeByte((byte) 0);
- } else if (snsf.getSelector() == SortedNumericSelector.Type.MAX) {
- output.writeByte((byte) 1);
- } else {
- throw new IllegalStateException("Unexpected sorted numeric selector type: " + snsf.getSelector());
- }
- }
- output.writeByte((byte) (sortField.getReverse() ? 0 : 1));
-
- // write missing value
- Object missingValue = sortField.getMissingValue();
- if (missingValue == null) {
- output.writeByte((byte) 0);
- } else {
- switch(sortType) {
- case STRING:
- if (missingValue == SortField.STRING_LAST) {
- output.writeByte((byte) 1);
- } else if (missingValue == SortField.STRING_FIRST) {
- output.writeByte((byte) 2);
- } else {
- throw new AssertionError("unrecognized missing value for STRING field \"" + sortField.getField() + "\": " + missingValue);
- }
- break;
- case LONG:
- output.writeByte((byte) 1);
- output.writeLong(((Long) missingValue).longValue());
- break;
- case INT:
- output.writeByte((byte) 1);
- output.writeInt(((Integer) missingValue).intValue());
- break;
- case DOUBLE:
- output.writeByte((byte) 1);
- output.writeLong(Double.doubleToLongBits(((Double) missingValue).doubleValue()));
- break;
- case FLOAT:
- output.writeByte((byte) 1);
- output.writeInt(Float.floatToIntBits(((Float) missingValue).floatValue()));
- break;
- default:
- throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
- }
- }
- }
-
- CodecUtil.writeFooter(output);
- }
+ public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws IOException {
+ throw new UnsupportedOperationException("This format can only be used for reading");
}
/** File extension used to store {@link SegmentInfo}. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70Codec.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70Codec.java
index 7f9aed0..d04d554 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70Codec.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70Codec.java
@@ -37,7 +37,6 @@ import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
import org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat;
import org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat;
import org.apache.lucene.codecs.lucene60.Lucene60PointsFormat;
-import org.apache.lucene.codecs.lucene62.Lucene62SegmentInfoFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@@ -55,7 +54,7 @@ import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
public class Lucene70Codec extends Codec {
private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();
- private final SegmentInfoFormat segmentInfosFormat = new Lucene62SegmentInfoFormat();
+ private final SegmentInfoFormat segmentInfosFormat = new Lucene70SegmentInfoFormat();
private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70SegmentInfoFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70SegmentInfoFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70SegmentInfoFormat.java
new file mode 100644
index 0000000..bd2bf06
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70SegmentInfoFormat.java
@@ -0,0 +1,439 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene70;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexWriter; // javadocs
+import org.apache.lucene.index.SegmentInfo; // javadocs
+import org.apache.lucene.index.SegmentInfos; // javadocs
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSelector;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSelector;
+import org.apache.lucene.search.SortedSetSortField;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.DataOutput; // javadocs
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.Version;
+
+/**
+ * Lucene 7.0 Segment info format.
+ * <p>
+ * Files:
+ * <ul>
+ * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Attributes, IndexSort, Footer
+ * </ul>
+ * Data types:
+ * <ul>
+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}</li>
+ * <li>SegSize --> {@link DataOutput#writeInt Int32}</li>
+ * <li>SegVersion --> {@link DataOutput#writeString String}</li>
+ * <li>SegMinVersion --> {@link DataOutput#writeString String}</li>
+ * <li>Files --> {@link DataOutput#writeSetOfStrings Set<String>}</li>
+ * <li>Diagnostics,Attributes --> {@link DataOutput#writeMapOfStrings Map<String,String>}</li>
+ * <li>IsCompoundFile --> {@link DataOutput#writeByte Int8}</li>
+ * <li>IndexSort --> {@link DataOutput#writeVInt Int32} count, followed by {@code count} SortField</li>
+ * <li>SortField --> {@link DataOutput#writeString String} field name, followed by {@link DataOutput#writeVInt Int32} sort type ID,
+ * followed by {@link DataOutput#writeByte Int8} indicatating reversed sort, followed by a type-specific encoding of the optional missing value
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
+ * </ul>
+ * Field Descriptions:
+ * <ul>
+ * <li>SegVersion is the code version that created the segment.</li>
+ * <li>SegMinVersion is the minimum code version that contributed documents to the segment.</li>
+ * <li>SegSize is the number of documents contained in the segment index.</li>
+ * <li>IsCompoundFile records whether the segment is written as a compound file or
+ * not. If this is -1, the segment is not a compound file. If it is 1, the segment
+ * is a compound file.</li>
+ * <li>The Diagnostics Map is privately written by {@link IndexWriter}, as a debugging aid,
+ * for each segment it creates. It includes metadata like the current Lucene
+ * version, OS, Java version, why the segment was created (merge, flush,
+ * addIndexes), etc.</li>
+ * <li>Files is a list of files referred to by this segment.</li>
+ * </ul>
+ *
+ * @see SegmentInfos
+ * @lucene.experimental
+ */
+public class Lucene70SegmentInfoFormat extends SegmentInfoFormat {
+
+ /** Sole constructor. */
+ public Lucene70SegmentInfoFormat() {
+ }
+
+ @Override
+ public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context) throws IOException {
+ final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene70SegmentInfoFormat.SI_EXTENSION);
+ try (ChecksumIndexInput input = dir.openChecksumInput(fileName, context)) {
+ Throwable priorE = null;
+ SegmentInfo si = null;
+ try {
+ int format = CodecUtil.checkIndexHeader(input, Lucene70SegmentInfoFormat.CODEC_NAME,
+ Lucene70SegmentInfoFormat.VERSION_START,
+ Lucene70SegmentInfoFormat.VERSION_CURRENT,
+ segmentID, "");
+ final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
+ byte hasMinVersion = input.readByte();
+ final Version minVersion;
+ switch (hasMinVersion) {
+ case 0:
+ minVersion = null;
+ break;
+ case 1:
+ minVersion = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
+ break;
+ default:
+ throw new CorruptIndexException("Illegal boolean value " + hasMinVersion, input);
+ }
+
+ final int docCount = input.readInt();
+ if (docCount < 0) {
+ throw new CorruptIndexException("invalid docCount: " + docCount, input);
+ }
+ final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
+
+ final Map<String,String> diagnostics = input.readMapOfStrings();
+ final Set<String> files = input.readSetOfStrings();
+ final Map<String,String> attributes = input.readMapOfStrings();
+
+ int numSortFields = input.readVInt();
+ Sort indexSort;
+ if (numSortFields > 0) {
+ SortField[] sortFields = new SortField[numSortFields];
+ for(int i=0;i<numSortFields;i++) {
+ String fieldName = input.readString();
+ int sortTypeID = input.readVInt();
+ SortField.Type sortType;
+ SortedSetSelector.Type sortedSetSelector = null;
+ SortedNumericSelector.Type sortedNumericSelector = null;
+ switch(sortTypeID) {
+ case 0:
+ sortType = SortField.Type.STRING;
+ break;
+ case 1:
+ sortType = SortField.Type.LONG;
+ break;
+ case 2:
+ sortType = SortField.Type.INT;
+ break;
+ case 3:
+ sortType = SortField.Type.DOUBLE;
+ break;
+ case 4:
+ sortType = SortField.Type.FLOAT;
+ break;
+ case 5:
+ sortType = SortField.Type.STRING;
+ byte selector = input.readByte();
+ if (selector == 0) {
+ sortedSetSelector = SortedSetSelector.Type.MIN;
+ } else if (selector == 1) {
+ sortedSetSelector = SortedSetSelector.Type.MAX;
+ } else if (selector == 2) {
+ sortedSetSelector = SortedSetSelector.Type.MIDDLE_MIN;
+ } else if (selector == 3) {
+ sortedSetSelector = SortedSetSelector.Type.MIDDLE_MAX;
+ } else {
+ throw new CorruptIndexException("invalid index SortedSetSelector ID: " + selector, input);
+ }
+ break;
+ case 6:
+ byte type = input.readByte();
+ if (type == 0) {
+ sortType = SortField.Type.LONG;
+ } else if (type == 1) {
+ sortType = SortField.Type.INT;
+ } else if (type == 2) {
+ sortType = SortField.Type.DOUBLE;
+ } else if (type == 3) {
+ sortType = SortField.Type.FLOAT;
+ } else {
+ throw new CorruptIndexException("invalid index SortedNumericSortField type ID: " + type, input);
+ }
+ byte numericSelector = input.readByte();
+ if (numericSelector == 0) {
+ sortedNumericSelector = SortedNumericSelector.Type.MIN;
+ } else if (numericSelector == 1) {
+ sortedNumericSelector = SortedNumericSelector.Type.MAX;
+ } else {
+ throw new CorruptIndexException("invalid index SortedNumericSelector ID: " + numericSelector, input);
+ }
+ break;
+ default:
+ throw new CorruptIndexException("invalid index sort field type ID: " + sortTypeID, input);
+ }
+ byte b = input.readByte();
+ boolean reverse;
+ if (b == 0) {
+ reverse = true;
+ } else if (b == 1) {
+ reverse = false;
+ } else {
+ throw new CorruptIndexException("invalid index sort reverse: " + b, input);
+ }
+
+ if (sortedSetSelector != null) {
+ sortFields[i] = new SortedSetSortField(fieldName, reverse, sortedSetSelector);
+ } else if (sortedNumericSelector != null) {
+ sortFields[i] = new SortedNumericSortField(fieldName, sortType, reverse, sortedNumericSelector);
+ } else {
+ sortFields[i] = new SortField(fieldName, sortType, reverse);
+ }
+
+ Object missingValue;
+ b = input.readByte();
+ if (b == 0) {
+ missingValue = null;
+ } else {
+ switch(sortType) {
+ case STRING:
+ if (b == 1) {
+ missingValue = SortField.STRING_LAST;
+ } else if (b == 2) {
+ missingValue = SortField.STRING_FIRST;
+ } else {
+ throw new CorruptIndexException("invalid missing value flag: " + b, input);
+ }
+ break;
+ case LONG:
+ if (b != 1) {
+ throw new CorruptIndexException("invalid missing value flag: " + b, input);
+ }
+ missingValue = input.readLong();
+ break;
+ case INT:
+ if (b != 1) {
+ throw new CorruptIndexException("invalid missing value flag: " + b, input);
+ }
+ missingValue = input.readInt();
+ break;
+ case DOUBLE:
+ if (b != 1) {
+ throw new CorruptIndexException("invalid missing value flag: " + b, input);
+ }
+ missingValue = Double.longBitsToDouble(input.readLong());
+ break;
+ case FLOAT:
+ if (b != 1) {
+ throw new CorruptIndexException("invalid missing value flag: " + b, input);
+ }
+ missingValue = Float.intBitsToFloat(input.readInt());
+ break;
+ default:
+ throw new AssertionError("unhandled sortType=" + sortType);
+ }
+ }
+ if (missingValue != null) {
+ sortFields[i].setMissingValue(missingValue);
+ }
+ }
+ indexSort = new Sort(sortFields);
+ } else if (numSortFields < 0) {
+ throw new CorruptIndexException("invalid index sort field count: " + numSortFields, input);
+ } else {
+ indexSort = null;
+ }
+
+ si = new SegmentInfo(dir, version, minVersion, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, indexSort);
+ si.setFiles(files);
+ } catch (Throwable exception) {
+ priorE = exception;
+ } finally {
+ CodecUtil.checkFooter(input, priorE);
+ }
+ return si;
+ }
+ }
+
+ @Override
+ public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {
+ final String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene70SegmentInfoFormat.SI_EXTENSION);
+
+ try (IndexOutput output = dir.createOutput(fileName, ioContext)) {
+ // Only add the file once we've successfully created it, else IFD assert can trip:
+ si.addFile(fileName);
+ CodecUtil.writeIndexHeader(output,
+ Lucene70SegmentInfoFormat.CODEC_NAME,
+ Lucene70SegmentInfoFormat.VERSION_CURRENT,
+ si.getId(),
+ "");
+ Version version = si.getVersion();
+ if (version.major < 7) {
+ throw new IllegalArgumentException("invalid major version: should be >= 7 but got: " + version.major + " segment=" + si);
+ }
+ // Write the Lucene version that created this segment, since 3.1
+ output.writeInt(version.major);
+ output.writeInt(version.minor);
+ output.writeInt(version.bugfix);
+
+ // Write the min Lucene version that contributed docs to the segment, since 7.0
+ if (si.getMinVersion() != null) {
+ output.writeByte((byte) 1);
+ Version minVersion = si.getMinVersion();
+ output.writeInt(minVersion.major);
+ output.writeInt(minVersion.minor);
+ output.writeInt(minVersion.bugfix);
+ } else {
+ output.writeByte((byte) 0);
+ }
+
+ assert version.prerelease == 0;
+ output.writeInt(si.maxDoc());
+
+ output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
+ output.writeMapOfStrings(si.getDiagnostics());
+ Set<String> files = si.files();
+ for (String file : files) {
+ if (!IndexFileNames.parseSegmentName(file).equals(si.name)) {
+ throw new IllegalArgumentException("invalid files: expected segment=" + si.name + ", got=" + files);
+ }
+ }
+ output.writeSetOfStrings(files);
+ output.writeMapOfStrings(si.getAttributes());
+
+ Sort indexSort = si.getIndexSort();
+ int numSortFields = indexSort == null ? 0 : indexSort.getSort().length;
+ output.writeVInt(numSortFields);
+ for (int i = 0; i < numSortFields; ++i) {
+ SortField sortField = indexSort.getSort()[i];
+ SortField.Type sortType = sortField.getType();
+ output.writeString(sortField.getField());
+ int sortTypeID;
+ switch (sortField.getType()) {
+ case STRING:
+ sortTypeID = 0;
+ break;
+ case LONG:
+ sortTypeID = 1;
+ break;
+ case INT:
+ sortTypeID = 2;
+ break;
+ case DOUBLE:
+ sortTypeID = 3;
+ break;
+ case FLOAT:
+ sortTypeID = 4;
+ break;
+ case CUSTOM:
+ if (sortField instanceof SortedSetSortField) {
+ sortTypeID = 5;
+ sortType = SortField.Type.STRING;
+ } else if (sortField instanceof SortedNumericSortField) {
+ sortTypeID = 6;
+ sortType = ((SortedNumericSortField) sortField).getNumericType();
+ } else {
+ throw new IllegalStateException("Unexpected SortedNumericSortField " + sortField);
+ }
+ break;
+ default:
+ throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
+ }
+ output.writeVInt(sortTypeID);
+ if (sortTypeID == 5) {
+ SortedSetSortField ssf = (SortedSetSortField) sortField;
+ if (ssf.getSelector() == SortedSetSelector.Type.MIN) {
+ output.writeByte((byte) 0);
+ } else if (ssf.getSelector() == SortedSetSelector.Type.MAX) {
+ output.writeByte((byte) 1);
+ } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MIN) {
+ output.writeByte((byte) 2);
+ } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MAX) {
+ output.writeByte((byte) 3);
+ } else {
+ throw new IllegalStateException("Unexpected SortedSetSelector type: " + ssf.getSelector());
+ }
+ } else if (sortTypeID == 6) {
+ SortedNumericSortField snsf = (SortedNumericSortField) sortField;
+ if (snsf.getNumericType() == SortField.Type.LONG) {
+ output.writeByte((byte) 0);
+ } else if (snsf.getNumericType() == SortField.Type.INT) {
+ output.writeByte((byte) 1);
+ } else if (snsf.getNumericType() == SortField.Type.DOUBLE) {
+ output.writeByte((byte) 2);
+ } else if (snsf.getNumericType() == SortField.Type.FLOAT) {
+ output.writeByte((byte) 3);
+ } else {
+ throw new IllegalStateException("Unexpected SortedNumericSelector type: " + snsf.getNumericType());
+ }
+ if (snsf.getSelector() == SortedNumericSelector.Type.MIN) {
+ output.writeByte((byte) 0);
+ } else if (snsf.getSelector() == SortedNumericSelector.Type.MAX) {
+ output.writeByte((byte) 1);
+ } else {
+ throw new IllegalStateException("Unexpected sorted numeric selector type: " + snsf.getSelector());
+ }
+ }
+ output.writeByte((byte) (sortField.getReverse() ? 0 : 1));
+
+ // write missing value
+ Object missingValue = sortField.getMissingValue();
+ if (missingValue == null) {
+ output.writeByte((byte) 0);
+ } else {
+ switch(sortType) {
+ case STRING:
+ if (missingValue == SortField.STRING_LAST) {
+ output.writeByte((byte) 1);
+ } else if (missingValue == SortField.STRING_FIRST) {
+ output.writeByte((byte) 2);
+ } else {
+ throw new AssertionError("unrecognized missing value for STRING field \"" + sortField.getField() + "\": " + missingValue);
+ }
+ break;
+ case LONG:
+ output.writeByte((byte) 1);
+ output.writeLong(((Long) missingValue).longValue());
+ break;
+ case INT:
+ output.writeByte((byte) 1);
+ output.writeInt(((Integer) missingValue).intValue());
+ break;
+ case DOUBLE:
+ output.writeByte((byte) 1);
+ output.writeLong(Double.doubleToLongBits(((Double) missingValue).doubleValue()));
+ break;
+ case FLOAT:
+ output.writeByte((byte) 1);
+ output.writeInt(Float.floatToIntBits(((Float) missingValue).floatValue()));
+ break;
+ default:
+ throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
+ }
+ }
+ }
+
+ CodecUtil.writeFooter(output);
+ }
+ }
+
+ /** File extension used to store {@link SegmentInfo}. */
+ public final static String SI_EXTENSION = "si";
+ static final String CODEC_NAME = "Lucene70SegmentInfo";
+ static final int VERSION_START = 0;
+ static final int VERSION_CURRENT = VERSION_START;
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index f3bdfb0..c7ad0f4 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -697,7 +697,7 @@ public final class CheckIndex implements Closeable {
long startOpenReaderNS = System.nanoTime();
if (infoStream != null)
infoStream.print(" test: open reader.........");
- reader = new SegmentReader(info, IOContext.DEFAULT);
+ reader = new SegmentReader(info, sis.getIndexCreatedVersionMajor(), IOContext.DEFAULT);
msg(infoStream, String.format(Locale.ROOT, "OK [took %.3f sec]", nsToSec(System.nanoTime()-startOpenReaderNS)));
segInfoStat.openReaderPassed = true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
index 48901e5..ed50650 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
@@ -178,7 +178,7 @@ class DocumentsWriterPerThread {
assert numDocsInRAM == 0 : "num docs " + numDocsInRAM;
deleteSlice = deleteQueue.newSlice();
- segmentInfo = new SegmentInfo(directoryOrig, Version.LATEST, segmentName, -1, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), indexWriterConfig.getIndexSort());
+ segmentInfo = new SegmentInfo(directoryOrig, Version.LATEST, Version.LATEST, segmentName, -1, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), indexWriterConfig.getIndexSort());
assert numDocsInRAM == 0;
if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
infoStream.message("DWPT", Thread.currentThread().getName() + " init seg=" + segmentName + " delQueue=" + deleteQueue);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
index 5949fca..fd36ecb 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
@@ -27,7 +27,6 @@ import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.PointsReader;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
-import org.apache.lucene.search.Sort;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
@@ -104,8 +103,8 @@ public abstract class FilterCodecReader extends CodecReader {
}
@Override
- public Sort getIndexSort() {
- return in.getIndexSort();
+ public LeafMetaData getMetaData() {
+ return in.getMetaData();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
index 0a3ec7f..f3d8112 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Iterator;
-import org.apache.lucene.search.Sort;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -398,9 +397,9 @@ public abstract class FilterLeafReader extends LeafReader {
}
@Override
- public Sort getIndexSort() {
+ public LeafMetaData getMetaData() {
ensureOpen();
- return in.getIndexSort();
+ return in.getMetaData();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 899643a..9a29150 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -30,7 +30,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map.Entry;
-import java.util.Objects;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
@@ -855,7 +854,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// against an index that's currently open for
// searching. In this case we write the next
// segments_N file with no segments:
- final SegmentInfos sis = new SegmentInfos(Version.LATEST);
+ final SegmentInfos sis = new SegmentInfos(Version.LATEST.major);
try {
final SegmentInfos previous = SegmentInfos.readLatestCommit(directory);
sis.updateGenerationVersionAndCounter(previous);
@@ -2654,12 +2653,12 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
infoStream.message("IW", "addIndexes: process directory " + dir);
}
SegmentInfos sis = SegmentInfos.readLatestCommit(dir); // read infos from dir
- if (Objects.equals(segmentInfos.getIndexCreatedVersion(), sis.getIndexCreatedVersion()) == false) {
+ if (segmentInfos.getIndexCreatedVersionMajor() != sis.getIndexCreatedVersionMajor()) {
throw new IllegalArgumentException("Cannot use addIndexes(Directory) with indexes that have been created "
- + "by a different Lucene version. The current index was generated by "
- + segmentInfos.getIndexCreatedVersion()
- + " while one of the directories contains an index that was generated with "
- + sis.getIndexCreatedVersion());
+ + "by a different Lucene version. The current index was generated by Lucene "
+ + segmentInfos.getIndexCreatedVersionMajor()
+ + " while one of the directories contains an index that was generated with Lucene "
+ + sis.getIndexCreatedVersionMajor());
}
totalMaxDoc += sis.totalMaxDoc();
commits.add(sis);
@@ -2747,7 +2746,26 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
return seqNo;
}
-
+
+ private void validateMergeReader(CodecReader leaf) {
+ LeafMetaData segmentMeta = leaf.getMetaData();
+ if (segmentInfos.getIndexCreatedVersionMajor() != segmentMeta.getCreatedVersionMajor()) {
+ throw new IllegalArgumentException("Cannot merge a segment that has been created with major version "
+ + segmentMeta.getCreatedVersionMajor() + " into this index which has been created by major version "
+ + segmentInfos.getIndexCreatedVersionMajor());
+ }
+
+ if (segmentInfos.getIndexCreatedVersionMajor() >= 7 && segmentMeta.getMinVersion() == null) {
+ throw new IllegalStateException("Indexes created on or after Lucene 7 must record the created version major, but " + leaf + " hides it");
+ }
+
+ Sort leafIndexSort = segmentMeta.getSort();
+ if (config.getIndexSort() != null && leafIndexSort != null
+ && config.getIndexSort().equals(leafIndexSort) == false) {
+ throw new IllegalArgumentException("cannot change index sort from " + leafIndexSort + " to " + config.getIndexSort());
+ }
+ }
+
/**
* Merges the provided indexes into this index.
*
@@ -2801,12 +2819,10 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
flush(false, true);
String mergedName = newSegmentName();
+
for (CodecReader leaf : readers) {
numDocs += leaf.numDocs();
- Sort leafIndexSort = leaf.getIndexSort();
- if (indexSort != null && leafIndexSort != null && indexSort.equals(leafIndexSort) == false) {
- throw new IllegalArgumentException("cannot change index sort from " + leafIndexSort + " to " + indexSort);
- }
+ validateMergeReader(leaf);
}
// Best-effort up front check:
@@ -2818,7 +2834,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// abortable so that IW.close(false) is able to stop it
TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory);
- SegmentInfo info = new SegmentInfo(directoryOrig, Version.LATEST, mergedName, -1,
+ // We set the min version to null for now, it will be set later by SegmentMerger
+ SegmentInfo info = new SegmentInfo(directoryOrig, Version.LATEST, null, mergedName, -1,
false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), config.getIndexSort());
SegmentMerger merger = new SegmentMerger(Arrays.asList(readers), info, infoStream, trackingDir,
@@ -2907,7 +2924,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
//System.out.println("copy seg=" + info.info.name + " version=" + info.info.getVersion());
// Same SI as before but we change directory and name
- SegmentInfo newInfo = new SegmentInfo(directoryOrig, info.info.getVersion(), segName, info.info.maxDoc(),
+ SegmentInfo newInfo = new SegmentInfo(directoryOrig, info.info.getVersion(), info.info.getMinVersion(), segName, info.info.maxDoc(),
info.info.getUseCompoundFile(), info.info.getCodec(),
info.info.getDiagnostics(), info.info.getId(), info.info.getAttributes(), info.info.getIndexSort());
SegmentCommitInfo newInfoPerCommit = new SegmentCommitInfo(newInfo, info.getDelCount(), info.getDelGen(),
@@ -4117,7 +4134,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// ConcurrentMergePolicy we keep deterministic segment
// names.
final String mergeSegmentName = newSegmentName();
- SegmentInfo si = new SegmentInfo(directoryOrig, Version.LATEST, mergeSegmentName, -1, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), config.getIndexSort());
+ // We set the min version to null for now, it will be set later by SegmentMerger
+ SegmentInfo si = new SegmentInfo(directoryOrig, Version.LATEST, null, mergeSegmentName, -1, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), config.getIndexSort());
Map<String,String> details = new HashMap<>();
details.put("mergeMaxNumSegments", "" + merge.maxNumSegments);
details.put("mergeFactor", Integer.toString(merge.segments.size()));
@@ -4322,7 +4340,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// Let the merge wrap readers
List<CodecReader> mergeReaders = new ArrayList<>();
for (SegmentReader reader : merge.readers) {
- mergeReaders.add(merge.wrapForMerge(reader));
+ CodecReader wrappedReader = merge.wrapForMerge(reader);
+ validateMergeReader(wrappedReader);
+ mergeReaders.add(wrappedReader);
}
final SegmentMerger merger = new SegmentMerger(mergeReaders,
merge.info.info, infoStream, dirWrapper,
@@ -4608,7 +4628,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// For infoStream output
synchronized SegmentInfos toLiveInfos(SegmentInfos sis) {
- final SegmentInfos newSIS = new SegmentInfos(sis.getIndexCreatedVersion());
+ final SegmentInfos newSIS = new SegmentInfos(sis.getIndexCreatedVersionMajor());
final Map<SegmentCommitInfo,SegmentCommitInfo> liveSIS = new HashMap<>();
for(SegmentCommitInfo info : segmentInfos) {
liveSIS.put(info, info);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/index/LeafMetaData.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LeafMetaData.java b/lucene/core/src/java/org/apache/lucene/index/LeafMetaData.java
new file mode 100644
index 0000000..567d43e
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LeafMetaData.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.util.Version;
+
+/**
+ * Provides read-only metadata about a leaf.
+ * @lucene.experimental
+ */
+public final class LeafMetaData {
+
+ private final int createdVersionMajor;
+ private final Version minVersion;
+ private final Sort sort;
+
+ /** Expert: Sole constructor. Public for use by custom {@link LeafReader} impls. */
+ public LeafMetaData(int createdVersionMajor, Version minVersion, Sort sort) {
+ this.createdVersionMajor = createdVersionMajor;
+ if (createdVersionMajor > Version.LATEST.major) {
+ throw new IllegalArgumentException("createdVersionMajor is in the future: " + createdVersionMajor);
+ }
+ if (createdVersionMajor < 6) {
+ throw new IllegalArgumentException("createdVersionMajor must be >= 6, got: " + createdVersionMajor);
+ }
+ if (minVersion != null && minVersion.onOrAfter(Version.LUCENE_7_0_0) == false) {
+ throw new IllegalArgumentException("minVersion must be >= 7.0.0: " + minVersion);
+ }
+ if (createdVersionMajor >= 7 && minVersion == null) {
+ throw new IllegalArgumentException("minVersion must be set when createdVersionMajor is >= 7");
+ }
+ this.minVersion = minVersion;
+ this.sort = sort;
+ }
+
+ /** Get the Lucene version that created this index. This can be used to implement
+ * backward compatibility on top of the codec API. A return value of {@code 6}
+ * indicates that the created version is unknown. */
+ public int getCreatedVersionMajor() {
+ return createdVersionMajor;
+ }
+
+ /**
+ * Return the minimum Lucene version that contributed documents to this index,
+ * or {@code null} if this information is not available.
+ */
+ public Version getMinVersion() {
+ return minVersion;
+ }
+
+ /**
+ * Return the order in which documents from this index are sorted, or
+ * {@code null} if documents are in no particular order.
+ */
+ public Sort getSort() {
+ return sort;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
index 13c8646..c738bc5 100644
--- a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.index.IndexReader.CacheHelper;
-import org.apache.lucene.search.Sort;
import org.apache.lucene.util.Bits;
/** {@code LeafReader} is an abstract class, providing an interface for accessing an
@@ -246,6 +245,8 @@ public abstract class LeafReader extends IndexReader {
*/
public abstract void checkIntegrity() throws IOException;
- /** Returns null if this leaf is unsorted, or the {@link Sort} that it was sorted by */
- public abstract Sort getIndexSort();
+ /**
+ * Return metadata about this leaf.
+ * @lucene.experimental */
+ public abstract LeafMetaData getMetaData();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
index fffb693..3a3573a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
@@ -24,7 +24,6 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
-import org.apache.lucene.search.Sort;
import org.apache.lucene.util.Bits;
/** This is a hack to make index sorting fast, with a {@link LeafReader} that always returns merge instances when you ask for the codec readers. */
@@ -235,7 +234,7 @@ class MergeReaderWrapper extends LeafReader {
}
@Override
- public Sort getIndexSort() {
- return in.getIndexSort();
+ public LeafMetaData getMetaData() {
+ return in.getMetaData();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23b002a0/lucene/core/src/java/org/apache/lucene/index/MergeState.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeState.java b/lucene/core/src/java/org/apache/lucene/index/MergeState.java
index a7c8307..9ad69f6 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergeState.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergeState.java
@@ -231,7 +231,7 @@ public class MergeState {
List<CodecReader> readers = new ArrayList<>(originalReaders.size());
for (CodecReader leaf : originalReaders) {
- Sort segmentSort = leaf.getIndexSort();
+ Sort segmentSort = leaf.getMetaData().getSort();
if (segmentSort == null) {
// This segment was written by flush, so documents are not yet sorted, so we sort them now: