You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/09/27 12:23:25 UTC
svn commit: r1627943 [1/2] - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/
lucene/backward-codecs/src/java/org/apache/...
Author: rmuir
Date: Sat Sep 27 10:23:23 2014
New Revision: 1627943
URL: http://svn.apache.org/r1627943
Log:
LUCENE-5969, LUCENE-5412: add more infos/metadata safety
Added:
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/BitVector.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/BitVector.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWSegmentInfoFormat.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWSegmentInfoFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWSegmentInfoFormat.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWSegmentInfoFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50LiveDocsFormat.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50LiveDocsFormat.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package.html
- copied unchanged from r1627941, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package.html
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
- copied unchanged from r1627941, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
Removed:
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46Codec.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosWriter.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWCodec.java
lucene/dev/branches/branch_5x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java
lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene410/package.html
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosReader.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoReader.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/StringHelper.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
lucene/dev/branches/branch_5x/lucene/misc/ (props changed)
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyFieldInfosFormat.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/solrj/ (props changed)
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java Sat Sep 27 10:23:23 2014
@@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -49,8 +50,8 @@ class Lucene40FieldInfosReader extends F
}
@Override
- public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
+ public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
+ final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
IndexInput input = directory.openInput(fileName, iocontext);
boolean success = false;
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java Sat Sep 27 10:23:23 2014
@@ -72,7 +72,6 @@ import org.apache.lucene.store.DataOutpu
@Deprecated
public class Lucene40SegmentInfoFormat extends SegmentInfoFormat {
private final SegmentInfoReader reader = new Lucene40SegmentInfoReader();
- private final SegmentInfoWriter writer = new Lucene40SegmentInfoWriter();
/** Sole constructor. */
public Lucene40SegmentInfoFormat() {
@@ -83,11 +82,9 @@ public class Lucene40SegmentInfoFormat e
return reader;
}
- // we must unfortunately support write, to allow addIndexes to write a new .si with rewritten filenames:
- // see LUCENE-5377
@Override
public SegmentInfoWriter getSegmentInfoWriter() {
- return writer;
+ throw new UnsupportedOperationException("this codec can only be used for reading");
}
/** File extension used to store {@link SegmentInfo}. */
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java Sat Sep 27 10:23:23 2014
@@ -73,7 +73,7 @@ public class Lucene40SegmentInfoReader e
CodecUtil.checkEOF(input);
- final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
+ final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, null);
si.setFiles(files);
success = true;
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java Sat Sep 27 10:23:23 2014
@@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -49,8 +50,8 @@ final class Lucene42FieldInfosReader ext
}
@Override
- public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene42FieldInfosFormat.EXTENSION);
+ public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
+ final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, "", Lucene42FieldInfosFormat.EXTENSION);
IndexInput input = directory.openInput(fileName, iocontext);
boolean success = false;
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46Codec.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46Codec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46Codec.java Sat Sep 27 10:23:23 2014
@@ -100,7 +100,7 @@ public class Lucene46Codec extends Codec
}
@Override
- public final SegmentInfoFormat segmentInfoFormat() {
+ public SegmentInfoFormat segmentInfoFormat() {
return segmentInfosFormat;
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java Sat Sep 27 10:23:23 2014
@@ -99,7 +99,7 @@ public class Lucene49Codec extends Codec
}
@Override
- public final SegmentInfoFormat segmentInfoFormat() {
+ public SegmentInfoFormat segmentInfoFormat() {
return segmentInfosFormat;
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java Sat Sep 27 10:23:23 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@@ -45,11 +46,11 @@ public class Lucene40FieldInfosWriter ex
}
@Override
- public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
+ public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
if (!segmentSuffix.isEmpty()) {
throw new UnsupportedOperationException("4.0 does not support fieldinfo updates");
}
- final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
+ final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
IndexOutput output = directory.createOutput(fileName, context);
boolean success = false;
try {
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java Sat Sep 27 10:23:23 2014
@@ -7,9 +7,9 @@ import org.apache.lucene.codecs.FieldInf
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.util.LuceneTestCase;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -74,4 +74,11 @@ public final class Lucene40RWCodec exten
public PostingsFormat getPostingsFormatForField(String field) {
return postings;
}
+
+ private static final SegmentInfoFormat segmentInfos = new Lucene40RWSegmentInfoFormat();
+
+ @Override
+ public SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfos;
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java Sat Sep 27 10:23:23 2014
@@ -6,12 +6,14 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosWriter;
import org.apache.lucene.codecs.lucene40.Lucene40RWDocValuesFormat;
import org.apache.lucene.codecs.lucene40.Lucene40RWNormsFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40RWSegmentInfoFormat;
import org.apache.lucene.codecs.lucene40.Lucene40RWTermVectorsFormat;
import org.apache.lucene.util.LuceneTestCase;
@@ -73,4 +75,11 @@ public class Lucene41RWCodec extends Luc
public TermVectorsFormat termVectorsFormat() {
return vectors;
}
+
+ private static final SegmentInfoFormat segmentInfos = new Lucene40RWSegmentInfoFormat();
+
+ @Override
+ public SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfos;
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosWriter.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosWriter.java Sat Sep 27 10:23:23 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@@ -45,11 +46,11 @@ public final class Lucene42FieldInfosWri
}
@Override
- public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
+ public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
if (!segmentSuffix.isEmpty()) {
throw new UnsupportedOperationException("4.2 does not support fieldinfo updates");
}
- final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene42FieldInfosFormat.EXTENSION);
+ final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, "", Lucene42FieldInfosFormat.EXTENSION);
IndexOutput output = directory.createOutput(fileName, context);
boolean success = false;
try {
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java Sat Sep 27 10:23:23 2014
@@ -23,6 +23,8 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40RWSegmentInfoFormat;
import org.apache.lucene.util.LuceneTestCase;
/**
@@ -55,4 +57,11 @@ public class Lucene42RWCodec extends Luc
public FieldInfosFormat fieldInfosFormat() {
return fieldInfosFormat;
}
+
+ private static final SegmentInfoFormat segmentInfos = new Lucene40RWSegmentInfoFormat();
+
+ @Override
+ public SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfos;
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java Sat Sep 27 10:23:23 2014
@@ -23,6 +23,8 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40RWSegmentInfoFormat;
import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosFormat;
import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosWriter;
import org.apache.lucene.codecs.lucene42.Lucene42RWNormsFormat;
@@ -58,4 +60,11 @@ public class Lucene45RWCodec extends Luc
public NormsFormat normsFormat() {
return norms;
}
+
+ private static final SegmentInfoFormat segmentInfos = new Lucene40RWSegmentInfoFormat();
+
+ @Override
+ public SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfos;
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java Sat Sep 27 10:23:23 2014
@@ -19,6 +19,8 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.SegmentInfoWriter;
import org.apache.lucene.codecs.lucene42.Lucene42RWNormsFormat;
import org.apache.lucene.codecs.lucene45.Lucene45RWDocValuesFormat;
@@ -41,4 +43,11 @@ public class Lucene46RWCodec extends Luc
public NormsFormat normsFormat() {
return norms;
}
+
+ private static final SegmentInfoFormat segmentInfos = new Lucene46RWSegmentInfoFormat();
+
+ @Override
+ public SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfos;
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWCodec.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWCodec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWCodec.java Sat Sep 27 10:23:23 2014
@@ -19,6 +19,8 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.lucene46.Lucene46RWSegmentInfoFormat;
/**
* Read-write version of {@link Lucene49Codec} for testing.
@@ -39,4 +41,11 @@ public class Lucene49RWCodec extends Luc
public NormsFormat normsFormat() {
return norms;
}
+
+ private static final SegmentInfoFormat segmentInfos = new Lucene46RWSegmentInfoFormat();
+
+ @Override
+ public SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfos;
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java Sat Sep 27 10:23:23 2014
@@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -47,8 +48,8 @@ import static org.apache.lucene.codecs.s
public class SimpleTextFieldInfosReader extends FieldInfosReader {
@Override
- public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION);
+ public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
+ final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
ChecksumIndexInput input = directory.openChecksumInput(fileName, iocontext);
BytesRefBuilder scratch = new BytesRefBuilder();
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java Sat Sep 27 10:23:23 2014
@@ -25,6 +25,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@@ -61,8 +62,8 @@ public class SimpleTextFieldInfosWriter
final static BytesRef ATT_VALUE = new BytesRef(" value ");
@Override
- public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION);
+ public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
+ final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
IndexOutput out = directory.createOutput(fileName, context);
BytesRefBuilder scratch = new BytesRefBuilder();
boolean success = false;
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java Sat Sep 27 10:23:23 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.simplet
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -109,7 +110,7 @@ public class SimpleTextSegmentInfoReader
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), SI_ID);
- final String id = readString(SI_ID.length, scratch);
+ final byte[] id = Arrays.copyOfRange(scratch.bytes(), SI_ID.length, scratch.length());
SimpleTextUtil.checkFooter(input);
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java Sat Sep 27 10:23:23 2014
@@ -107,7 +107,7 @@ public class SimpleTextSegmentInfoWriter
}
SimpleTextUtil.write(output, SI_ID);
- SimpleTextUtil.write(output, si.getId(), scratch);
+ SimpleTextUtil.write(output, new BytesRef(si.getId()));
SimpleTextUtil.writeNewline(output);
SimpleTextUtil.writeChecksum(output, scratch);
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java Sat Sep 27 10:23:23 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFormatTooNewException;
@@ -31,6 +32,7 @@ import org.apache.lucene.store.IndexInpu
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.StringHelper;
/**
* Utility class for reading and writing versioned headers.
@@ -80,8 +82,7 @@ public final class CodecUtil {
* @throws IOException If there is an I/O error writing to the underlying medium.
* @throws IllegalArgumentException If the codec name is not simple ASCII, or is more than 127 characters in length
*/
- public static void writeHeader(DataOutput out, String codec, int version)
- throws IOException {
+ public static void writeHeader(DataOutput out, String codec, int version) throws IOException {
BytesRef bytes = new BytesRef(codec);
if (bytes.length != codec.length() || bytes.length >= 128) {
throw new IllegalArgumentException("codec must be simple ASCII, less than 128 characters in length [got " + codec + "]");
@@ -90,6 +91,40 @@ public final class CodecUtil {
out.writeString(codec);
out.writeInt(version);
}
+
+ /**
+ * Writes a codec header for a per-segment, which records both a string to
+ * identify the file, a version number, and the unique ID of the segment.
+ * This header can be parsed and validated with
+ * {@link #checkSegmentHeader(DataInput, String, int, int, byte[]) checkSegmentHeader()}.
+ * <p>
+ * CodecSegmentHeader --> CodecHeader,SegmentID
+ * <ul>
+ * <li>CodecHeader --> {@link #writeHeader}
+ * <li>SegmentID --> {@link DataOutput#writeByte byte}<sup>16</sup>.
+ * Unique identifier for the segment.
+ * </ul>
+ * <p>
+ * Note that the length of a segment header depends only upon the
+ * name of the codec, so this length can be computed at any time
+ * with {@link #headerLength(String)}.
+ *
+ * @param out Output stream
+ * @param codec String to identify this file. It should be simple ASCII,
+ * less than 128 characters in length.
+ * @param segmentID Unique identifier for the segment
+ * @param version Version number
+ * @throws IOException If there is an I/O error writing to the underlying medium.
+ * @throws IllegalArgumentException If the codec name is not simple ASCII, or
+ * is more than 127 characters in length, or if segmentID is invalid.
+ */
+ public static void writeSegmentHeader(DataOutput out, String codec, int version, byte[] segmentID) throws IOException {
+ if (segmentID.length != StringHelper.ID_LENGTH) {
+ throw new IllegalArgumentException("Invalid id: " + StringHelper.idToString(segmentID));
+ }
+ writeHeader(out, codec, version);
+ out.writeBytes(segmentID, 0, segmentID.length);
+ }
/**
* Computes the length of a codec header.
@@ -101,6 +136,17 @@ public final class CodecUtil {
public static int headerLength(String codec) {
return 9+codec.length();
}
+
+ /**
+ * Computes the length of a segment header.
+ *
+ * @param codec Codec name.
+ * @return length of the entire segment header.
+ * @see #writeSegmentHeader(DataOutput, String, int, byte[])
+ */
+ public static int segmentHeaderLength(String codec) {
+ return headerLength(codec) + StringHelper.ID_LENGTH;
+ }
/**
* Reads and validates a header previously written with
@@ -129,9 +175,7 @@ public final class CodecUtil {
* @throws IOException If there is an I/O error reading from the underlying medium.
* @see #writeHeader(DataOutput, String, int)
*/
- public static int checkHeader(DataInput in, String codec, int minVersion, int maxVersion)
- throws IOException {
-
+ public static int checkHeader(DataInput in, String codec, int minVersion, int maxVersion) throws IOException {
// Safety to guard against reading a bogus string:
final int actualHeader = in.readInt();
if (actualHeader != CODEC_MAGIC) {
@@ -162,6 +206,48 @@ public final class CodecUtil {
}
/**
+ * Reads and validates a header previously written with
+ * {@link #writeSegmentHeader(DataOutput, String, int, byte[])}.
+ * <p>
+ * When reading a file, supply the expected <code>codec</code>,
+ * expected version range (<code>minVersion to maxVersion</code>),
+ * and segment ID.
+ *
+ * @param in Input stream, positioned at the point where the
+ * header was previously written. Typically this is located
+ * at the beginning of the file.
+ * @param codec The expected codec name.
+ * @param minVersion The minimum supported expected version number.
+ * @param maxVersion The maximum supported expected version number.
+ * @param segmentID The expected segment this file belongs to.
+ * @return The actual version found, when a valid header is found
+ * that matches <code>codec</code>, with an actual version
+ * where <code>minVersion <= actual <= maxVersion</code>,
+ * and matching <code>segmentID</code>
+ * Otherwise an exception is thrown.
+ * @throws CorruptIndexException If the first four bytes are not
+ * {@link #CODEC_MAGIC}, or if the actual codec found is
+ * not <code>codec</code>, or if the <code>segmentID</code>
+ * does not match.
+ * @throws IndexFormatTooOldException If the actual version is less
+ * than <code>minVersion</code>.
+ * @throws IndexFormatTooNewException If the actual version is greater
+ * than <code>maxVersion</code>.
+ * @throws IOException If there is an I/O error reading from the underlying medium.
+ * @see #writeSegmentHeader(DataOutput, String, int, byte[])
+ */
+ public static int checkSegmentHeader(DataInput in, String codec, int minVersion, int maxVersion, byte[] segmentID) throws IOException {
+ int version = checkHeader(in, codec, minVersion, maxVersion);
+ byte id[] = new byte[StringHelper.ID_LENGTH];
+ in.readBytes(id, 0, id.length);
+ if (!Arrays.equals(id, segmentID)) {
+ throw new CorruptIndexException("file mismatch, expected segment id=" + StringHelper.idToString(segmentID)
+ + ", got=" + StringHelper.idToString(id), in);
+ }
+ return version;
+ }
+
+ /**
* Writes a codec footer, which records both a checksum
* algorithm ID and a checksum. This footer can
* be parsed and validated with
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java Sat Sep 27 10:23:23 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs;
import java.io.IOException;
import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -35,5 +36,5 @@ public abstract class FieldInfosReader {
/** Read the {@link FieldInfos} previously written with {@link
* FieldInfosWriter}. */
- public abstract FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException;
+ public abstract FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException;
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java Sat Sep 27 10:23:23 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs;
import java.io.IOException;
import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -35,5 +36,5 @@ public abstract class FieldInfosWriter {
/** Writes the provided {@link FieldInfos} to the
* directory. */
- public abstract void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException;
+ public abstract void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException;
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene410/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene410/package.html?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene410/package.html (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene410/package.html Sat Sep 27 10:23:23 2014
@@ -21,384 +21,5 @@
</head>
<body>
Lucene 4.10 file format.
-
-<h1>Apache Lucene - Index File Formats</h1>
-<div>
-<ul>
-<li><a href="#Introduction">Introduction</a></li>
-<li><a href="#Definitions">Definitions</a>
- <ul>
- <li><a href="#Inverted_Indexing">Inverted Indexing</a></li>
- <li><a href="#Types_of_Fields">Types of Fields</a></li>
- <li><a href="#Segments">Segments</a></li>
- <li><a href="#Document_Numbers">Document Numbers</a></li>
- </ul>
-</li>
-<li><a href="#Overview">Index Structure Overview</a></li>
-<li><a href="#File_Naming">File Naming</a></li>
-<li><a href="#file-names">Summary of File Extensions</a></li>
- <ul>
- <li><a href="#Lock_File">Lock File</a></li>
- <li><a href="#History">History</a></li>
- <li><a href="#Limitations">Limitations</a></li>
- </ul>
-</ul>
-</div>
-<a name="Introduction"></a>
-<h2>Introduction</h2>
-<div>
-<p>This document defines the index file formats used in this version of Lucene.
-If you are using a different version of Lucene, please consult the copy of
-<code>docs/</code> that was distributed with
-the version you are using.</p>
-<p>Apache Lucene is written in Java, but several efforts are underway to write
-<a href="http://wiki.apache.org/lucene-java/LuceneImplementations">versions of
-Lucene in other programming languages</a>. If these versions are to remain
-compatible with Apache Lucene, then a language-independent definition of the
-Lucene index format is required. This document thus attempts to provide a
-complete and independent definition of the Apache Lucene file formats.</p>
-<p>As Lucene evolves, this document should evolve. Versions of Lucene in
-different programming languages should endeavor to agree on file formats, and
-generate new versions of this document.</p>
-</div>
-<a name="Definitions" id="Definitions"></a>
-<h2>Definitions</h2>
-<div>
-<p>The fundamental concepts in Lucene are index, document, field and term.</p>
-<p>An index contains a sequence of documents.</p>
-<ul>
-<li>A document is a sequence of fields.</li>
-<li>A field is a named sequence of terms.</li>
-<li>A term is a sequence of bytes.</li>
-</ul>
-<p>The same sequence of bytes in two different fields is considered a different
-term. Thus terms are represented as a pair: the string naming the field, and the
-bytes within the field.</p>
-<a name="Inverted_Indexing"></a>
-<h3>Inverted Indexing</h3>
-<p>The index stores statistics about terms in order to make term-based search
-more efficient. Lucene's index falls into the family of indexes known as an
-<i>inverted index.</i> This is because it can list, for a term, the documents
-that contain it. This is the inverse of the natural relationship, in which
-documents list terms.</p>
-<a name="Types_of_Fields"></a>
-<h3>Types of Fields</h3>
-<p>In Lucene, fields may be <i>stored</i>, in which case their text is stored
-in the index literally, in a non-inverted manner. Fields that are inverted are
-called <i>indexed</i>. A field may be both stored and indexed.</p>
-<p>The text of a field may be <i>tokenized</i> into terms to be indexed, or the
-text of a field may be used literally as a term to be indexed. Most fields are
-tokenized, but sometimes it is useful for certain identifier fields to be
-indexed literally.</p>
-<p>See the {@link org.apache.lucene.document.Field Field}
-java docs for more information on Fields.</p>
-<a name="Segments" id="Segments"></a>
-<h3>Segments</h3>
-<p>Lucene indexes may be composed of multiple sub-indexes, or <i>segments</i>.
-Each segment is a fully independent index, which could be searched separately.
-Indexes evolve by:</p>
-<ol>
-<li>Creating new segments for newly added documents.</li>
-<li>Merging existing segments.</li>
-</ol>
-<p>Searches may involve multiple segments and/or multiple indexes, each index
-potentially composed of a set of segments.</p>
-<a name="Document_Numbers"></a>
-<h3>Document Numbers</h3>
-<p>Internally, Lucene refers to documents by an integer <i>document number</i>.
-The first document added to an index is numbered zero, and each subsequent
-document added gets a number one greater than the previous.</p>
-<p>Note that a document's number may change, so caution should be taken when
-storing these numbers outside of Lucene. In particular, numbers may change in
-the following situations:</p>
-<ul>
-<li>
-<p>The numbers stored in each segment are unique only within the segment, and
-must be converted before they can be used in a larger context. The standard
-technique is to allocate each segment a range of values, based on the range of
-numbers used in that segment. To convert a document number from a segment to an
-external value, the segment's <i>base</i> document number is added. To convert
-an external value back to a segment-specific value, the segment is identified
-by the range that the external value is in, and the segment's base value is
-subtracted. For example two five document segments might be combined, so that
-the first segment has a base value of zero, and the second of five. Document
-three from the second segment would have an external value of eight.</p>
-</li>
-<li>
-<p>When documents are deleted, gaps are created in the numbering. These are
-eventually removed as the index evolves through merging. Deleted documents are
-dropped when segments are merged. A freshly-merged segment thus has no gaps in
-its numbering.</p>
-</li>
-</ul>
-</div>
-<a name="Overview" id="Overview"></a>
-<h2>Index Structure Overview</h2>
-<div>
-<p>Each segment index maintains the following:</p>
-<ul>
-<li>
-{@link org.apache.lucene.codecs.lucene46.Lucene46SegmentInfoFormat Segment info}.
- This contains metadata about a segment, such as the number of documents,
- what files it uses,
-</li>
-<li>
-{@link org.apache.lucene.codecs.lucene46.Lucene46FieldInfosFormat Field names}.
- This contains the set of field names used in the index.
-</li>
-<li>
-{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Stored Field values}.
-This contains, for each document, a list of attribute-value pairs, where the attributes
-are field names. These are used to store auxiliary information about the document, such as
-its title, url, or an identifier to access a database. The set of stored fields are what is
-returned for each hit when searching. This is keyed by document number.
-</li>
-<li>
-{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term dictionary}.
-A dictionary containing all of the terms used in all of the
-indexed fields of all of the documents. The dictionary also contains the number
-of documents which contain the term, and pointers to the term's frequency and
-proximity data.
-</li>
-<li>
-{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Frequency data}.
-For each term in the dictionary, the numbers of all the
-documents that contain that term, and the frequency of the term in that
-document, unless frequencies are omitted (IndexOptions.DOCS_ONLY)
-</li>
-<li>
-{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Proximity data}.
-For each term in the dictionary, the positions that the
-term occurs in each document. Note that this will not exist if all fields in
-all documents omit position data.
-</li>
-<li>
-{@link org.apache.lucene.codecs.lucene49.Lucene49NormsFormat Normalization factors}.
-For each field in each document, a value is stored
-that is multiplied into the score for hits on that field.
-</li>
-<li>
-{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vectors}.
-For each field in each document, the term vector (sometimes
-called document vector) may be stored. A term vector consists of term text and
-term frequency. To add Term Vectors to your index see the
-{@link org.apache.lucene.document.Field Field} constructors
-</li>
-<li>
-{@link org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat Per-document values}.
-Like stored values, these are also keyed by document
-number, but are generally intended to be loaded into main memory for fast
-access. Whereas stored values are generally intended for summary results from
-searches, per-document values are useful for things like scoring factors.
-</li>
-<li>
-{@link org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat Deleted documents}.
-An optional file indicating which documents are deleted.
-</li>
-</ul>
-<p>Details on each of these are provided in their linked pages.</p>
-</div>
-<a name="File_Naming"></a>
-<h2>File Naming</h2>
-<div>
-<p>All files belonging to a segment have the same name with varying extensions.
-The extensions correspond to the different file formats described below. When
-using the Compound File format (default in 1.4 and greater) these files (except
-for the Segment info file, the Lock file, and Deleted documents file) are collapsed
-into a single .cfs file (see below for details)</p>
-<p>Typically, all segments in an index are stored in a single directory,
-although this is not required.</p>
-<p>As of version 2.1 (lock-less commits), file names are never re-used.
-That is, when any file is saved
-to the Directory it is given a never before used filename. This is achieved
-using a simple generations approach. For example, the first segments file is
-segments_1, then segments_2, etc. The generation is a sequential long integer
-represented in alpha-numeric (base 36) form.</p>
-</div>
-<a name="file-names" id="file-names"></a>
-<h2>Summary of File Extensions</h2>
-<div>
-<p>The following table summarizes the names and extensions of the files in
-Lucene:</p>
-<table cellspacing="1" cellpadding="4">
-<tr>
-<th>Name</th>
-<th>Extension</th>
-<th>Brief Description</th>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.index.SegmentInfos Segments File}</td>
-<td>segments_N</td>
-<td>Stores information about a commit point</td>
-</tr>
-<tr>
-<td><a href="#Lock_File">Lock File</a></td>
-<td>write.lock</td>
-<td>The Write lock prevents multiple IndexWriters from writing to the same
-file.</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene46.Lucene46SegmentInfoFormat Segment Info}</td>
-<td>.si</td>
-<td>Stores metadata about a segment</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.store.CompoundFileDirectory Compound File}</td>
-<td>.cfs, .cfe</td>
-<td>An optional "virtual" file consisting of all the other index files for
-systems that frequently run out of file handles.</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene46.Lucene46FieldInfosFormat Fields}</td>
-<td>.fnm</td>
-<td>Stores information about the fields</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Field Index}</td>
-<td>.fdx</td>
-<td>Contains pointers to field data</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Field Data}</td>
-<td>.fdt</td>
-<td>The stored fields for documents</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Dictionary}</td>
-<td>.tim</td>
-<td>The term dictionary, stores term info</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Index}</td>
-<td>.tip</td>
-<td>The index into the Term Dictionary</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Frequencies}</td>
-<td>.doc</td>
-<td>Contains the list of docs which contain each term along with frequency</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Positions}</td>
-<td>.pos</td>
-<td>Stores position information about where a term occurs in the index</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Payloads}</td>
-<td>.pay</td>
-<td>Stores additional per-position metadata information such as character offsets and user payloads</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene49.Lucene49NormsFormat Norms}</td>
-<td>.nvd, .nvm</td>
-<td>Encodes length and boost factors for docs and fields</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat Per-Document Values}</td>
-<td>.dvd, .dvm</td>
-<td>Encodes additional scoring factors or other per-document information.</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vector Index}</td>
-<td>.tvx</td>
-<td>Stores offset into the document data file</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vector Documents}</td>
-<td>.tvd</td>
-<td>Contains information about each document that has term vectors</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vector Fields}</td>
-<td>.tvf</td>
-<td>The field level info about term vectors</td>
-</tr>
-<tr>
-<td>{@link org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat Deleted Documents}</td>
-<td>.del</td>
-<td>Info about what files are deleted</td>
-</tr>
-</table>
-</div>
-<a name="Lock_File" id="Lock_File"></a>
-<h2>Lock File</h2>
-The write lock, which is stored in the index directory by default, is named
-"write.lock". If the lock directory is different from the index directory then
-the write lock will be named "XXXX-write.lock" where XXXX is a unique prefix
-derived from the full path to the index directory. When this file is present, a
-writer is currently modifying the index (adding or removing documents). This
-lock file ensures that only one writer is modifying the index at a time.</p>
-<a name="History"></a>
-<h2>History</h2>
-<p>Compatibility notes are provided in this document, describing how file
-formats have changed from prior versions:</p>
-<ul>
-<li>In version 2.1, the file format was changed to allow lock-less commits (ie,
-no more commit lock). The change is fully backwards compatible: you can open a
-pre-2.1 index for searching or adding/deleting of docs. When the new segments
-file is saved (committed), it will be written in the new file format (meaning
-no specific "upgrade" process is needed). But note that once a commit has
-occurred, pre-2.1 Lucene will not be able to read the index.</li>
-<li>In version 2.3, the file format was changed to allow segments to share a
-single set of doc store (vectors & stored fields) files. This allows for
-faster indexing in certain cases. The change is fully backwards compatible (in
-the same way as the lock-less commits change in 2.1).</li>
-<li>In version 2.4, Strings are now written as true UTF-8 byte sequence, not
-Java's modified UTF-8. See <a href="http://issues.apache.org/jira/browse/LUCENE-510">
-LUCENE-510</a> for details.</li>
-<li>In version 2.9, an optional opaque Map<String,String> CommitUserData
-may be passed to IndexWriter's commit methods (and later retrieved), which is
-recorded in the segments_N file. See <a href="http://issues.apache.org/jira/browse/LUCENE-1382">
-LUCENE-1382</a> for details. Also,
-diagnostics were added to each segment written recording details about why it
-was written (due to flush, merge; which OS/JRE was used; etc.). See issue
-<a href="http://issues.apache.org/jira/browse/LUCENE-1654">LUCENE-1654</a> for details.</li>
-<li>In version 3.0, compressed fields are no longer written to the index (they
-can still be read, but on merge the new segment will write them, uncompressed).
-See issue <a href="http://issues.apache.org/jira/browse/LUCENE-1960">LUCENE-1960</a>
-for details.</li>
-<li>In version 3.1, segments records the code version that created them. See
-<a href="http://issues.apache.org/jira/browse/LUCENE-2720">LUCENE-2720</a> for details.
-Additionally segments track explicitly whether or not they have term vectors.
-See <a href="http://issues.apache.org/jira/browse/LUCENE-2811">LUCENE-2811</a>
-for details.</li>
-<li>In version 3.2, numeric fields are written as natively to stored fields
-file, previously they were stored in text format only.</li>
-<li>In version 3.4, fields can omit position data while still indexing term
-frequencies.</li>
-<li>In version 4.0, the format of the inverted index became extensible via
-the {@link org.apache.lucene.codecs.Codec Codec} api. Fast per-document storage
-({@code DocValues}) was introduced. Normalization factors need no longer be a
-single byte, they can be any {@link org.apache.lucene.index.NumericDocValues NumericDocValues}.
-Terms need not be unicode strings, they can be any byte sequence. Term offsets
-can optionally be indexed into the postings lists. Payloads can be stored in the
-term vectors.</li>
-<li>In version 4.1, the format of the postings list changed to use either
-of FOR compression or variable-byte encoding, depending upon the frequency
-of the term. Terms appearing only once were changed to inline directly into
-the term dictionary. Stored fields are compressed by default. </li>
-<li>In version 4.2, term vectors are compressed by default. DocValues has
-a new multi-valued type (SortedSet), that can be used for faceting/grouping/joining
-on multi-valued fields.</li>
-<li>In version 4.5, DocValues were extended to explicitly represent missing values.</li>
-<li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to
-allow updating NumericDocValues fields.</li>
-<li>In version 4.8, checksum footers were added to the end of each index file
-for improved data integrity. Specifically, the last 8 bytes of every index file
-contain the zlib-crc32 checksum of the file.</li>
-<li>In version 4.9, DocValues has a new multi-valued numeric type (SortedNumeric)
-that is suitable for faceting/sorting/analytics.
-</li>
-</ul>
-<a name="Limitations" id="Limitations"></a>
-<h2>Limitations</h2>
-<div>
-<p>Lucene uses a Java <code>int</code> to refer to
-document numbers, and the index file format uses an <code>Int32</code>
-on-disk to store document numbers. This is a limitation
-of both the index file format and the current implementation. Eventually these
-should be replaced with either <code>UInt64</code> values, or
-better yet, {@link org.apache.lucene.store.DataOutput#writeVInt VInt} values which have no limit.</p>
-</div>
</body>
</html>
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java Sat Sep 27 10:23:23 2014
@@ -27,7 +27,6 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat;
import org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat;
import org.apache.lucene.codecs.lucene49.Lucene49NormsFormat;
@@ -49,7 +48,7 @@ public class Lucene50Codec extends Codec
private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
private final FieldInfosFormat fieldInfosFormat = new Lucene50FieldInfosFormat();
private final SegmentInfoFormat segmentInfosFormat = new Lucene50SegmentInfoFormat();
- private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
+ private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java Sat Sep 27 10:23:23 2014
@@ -25,6 +25,7 @@ import org.apache.lucene.codecs.FieldInf
import org.apache.lucene.codecs.FieldInfosReader;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.DataOutput;
/**
@@ -35,7 +36,7 @@ import org.apache.lucene.store.DataOutpu
* FieldBits,DocValuesBits,DocValuesGen,Attributes> <sup>FieldsCount</sup>,Footer</p>
* <p>Data types:
* <ul>
- * <li>Header --> {@link CodecUtil#checkHeader CodecHeader}</li>
+ * <li>Header --> {@link CodecUtil#checkSegmentHeader SegmentHeader}</li>
* <li>FieldsCount --> {@link DataOutput#writeVInt VInt}</li>
* <li>FieldName --> {@link DataOutput#writeString String}</li>
* <li>FieldBits, DocValuesBits --> {@link DataOutput#writeByte Byte}</li>
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosReader.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosReader.java Sat Sep 27 10:23:23 2014
@@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -47,15 +48,16 @@ final class Lucene50FieldInfosReader ext
}
@Override
- public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext context) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, Lucene50FieldInfosFormat.EXTENSION);
+ public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException {
+ final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene50FieldInfosFormat.EXTENSION);
try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
Throwable priorE = null;
FieldInfo infos[] = null;
try {
- CodecUtil.checkHeader(input, Lucene50FieldInfosFormat.CODEC_NAME,
+ CodecUtil.checkSegmentHeader(input, Lucene50FieldInfosFormat.CODEC_NAME,
Lucene50FieldInfosFormat.FORMAT_START,
- Lucene50FieldInfosFormat.FORMAT_CURRENT);
+ Lucene50FieldInfosFormat.FORMAT_CURRENT,
+ segmentInfo.getId());
final int size = input.readVInt(); //read in the size
infos = new FieldInfo[size];
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosWriter.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosWriter.java Sat Sep 27 10:23:23 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -43,10 +44,10 @@ final class Lucene50FieldInfosWriter ext
}
@Override
- public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, Lucene50FieldInfosFormat.EXTENSION);
+ public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
+ final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene50FieldInfosFormat.EXTENSION);
try (IndexOutput output = directory.createOutput(fileName, context)) {
- CodecUtil.writeHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT);
+ CodecUtil.writeSegmentHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId());
output.writeVInt(infos.size());
for (FieldInfo fi : infos) {
fi.checkConsistency();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoReader.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoReader.java Sat Sep 27 10:23:23 2014
@@ -30,6 +30,7 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.Version;
/**
@@ -69,7 +70,8 @@ public class Lucene50SegmentInfoReader e
final Map<String,String> diagnostics = input.readStringStringMap();
final Set<String> files = input.readStringSet();
- String id = input.readString();
+ byte[] id = new byte[StringHelper.ID_LENGTH];
+ input.readBytes(id, 0, id.length);
si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, id);
si.setFiles(files);
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoWriter.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoWriter.java Sat Sep 27 10:23:23 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.lucene5
*/
import java.io.IOException;
+import java.util.Set;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.SegmentInfoWriter;
@@ -28,6 +29,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.Version;
/**
@@ -48,10 +50,8 @@ public class Lucene50SegmentInfoWriter e
final String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene50SegmentInfoFormat.SI_EXTENSION);
si.addFile(fileName);
- final IndexOutput output = dir.createOutput(fileName, ioContext);
-
boolean success = false;
- try {
+ try (IndexOutput output = dir.createOutput(fileName, ioContext)) {
CodecUtil.writeHeader(output, Lucene50SegmentInfoFormat.CODEC_NAME, Lucene50SegmentInfoFormat.VERSION_CURRENT);
Version version = si.getVersion();
if (version.major < 5) {
@@ -63,17 +63,24 @@ public class Lucene50SegmentInfoWriter e
output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
output.writeStringStringMap(si.getDiagnostics());
- output.writeStringSet(si.files());
- output.writeString(si.getId());
+ Set<String> files = si.files();
+ for (String file : files) {
+ if (!IndexFileNames.parseSegmentName(file).equals(si.name)) {
+ throw new IllegalArgumentException("invalid files: expected segment=" + si.name + ", got=" + files);
+ }
+ }
+ output.writeStringSet(files);
+ byte[] id = si.getId();
+ if (id.length != StringHelper.ID_LENGTH) {
+ throw new IllegalArgumentException("invalid id, got=" + StringHelper.idToString(id));
+ }
+ output.writeBytes(id, 0, id.length);
CodecUtil.writeFooter(output);
success = true;
} finally {
if (!success) {
- IOUtils.closeWhileHandlingException(output);
// TODO: are we doing this outside of the tracking wrapper? why must SIWriter cleanup like this?
IOUtils.deleteFilesIgnoringExceptions(si.dir, fileName);
- } else {
- output.close();
}
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Sat Sep 27 10:23:23 2014
@@ -48,6 +48,7 @@ import org.apache.lucene.util.CommandLin
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LongBitSet;
+import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.Version;
@@ -515,7 +516,7 @@ public class CheckIndex {
}
msg(infoStream, "Segments file=" + segmentsFileName + " numSegments=" + numSegments
- + " " + versionString + " id=" + sis.getId() + " format=" + sFormat + userDataString);
+ + " " + versionString + " id=" + StringHelper.idToString(sis.getId()) + " format=" + sFormat + userDataString);
if (onlySegments != null) {
result.partial = true;
@@ -566,7 +567,7 @@ public class CheckIndex {
try {
msg(infoStream, " version=" + (version == null ? "3.0" : version));
- msg(infoStream, " id=" + info.info.getId());
+ msg(infoStream, " id=" + StringHelper.idToString(info.info.getId()));
final Codec codec = info.info.getCodec();
msg(infoStream, " codec=" + codec);
segInfoStat.codec = codec;
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Sat Sep 27 10:23:23 2014
@@ -119,7 +119,7 @@ final class DefaultIndexingChain extends
// FreqProxTermsWriter does this with
// FieldInfo.storePayload.
FieldInfosWriter infosWriter = docWriter.codec.fieldInfosFormat().getFieldInfosWriter();
- infosWriter.write(state.directory, state.segmentInfo.name, "", state.fieldInfos, IOContext.DEFAULT);
+ infosWriter.write(state.directory, state.segmentInfo, "", state.fieldInfos, IOContext.DEFAULT);
}
/** Writes all buffered doc values (called from {@link #flush}). */
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Sep 27 10:23:23 2014
@@ -2591,67 +2591,33 @@ public class IndexWriter implements Clos
}
/** Copies the segment files as-is into the IndexWriter's directory. */
- private SegmentCommitInfo copySegmentAsIs(SegmentCommitInfo info, String segName, IOContext context)
- throws IOException {
-
- // note: we don't really need this fis (its copied), but we load it up
- // so we don't pass a null value to the si writer
- FieldInfos fis = SegmentReader.readFieldInfos(info);
+ private SegmentCommitInfo copySegmentAsIs(SegmentCommitInfo info, String segName, IOContext context) throws IOException {
//System.out.println("copy seg=" + info.info.name + " version=" + info.info.getVersion());
// Same SI as before but we change directory and name
SegmentInfo newInfo = new SegmentInfo(directory, info.info.getVersion(), segName, info.info.getDocCount(),
info.info.getUseCompoundFile(), info.info.getCodec(),
- info.info.getDiagnostics(), StringHelper.randomId());
- SegmentCommitInfo newInfoPerCommit = new SegmentCommitInfo(newInfo,
- info.getDelCount(), info.getDelGen(), info.getFieldInfosGen(),
- info.getDocValuesGen());
-
- Set<String> segFiles = new HashSet<>();
-
- // Build up new segment's file names. Must do this
- // before writing SegmentInfo:
- for (String file: info.files()) {
- final String newFileName;
- newFileName = segName + IndexFileNames.stripSegmentName(file);
- segFiles.add(newFileName);
- }
- newInfo.setFiles(segFiles);
-
- // We must rewrite the SI file because it references segment name in its list of files, etc
- TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory);
+ info.info.getDiagnostics(), info.info.getId());
+ SegmentCommitInfo newInfoPerCommit = new SegmentCommitInfo(newInfo, info.getDelCount(), info.getDelGen(),
+ info.getFieldInfosGen(), info.getDocValuesGen());
+
+ newInfo.setFiles(info.files());
boolean success = false;
try {
-
- newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, newInfo, fis, context);
-
- final Collection<String> siFiles = trackingDir.getCreatedFiles();
-
// Copy the segment's files
for (String file: info.files()) {
+ final String newFileName = newInfo.namedForThisSegment(file);
- final String newFileName = segName + IndexFileNames.stripSegmentName(file);
-
- if (siFiles.contains(newFileName)) {
- // We already rewrote this above
- continue;
- }
-
- assert !slowFileExists(directory, newFileName): "file \"" + newFileName + "\" already exists; siFiles=" + siFiles;
+ assert !slowFileExists(directory, newFileName): "file \"" + newFileName + "\" already exists; newInfo.files=" + newInfo.files();
info.info.dir.copy(directory, file, newFileName, context);
}
success = true;
} finally {
if (!success) {
- for(String file : newInfo.files()) {
- try {
- directory.deleteFile(file);
- } catch (Throwable t) {
- }
- }
+ IOUtils.deleteFilesIgnoringExceptions(directory, newInfo.files().toArray(new String[0]));
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java Sat Sep 27 10:23:23 2014
@@ -451,7 +451,7 @@ class ReadersAndUpdates {
final IOContext infosContext = new IOContext(new FlushInfo(info.info.getDocCount(), estInfosSize));
// separately also track which files were created for this gen
final TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);
- infosFormat.getFieldInfosWriter().write(trackingDir, info.info.name, segmentSuffix, fieldInfos, infosContext);
+ infosFormat.getFieldInfosWriter().write(trackingDir, info.info, segmentSuffix, fieldInfos, infosContext);
info.advanceFieldInfosGen();
return trackingDir.getCreatedFiles();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java?rev=1627943&r1=1627942&r2=1627943&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java Sat Sep 27 10:23:23 2014
@@ -109,7 +109,14 @@ public class SegmentCommitInfo {
@Deprecated
public void setGenUpdatesFiles(Map<Long,Set<String>> genUpdatesFiles) {
this.genUpdatesFiles.clear();
- this.genUpdatesFiles.putAll(genUpdatesFiles);
+ for (Map.Entry<Long,Set<String>> kv : genUpdatesFiles.entrySet()) {
+ // rename the set
+ Set<String> set = new HashSet<>();
+ for (String file : kv.getValue()) {
+ set.add(info.namedForThisSegment(file));
+ }
+ this.genUpdatesFiles.put(kv.getKey(), set);
+ }
}
/** Returns the per-field DocValues updates files. */
@@ -120,7 +127,14 @@ public class SegmentCommitInfo {
/** Sets the DocValues updates file names, per field number. Does not deep clone the map. */
public void setDocValuesUpdatesFiles(Map<Integer,Set<String>> dvUpdatesFiles) {
this.dvUpdatesFiles.clear();
- this.dvUpdatesFiles.putAll(dvUpdatesFiles);
+ for (Map.Entry<Integer,Set<String>> kv : dvUpdatesFiles.entrySet()) {
+ // rename the set
+ Set<String> set = new HashSet<>();
+ for (String file : kv.getValue()) {
+ set.add(info.namedForThisSegment(file));
+ }
+ this.dvUpdatesFiles.put(kv.getKey(), set);
+ }
}
/** Returns the FieldInfos file names. */
@@ -131,7 +145,9 @@ public class SegmentCommitInfo {
/** Sets the FieldInfos file names. */
public void setFieldInfosFiles(Set<String> fieldInfosFiles) {
this.fieldInfosFiles.clear();
- this.fieldInfosFiles.addAll(fieldInfosFiles);
+ for (String file : fieldInfosFiles) {
+ this.fieldInfosFiles.add(info.namedForThisSegment(file));
+ }
}
/** Called when we succeed in writing deletes */