You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/05/20 15:24:30 UTC
svn commit: r1340710 - in /lucene/dev/branches/lucene4055/lucene:
core/src/java/org/apache/lucene/codecs/
core/src/java/org/apache/lucene/codecs/lucene3x/
core/src/java/org/apache/lucene/codecs/lucene40/
core/src/java/org/apache/lucene/codecs/simpletex...
Author: mikemccand
Date: Sun May 20 13:24:29 2012
New Revision: 1340710
URL: http://svn.apache.org/viewvc?rev=1340710&view=rev
Log:
LUCENE-4055: fix more nocommits
Modified:
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfosReader.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexOutput.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/util/CodecUtil.java
lucene/dev/branches/lucene4055/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/branches/lucene4055/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfosWriter.java
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfosReader.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfosReader.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfosReader.java Sun May 20 13:24:29 2012
@@ -42,5 +42,5 @@ public abstract class SegmentInfosReader
* @param infos empty instance to be populated with data
* @throws IOException
*/
- public abstract SegmentInfo read(Directory directory, String segmentName) throws IOException;
+ public abstract SegmentInfo read(Directory directory, String segmentName, IOContext context) throws IOException;
}
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java Sun May 20 13:24:29 2012
@@ -35,9 +35,6 @@ import org.apache.lucene.index.SegmentIn
public class Lucene3xSegmentInfosFormat extends SegmentInfosFormat {
private final SegmentInfosReader reader = new Lucene3xSegmentInfosReader();
- // nocommit explain or remove this!:
- public static final String SI_EXTENSION = "si";
-
/** This format adds optional per-segment String
* diagnostics storage, and switches userData to Map */
public static final int FORMAT_DIAGNOSTICS = -9;
@@ -48,20 +45,14 @@ public class Lucene3xSegmentInfosFormat
/** Each segment records the Lucene version that created it. */
public static final int FORMAT_3_1 = -11;
- // nocommit we should nuke FORMAT_4_0!?
-
/** Each segment records whether its postings are written
* in the new flex format */
- public static final int FORMAT_4_0 = -12;
+ public static final int FORMAT_4X_UPGRADE = -12;
- /** This must always point to the most recent file format.
- * whenever you add a new format, make it 1 smaller (negative version logic)! */
- // TODO: move this, as its currently part of required preamble
- public static final int FORMAT_CURRENT = FORMAT_4_0;
+ /** Extension used for saving each SegmentInfo, once a 3.x
+ * index is first committed to with 4.0. */
+ public static final String SI_EXTENSION = "si";
- /** This must always point to the first supported file format. */
- public static final int FORMAT_MINIMUM = FORMAT_DIAGNOSTICS;
-
@Override
public SegmentInfosReader getSegmentInfosReader() {
return reader;
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java Sun May 20 13:24:29 2012
@@ -89,19 +89,18 @@ public class Lucene3xSegmentInfosReader
}
@Override
- public SegmentInfo read(Directory directory, String segmentName) throws IOException {
- return read(directory, segmentName, Lucene3xSegmentInfosFormat.FORMAT_4_0);
+ public SegmentInfo read(Directory directory, String segmentName, IOContext context) throws IOException {
+ return read(directory, segmentName, Lucene3xSegmentInfosFormat.FORMAT_4X_UPGRADE, context);
}
- public SegmentInfo read(Directory directory, String segmentName, int format) throws IOException {
+ public SegmentInfo read(Directory directory, String segmentName, int format, IOContext context) throws IOException {
// NOTE: this is NOT how 3.x is really written...
String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene3xSegmentInfosFormat.SI_EXTENSION);
- // nocommit what IOCtx
boolean success = false;
- IndexInput input = directory.openInput(fileName, IOContext.READONCE);
+ IndexInput input = directory.openInput(fileName, context);
try {
SegmentInfo si = readSegmentInfo(segmentName, directory, format, input);
@@ -118,13 +117,13 @@ public class Lucene3xSegmentInfosReader
private SegmentInfo readSegmentInfo(String segmentName, Directory dir, int format, IndexInput input) throws IOException {
// check that it is a format we can understand
- if (format > Lucene3xSegmentInfosFormat.FORMAT_MINIMUM) {
+ if (format > Lucene3xSegmentInfosFormat.FORMAT_DIAGNOSTICS) {
throw new IndexFormatTooOldException(input, format,
- Lucene3xSegmentInfosFormat.FORMAT_MINIMUM, Lucene3xSegmentInfosFormat.FORMAT_CURRENT);
+ Lucene3xSegmentInfosFormat.FORMAT_DIAGNOSTICS, Lucene3xSegmentInfosFormat.FORMAT_4X_UPGRADE);
}
- if (format < Lucene3xSegmentInfosFormat.FORMAT_CURRENT) {
+ if (format < Lucene3xSegmentInfosFormat.FORMAT_4X_UPGRADE) {
throw new IndexFormatTooNewException(input, format,
- Lucene3xSegmentInfosFormat.FORMAT_MINIMUM, Lucene3xSegmentInfosFormat.FORMAT_CURRENT);
+ Lucene3xSegmentInfosFormat.FORMAT_DIAGNOSTICS, Lucene3xSegmentInfosFormat.FORMAT_4X_UPGRADE);
}
final String version;
if (format <= Lucene3xSegmentInfosFormat.FORMAT_3_1) {
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java Sun May 20 13:24:29 2012
@@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene3
import java.io.IOException;
import java.util.Set;
-import java.util.Arrays; //nocommit
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.TermVectorsReader;
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java Sun May 20 13:24:29 2012
@@ -58,10 +58,11 @@ public class Lucene40PostingsReader exte
// private String segment;
- // nocommit don't pass FIS here...
public Lucene40PostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext ioContext, String segmentSuffix) throws IOException {
freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION),
ioContext);
+ // nocommit don't consult FieldInfos here... then nuke
+ // fieldInfos arg...
// nocommit we can assert FIS.hasProx == our hasProx here...
// this.segment = segmentInfo.name;
if (fieldInfos.hasProx()) {
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java Sun May 20 13:24:29 2012
@@ -41,9 +41,9 @@ import org.apache.lucene.util.IOUtils;
public class Lucene40SegmentInfosReader extends SegmentInfosReader {
@Override
- public SegmentInfo read(Directory dir, String segment) throws IOException {
+ public SegmentInfo read(Directory dir, String segment, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene40SegmentInfosFormat.SI_EXTENSION);
- final IndexInput input = dir.openInput(fileName, IOContext.READONCE);
+ final IndexInput input = dir.openInput(fileName, context);
boolean success = false;
try {
final String version = input.readString();
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java Sun May 20 13:24:29 2012
@@ -49,13 +49,13 @@ public class SimpleTextNormsFormat exten
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new SimpleTextNormsPerDocConsumer(state, NORMS_SEG_SUFFIX);
+ return new SimpleTextNormsPerDocConsumer(state);
}
@Override
public PerDocProducer docsProducer(SegmentReadState state) throws IOException {
return new SimpleTextNormsPerDocProducer(state,
- BytesRef.getUTF8SortedAsUnicodeComparator(), NORMS_SEG_SUFFIX);
+ BytesRef.getUTF8SortedAsUnicodeComparator());
}
@Override
@@ -74,8 +74,8 @@ public class SimpleTextNormsFormat exten
SimpleTextPerDocProducer {
public SimpleTextNormsPerDocProducer(SegmentReadState state,
- Comparator<BytesRef> comp, String segmentSuffix) throws IOException {
- super(state, comp, segmentSuffix);
+ Comparator<BytesRef> comp) throws IOException {
+ super(state, comp, NORMS_SEG_SUFFIX);
}
@Override
@@ -105,9 +105,9 @@ public class SimpleTextNormsFormat exten
public static class SimpleTextNormsPerDocConsumer extends
SimpleTextPerDocConsumer {
- public SimpleTextNormsPerDocConsumer(PerDocWriteState state,
- String segmentSuffix) throws IOException {
- super(state, segmentSuffix);
+ public SimpleTextNormsPerDocConsumer(PerDocWriteState state)
+ throws IOException {
+ super(state, NORMS_SEG_SUFFIX);
}
@Override
@@ -129,25 +129,21 @@ public class SimpleTextNormsFormat exten
@Override
public void abort() {
Set<String> files = new HashSet<String>();
- filesInternal(state.segmentName, files, segmentSuffix);
+ filesInternal(state.segmentName, files);
IOUtils.deleteFilesIgnoringExceptions(state.directory,
SegmentInfo.findMatchingFiles(state.segmentName, state.directory, files).toArray(new String[0]));
}
public static void files(SegmentInfo segmentInfo, Set<String> files)
throws IOException {
- filesInternal(segmentInfo.name, files,
- NORMS_SEG_SUFFIX);
+ filesInternal(segmentInfo.name, files);
}
public static void filesInternal(String segmentName,
- Set<String> files, String segmentSuffix) {
- // nocommit simplify this: weird that we get suffix as
- // an arg... it's always a constant
- assert segmentSuffix.equals(NORMS_SEG_SUFFIX);
+ Set<String> files) {
String id = docValuesIdRegexp(segmentName);
files.add(IndexFileNames.segmentFileName(id, "",
- segmentSuffix));
+ NORMS_SEG_SUFFIX));
}
}
}
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java Sun May 20 13:24:29 2012
@@ -47,10 +47,10 @@ import static org.apache.lucene.codecs.s
public class SimpleTextSegmentInfosReader extends SegmentInfosReader {
@Override
- public SegmentInfo read(Directory directory, String segmentName) throws IOException {
+ public SegmentInfo read(Directory directory, String segmentName, IOContext context) throws IOException {
BytesRef scratch = new BytesRef();
String fileName = IndexFileNames.segmentFileName(segmentName, "", SimpleTextSegmentInfosFormat.SI_EXTENSION);
- IndexInput input = directory.openInput(fileName, IOContext.READONCE);
+ IndexInput input = directory.openInput(fileName, context);
boolean success = false;
try {
SimpleTextUtil.readLine(input, scratch);
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sun May 20 13:24:29 2012
@@ -663,7 +663,7 @@ public class IndexWriter implements Clos
rollbackSegments = segmentInfos.createBackupSegmentInfos(true);
// start with previous field numbers, but new FieldInfos
- globalFieldNumberMap = getOrLoadGlobalFieldNumberMap();
+ globalFieldNumberMap = getFieldNumberMap();
docWriter = new DocumentsWriter(codec, config, directory, this, globalFieldNumberMap, bufferedDeletesStream);
// Default deleter (for backwards compatibility) is
@@ -730,34 +730,27 @@ public class IndexWriter implements Clos
* Loads or returns the already loaded the global field number map for this {@link SegmentInfos}.
* If this {@link SegmentInfos} has no global field number map the returned instance is empty
*/
- private FieldNumberBiMap getOrLoadGlobalFieldNumberMap() throws IOException {
+ private FieldNumberBiMap getFieldNumberMap() throws IOException {
final FieldNumberBiMap map = new FieldNumberBiMap();
- if (segmentInfos.size() > 0) {
- // nocommit fixme for 3.x indices...
- /*
- if (segmentInfos.getFormat() > SegmentInfos.FORMAT_DIAGNOSTICS) {
- // Pre-3.1 index. In this case we sweep all
- // segments, merging their FieldInfos:
- for(SegmentInfo info : segmentInfos) {
- for(FieldInfo fi : getFieldInfos(info)) {
- map.addOrGet(fi.name, fi.number);
- }
- }
- } else {
- */
- // Already >= 3.1 index; just seed the FieldInfos
- // from the last segment
- for(FieldInfo fi : getFieldInfos(segmentInfos.info(segmentInfos.size()-1))) {
- map.addOrGet(fi.name, fi.number);
- }
+ SegmentInfo biggest = null;
+ for(SegmentInfo info : segmentInfos) {
+ if (biggest == null || (info.docCount-info.getDelCount()) > (biggest.docCount-biggest.getDelCount())) {
+ biggest = info;
+ }
+ }
- // nocommit we can also pull the DV types of the
- // fields... and catch DV type change on addDoc
- // instead of much later in merge
- //}
+ if (biggest != null) {
+ for(FieldInfo fi : getFieldInfos(biggest)) {
+ map.addOrGet(fi.name, fi.number);
+ }
}
+ // nocommit we can also pull the DV types of the
+ // fields... and catch DV type change on addDoc
+ // instead of much later in merge
+ //}
+
return map;
}
@@ -2025,29 +2018,14 @@ public class IndexWriter implements Clos
try {
if (useCompoundFile(newSegment)) {
String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "creating compound file " + compoundFileName);
- }
+
// Now build compound file
- // nocommit factor to use craeteCompoundFile method!?
- final Directory cfsDir = new CompoundFileDirectory(directory, compoundFileName, context, true);
- IOException prior = null;
- try {
- for(String fileName : newSegment.files()) {
- directory.copy(cfsDir, fileName, fileName, context);
- }
- } catch(IOException ex) {
- prior = ex;
- } finally {
- IOUtils.closeWhileHandlingException(prior, cfsDir);
- }
- // Perform the merge
+ Collection<String> files = createCompoundFile(infoStream, directory, compoundFileName, MergeState.CheckAbort.NONE, newSegment, context);
+ newSegment.setUseCompoundFile(true);
synchronized(this) {
- deleter.deleteNewFiles(newSegment.files());
+ deleter.deleteNewFiles(files);
}
-
- newSegment.setUseCompoundFile(true);
}
// Have codec write SegmentInfo. Must do this after
@@ -2326,7 +2304,7 @@ public class IndexWriter implements Clos
// Now create the compound file if needed
if (useCompoundFile) {
- createCompoundFile(directory, IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_EXTENSION), MergeState.CheckAbort.NONE, info, context);
+ createCompoundFile(infoStream, directory, IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_EXTENSION), MergeState.CheckAbort.NONE, info, context);
// delete new non cfs files directly: they were never
// registered with IFD
@@ -3422,15 +3400,12 @@ public class IndexWriter implements Clos
final String mergedName = merge.info.name;
- int mergedDocCount = 0;
-
List<SegmentInfo> sourceSegments = merge.segments;
IOContext context = new IOContext(merge.getMergeInfo());
final MergeState.CheckAbort checkAbort = new MergeState.CheckAbort(merge, directory);
SegmentMerger merger = new SegmentMerger(infoStream, directory, config.getTermIndexInterval(), mergedName, checkAbort,
- // nocommit
payloadProcessorProvider, new FieldInfos.Builder(globalFieldNumberMap), codec, context);
if (infoStream.isEnabled("IW")) {
@@ -3491,7 +3466,7 @@ public class IndexWriter implements Clos
// This is where all the work happens:
MergeState mergeState = merger.merge();
- mergedDocCount = merge.info.docCount = mergeState.mergedDocCount;
+ merge.info.docCount = mergeState.mergedDocCount;
// Record which codec was used to write the segment
@@ -3499,11 +3474,8 @@ public class IndexWriter implements Clos
// ctor when we make the merge.info:
merge.info.setCodec(codec);
- // nocommit should segment merger do this!? else
- // other places must do so...??? addIndexes...
-
if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "merge codec=" + codec + " docCount=" + mergedDocCount + "; merged segment has " +
+ infoStream.message("IW", "merge codec=" + codec + " docCount=" + merge.info.docCount + "; merged segment has " +
(mergeState.fieldInfos.hasVectors() ? "vectors" : "no vectors") + "; " +
(mergeState.fieldInfos.hasNorms() ? "norms" : "no norms") + "; " +
(mergeState.fieldInfos.hasDocValues() ? "docValues" : "no docValues") + "; " +
@@ -3525,10 +3497,7 @@ public class IndexWriter implements Clos
final String compoundFileName = IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
try {
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "create compound file " + compoundFileName);
- }
- createCompoundFile(directory, compoundFileName, checkAbort, merge.info, context);
+ createCompoundFile(infoStream, directory, compoundFileName, checkAbort, merge.info, context);
success = true;
} catch (IOException ioe) {
synchronized(this) {
@@ -3556,6 +3525,7 @@ public class IndexWriter implements Clos
}
}
+ // nocommit why do we set success back to false here!?
success = false;
synchronized(this) {
@@ -3576,14 +3546,21 @@ public class IndexWriter implements Clos
merge.info.setUseCompoundFile(true);
}
- // nocommit need try/success thingy...? ie must
- // remove all seg files if we fail to write the .si?
-
// Have codec write SegmentInfo. Must do this after
// creating CFS so that 1) .si isn't slurped into CFS,
// and 2) .si reflects useCompoundFile=true change
// above:
- codec.segmentInfosFormat().getSegmentInfosWriter().write(directory, merge.info, mergeState.fieldInfos, context);
+ boolean success2 = false;
+ try {
+ codec.segmentInfosFormat().getSegmentInfosWriter().write(directory, merge.info, mergeState.fieldInfos, context);
+ success2 = true;
+ } finally {
+ if (!success2) {
+ synchronized(this) {
+ deleter.deleteNewFiles(merge.info.files());
+ }
+ }
+ }
merge.info.clearFilesCache();
// nocommit ideally we would freeze merge.info here!!
@@ -3626,7 +3603,7 @@ public class IndexWriter implements Clos
}
}
- return mergedDocCount;
+ return merge.info.docCount;
}
synchronized void addMergeException(MergePolicy.OneMerge merge) {
@@ -3801,9 +3778,6 @@ public class IndexWriter implements Clos
pendingCommit = toSync;
}
- // nocommit move this back above...? problem is
- // prepareCommit writes on the _X.si files... which
- // of course need to be sync'd too...
// This call can take a long time -- 10s of seconds
// or more. We do it without sync:
boolean success = false;
@@ -3999,19 +3973,26 @@ public class IndexWriter implements Clos
* deletion files, this SegmentInfo must not reference such files when this
* method is called, because they are not allowed within a compound file.
*/
- static final Collection<String> createCompoundFile(Directory directory, String fileName, CheckAbort checkAbort, final SegmentInfo info, IOContext context)
+ static final Collection<String> createCompoundFile(InfoStream infoStream, Directory directory, String fileName, CheckAbort checkAbort, final SegmentInfo info, IOContext context)
throws IOException {
+
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "create compound file " + fileName);
+ }
assert info.getDocStoreOffset() == -1;
// Now merge all added files
Collection<String> files = info.files();
CompoundFileDirectory cfsDir = new CompoundFileDirectory(directory, fileName, context, true);
+ IOException prior = null;
try {
for (String file : files) {
directory.copy(cfsDir, file, file, context);
checkAbort.work(directory.fileLength(file));
}
+ } catch(IOException ex) {
+ prior = ex;
} finally {
- cfsDir.close();
+ IOUtils.closeWhileHandlingException(prior, cfsDir);
}
return files;
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Sun May 20 13:24:29 2012
@@ -60,6 +60,7 @@ public class SegmentInfo implements Clon
* - NO if there are no deletes
* - YES or higher if there are deletes at generation N
*/
+ // nocommit explain that codec need not save this....:
private long delGen;
/*
@@ -86,6 +87,7 @@ public class SegmentInfo implements Clon
//TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
private boolean docStoreIsCompoundFile; // whether doc store files are stored in compound file (*.cfx)
+ // nocommit explain that codec need not save this....:
private int delCount; // How many deleted docs in this segment
private Codec codec;
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Sun May 20 13:24:29 2012
@@ -280,7 +280,7 @@ public final class SegmentInfos implemen
String segName = input.readString();
Codec codec = Codec.forName(input.readString());
//System.out.println("SIS.read seg=" + seg + " codec=" + codec);
- SegmentInfo info = codec.segmentInfosFormat().getSegmentInfosReader().read(directory, segName);
+ SegmentInfo info = codec.segmentInfosFormat().getSegmentInfosReader().read(directory, segName, IOContext.READ);
info.setCodec(codec);
info.setDelGen(input.readLong());
info.setDelCount(input.readInt());
@@ -352,9 +352,10 @@ public final class SegmentInfos implemen
// list-of-segs plus delGen plus other stuff
// "generically" and then codec gets to write SI
+ final Set<String> upgradedSIFiles = new HashSet<String>();
+
try {
- // nocommit what IOCtx to use...
- segnOutput = new ChecksumIndexOutput(directory.createOutput(segmentFileName, new IOContext(new FlushInfo(totalDocCount(), 0))));
+ segnOutput = new ChecksumIndexOutput(directory.createOutput(segmentFileName, IOContext.DEFAULT));
CodecUtil.writeHeader(segnOutput, "segments", VERSION_40);
segnOutput.writeLong(version);
segnOutput.writeInt(counter); // write counter
@@ -367,19 +368,19 @@ public final class SegmentInfos implemen
segnOutput.writeInt(si.getDelCount());
assert si.dir == directory;
- // nocommit hacky!
+ // If this segment is pre-4.x, perform a one-time
+ // "ugprade" to write the .si file for it:
String version = si.getVersion();
if (version == null || version.startsWith("3.")) {
String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene3xSegmentInfosFormat.SI_EXTENSION);
if (!directory.fileExists(fileName)) {
//System.out.println("write 3x info seg=" + si.name + " version=" + si.getVersion() + " codec=" + si.getCodec().getName());
- write3xInfo(si);
+ upgradedSIFiles.add(write3xInfo(directory, si, IOContext.DEFAULT));
si.clearFilesCache();
}
}
}
segnOutput.writeStringStringMap(userData);
- segnOutput.prepareCommit();
pendingSegnOutput = segnOutput;
success = true;
} finally {
@@ -388,7 +389,13 @@ public final class SegmentInfos implemen
// but suppress any exception:
IOUtils.closeWhileHandlingException(segnOutput);
- // nocommit must also remove any written .si files...
+ for(String fileName : upgradedSIFiles) {
+ try {
+ directory.deleteFile(fileName);
+ } catch (Throwable t) {
+ // Suppress so we keep throwing the original exception
+ }
+ }
try {
// Try not to leave a truncated segments_N file in
@@ -401,18 +408,14 @@ public final class SegmentInfos implemen
}
}
- // nocommit copy of PreflexRWSegmentInfosWriter.write!!
-
@Deprecated
- public void write3xInfo(SegmentInfo si) throws IOException {
+ public static String write3xInfo(Directory dir, SegmentInfo si, IOContext context) throws IOException {
// NOTE: this is NOT how 3.x is really written...
String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene3xSegmentInfosFormat.SI_EXTENSION);
//System.out.println("UPGRADE write " + fileName);
- // nocommit what IOCtx
boolean success = false;
-
- IndexOutput output = si.dir.createOutput(fileName, new IOContext(new FlushInfo(0, 0)));
+ IndexOutput output = dir.createOutput(fileName, context);
try {
// we are about to write this SI in 3.x format, dropping all codec information, etc.
// so it had better be a 3.x segment or you will get very confusing errors later.
@@ -454,11 +457,17 @@ public final class SegmentInfos implemen
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(output);
- si.dir.deleteFile(fileName);
+ try {
+ si.dir.deleteFile(fileName);
+ } catch (Throwable t) {
+ // Suppress so we keep throwing the original exception
+ }
} else {
output.close();
}
}
+
+ return fileName;
}
/**
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexOutput.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexOutput.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexOutput.java Sun May 20 13:24:29 2012
@@ -26,7 +26,6 @@ import java.util.zip.Checksum;
*
* @lucene.internal
*/
-// nocommit fixme to not seek backwards...
public class ChecksumIndexOutput extends IndexOutput {
IndexOutput main;
Checksum digest;
@@ -72,29 +71,6 @@ public class ChecksumIndexOutput extends
throw new UnsupportedOperationException();
}
- /**
- * Starts but does not complete the commit of this file (=
- * writing of the final checksum at the end). After this
- * is called must call {@link #finishCommit} and the
- * {@link #close} to complete the commit.
- */
- public void prepareCommit() throws IOException {
- final long checksum = getChecksum();
- // Intentionally write a mismatched checksum. This is
- // because we want to 1) test, as best we can, that we
- // are able to write a long to the file, but 2) not
- // actually "commit" the file yet. This (prepare
- // commit) is phase 1 of a two-phase commit.
- // nocommit fixme... or just nuke? appending codec
- // fails w/ this:
- /*
- final long pos = main.getFilePointer();
- main.writeLong(checksum-1);
- main.flush();
- main.seek(pos);
- */
- }
-
/** See {@link #prepareCommit} */
public void finishCommit() throws IOException {
main.writeLong(getChecksum());
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java Sun May 20 13:24:29 2012
@@ -109,7 +109,7 @@ public final class TrackingDirectoryWrap
@Override
public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
- // nocommit add dest to created files!?
+ createdFileNames.add(dest);
other.copy(to, src, dest, context);
}
@@ -118,11 +118,6 @@ public final class TrackingDirectoryWrap
return other.createSlicer(name, context);
}
- // nocommit no?
- //@Override
- //protected final void ensureOpen() throws
- //AlreadyClosedException {
-
public Set<String> getCreatedFiles() {
return createdFileNames;
}
Modified: lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/util/CodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/util/CodecUtil.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/util/CodecUtil.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/java/org/apache/lucene/util/CodecUtil.java Sun May 20 13:24:29 2012
@@ -129,7 +129,10 @@ public final class CodecUtil {
return checkHeaderNoMagic(in, codec, minVersion, maxVersion);
}
- // nocommit jdocs
+ /** Like {@link
+ * checkHeader(DataInput,String,int,int)} except this
+ * version assumes the first int has already been read
+ * and validated from the input. */
public static int checkHeaderNoMagic(DataInput in, String codec, int minVersion, int maxVersion) throws IOException {
final String actualCodec = in.readString();
if (!actualCodec.equals(codec)) {
Modified: lucene/dev/branches/lucene4055/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/lucene4055/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Sun May 20 13:24:29 2012
@@ -205,7 +205,7 @@ public class TestDoc extends LuceneTestC
false, null, false, 0, codec, null);
if (useCompoundFile) {
- Collection<String> filesToDelete = IndexWriter.createCompoundFile(dir, merged + ".cfs", MergeState.CheckAbort.NONE, info, newIOContext(random()));
+ Collection<String> filesToDelete = IndexWriter.createCompoundFile(InfoStream.getDefault(), dir, merged + ".cfs", MergeState.CheckAbort.NONE, info, newIOContext(random()));
info.setUseCompoundFile(true);
for (final String fileToDelete : filesToDelete) {
si1.dir.deleteFile(fileToDelete);
Modified: lucene/dev/branches/lucene4055/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4055/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfosWriter.java?rev=1340710&r1=1340709&r2=1340710&view=diff
==============================================================================
--- lucene/dev/branches/lucene4055/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfosWriter.java (original)
+++ lucene/dev/branches/lucene4055/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfosWriter.java Sun May 20 13:24:29 2012
@@ -46,57 +46,7 @@ class PreFlexRWSegmentInfosWriter extend
/** Save a single segment's info. */
@Override
- public void write(Directory dir ,SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException {
-
- String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene3xSegmentInfosFormat.SI_EXTENSION);
-
- boolean success = false;
-
- IndexOutput output = dir.createOutput(fileName, ioContext);
- try {
- // we are about to write this SI in 3.x format, dropping all codec information, etc.
- // so it had better be a 3.x segment or you will get very confusing errors later.
- assert si.getCodec() instanceof Lucene3xCodec : "broken test, trying to mix preflex with other codecs";
- assert si.getDelCount() <= si.docCount: "delCount=" + si.getDelCount() + " docCount=" + si.docCount + " segment=" + si.name;
- // Write the Lucene version that created this segment, since 3.1
- output.writeString(si.getVersion());
- output.writeString(si.name);
- output.writeInt(si.docCount);
- output.writeLong(si.getDelGen());
-
- output.writeInt(si.getDocStoreOffset());
- if (si.getDocStoreOffset() != -1) {
- output.writeString(si.getDocStoreSegment());
- output.writeByte((byte) (si.getDocStoreIsCompoundFile() ? 1:0));
- }
- // pre-4.0 indexes write a byte if there is a single norms file
- output.writeByte((byte) 1);
-
- Map<Integer,Long> normGen = si.getNormGen();
- if (normGen == null) {
- output.writeInt(SegmentInfo.NO);
- } else {
- output.writeInt(normGen.size());
- for (Entry<Integer,Long> entry : normGen.entrySet()) {
- output.writeLong(entry.getValue());
- }
- }
-
- output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
- output.writeInt(si.getDelCount());
- // hasProx:
- output.writeByte((byte) 1);
- output.writeStringStringMap(si.getDiagnostics());
- // hasVectors:
- output.writeByte((byte) 1);
-
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(output);
- } else {
- output.close();
- }
- }
+ public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException {
+ SegmentInfos.write3xInfo(dir, si, ioContext);
}
}