You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Michael McCandless <lu...@mikemccandless.com> on 2012/10/03 14:23:56 UTC
Re: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
codecs/src/java/org/apache/lucene/codecs/simpletext/ core/src/java/org/apache/lucene/index/
core/src/java/org/apache/lucene/store/ core/src/test/org/apache/lucene/codecs/perfield/
core/src/te
I didn't re-integrate; I just committed the patch.
But I can remove the branch anyway if you want...?
Mike McCandless
http://blog.mikemccandless.com
On Wed, Oct 3, 2012 at 8:19 AM, Uwe Schindler <uw...@thetaphi.de> wrote:
> Can you remove the branch after reintegrate?
>
> -----
> Uwe Schindler
> H.-H.-Meier-Allee 63, D-28213 Bremen
> http://www.thetaphi.de
> eMail: uwe@thetaphi.de
>
>
>> -----Original Message-----
>> From: mikemccand@apache.org [mailto:mikemccand@apache.org]
>> Sent: Wednesday, October 03, 2012 1:01 PM
>> To: commits@lucene.apache.org
>> Subject: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
>> codecs/src/java/org/apache/lucene/codecs/simpletext/
>> core/src/java/org/apache/lucene/index/
>> core/src/java/org/apache/lucene/store/
>> core/src/test/org/apache/lucene/codecs/perfield/ core/src/test/org...
>>
>> Author: mikemccand
>> Date: Wed Oct 3 11:01:13 2012
>> New Revision: 1393408
>>
>> URL: http://svn.apache.org/viewvc?rev=1393408&view=rev
>> Log:
>> LUCENE-4456: fix various bugs (see lucene/CHANGES.txt)
>>
>> Added:
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/Te
>> stFailIfUnreferencedFiles.java (with props)
>> Modified:
>> lucene/dev/trunk/lucene/CHANGES.txt
>>
>> lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletex
>> t/SimpleTextDocValuesConsumer.java
>>
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsW
>> riterPerThread.java
>>
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDele
>> ter.java
>>
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.j
>> ava
>>
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWri
>> terConfig.java
>>
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfo
>> PerCommit.java
>>
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.jav
>> a
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/Tes
>> tPerFieldPostingsFormat.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.jav
>> a
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
>> Reader.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
>> ReaderReopen.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
>> erExceptions.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
>> erWithThreads.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeleti
>> onPolicy.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistent
>> SnapshotDeletionPolicy.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUp
>> dates.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshot
>> DeletionPolicy.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
>> onRollback.java
>>
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
>> ons.java
>>
>> lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplit
>> ter.java
>> lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
>> lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
>> lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/util/LuceneTestCase.java
>>
>> Modified: lucene/dev/trunk/lucene/CHANGES.txt
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1393
>> 408&r1=1393407&r2=1393408&view=diff
>> ================================================================
>> ==============
>> --- lucene/dev/trunk/lucene/CHANGES.txt (original)
>> +++ lucene/dev/trunk/lucene/CHANGES.txt Wed Oct 3 11:01:13 2012
>> @@ -226,6 +226,14 @@ Bug Fixes
>> CheckIndex that would report no deletions when a segment has
>> deletions, and vice/versa. (Uwe Schindler, Robert Muir, Mike McCandless)
>>
>> +* LUCENE-4456: Fixed double-counting sizeInBytes for a segment
>> + (affects how merge policies pick merges); fixed CheckIndex's
>> + incorrect reporting of whether a segment has deletions; fixed case
>> + where on abort Lucene could remove files it didn't create; fixed
>> + many cases where IndexWriter could leave leftover files (on
>> + exception in various places, on reuse of a segment name after crash
>> + and recovery. (Uwe Schindler, Robert Muir, Mike McCandless)
>> +
>> Optimizations
>>
>> * LUCENE-4322: Decrease lucene-core JAR size. The core JAR size had
>> increased a
>>
>> Modified:
>> lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletex
>> t/SimpleTextDocValuesConsumer.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/ap
>> ache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java?rev=1393
>> 408&r1=1393407&r2=1393408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletex
>> t/SimpleTextDocValuesConsumer.java (original)
>> +++
>> lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletex
>> t/SimpleTextDocValuesConsumer.java Wed Oct 3 11:01:13 2012
>> @@ -198,6 +198,7 @@ public class SimpleTextDocValuesConsumer
>> IOUtils.close(output);
>> } else {
>> IOUtils.closeWhileHandlingException(output);
>> + dir.deleteFile(fileName);
>> }
>> }
>> }
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsW
>> riterPerThread.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
>> he/lucene/index/DocumentsWriterPerThread.java?rev=1393408&r1=1393407&
>> r2=1393408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsW
>> riterPerThread.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsW
>> riterPerThread.java Wed Oct 3 11:01:13 2012
>> @@ -137,6 +137,7 @@ class DocumentsWriterPerThread {
>> * currently buffered docs. This resets our state,
>> * discarding any docs added since last flush. */
>> void abort() {
>> + //System.out.println(Thread.currentThread().getName() + ": now abort
>> seg=" + segmentInfo.name);
>> hasAborted = aborting = true;
>> try {
>> if (infoStream.isEnabled("DWPT")) {
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDele
>> ter.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
>> he/lucene/index/IndexFileDeleter.java?rev=1393408&r1=1393407&r2=139340
>> 8&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDele
>> ter.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDele
>> ter.java Wed Oct 3 11:01:13 2012
>> @@ -333,12 +333,16 @@ final class IndexFileDeleter {
>> segmentPrefix2 = null;
>> }
>>
>> + Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
>> +
>> for(int i=0;i<files.length;i++) {
>> String fileName = files[i];
>> + m.reset(fileName);
>> if ((segmentName == null || fileName.startsWith(segmentPrefix1) ||
>> fileName.startsWith(segmentPrefix2)) &&
>> !fileName.endsWith("write.lock") &&
>> !refCounts.containsKey(fileName) &&
>> - !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
>> + !fileName.equals(IndexFileNames.SEGMENTS_GEN) &&
>> + (m.matches() || fileName.startsWith(IndexFileNames.SEGMENTS))) {
>> // Unreferenced file, so remove it
>> if (infoStream.isEnabled("IFD")) {
>> infoStream.message("IFD", "refresh [prefix=" + segmentName + "]:
>> removing newly created unreferenced file \"" + fileName + "\"");
>> @@ -554,7 +558,13 @@ final class IndexFileDeleter {
>> void deleteNewFiles(Collection<String> files) throws IOException {
>> assert locked();
>> for (final String fileName: files) {
>> - if (!refCounts.containsKey(fileName)) {
>> + // NOTE: it's very unusual yet possible for the
>> + // refCount to be present and 0: it can happen if you
>> + // open IW on a crashed index, and it removes a bunch
>> + // of unref'd files, and then you add new docs / do
>> + // merging, and it reuses that segment name.
>> + // TestCrash.testCrashAfterReopen can hit this:
>> + if (!refCounts.containsKey(fileName) || refCounts.get(fileName).count ==
>> 0) {
>> if (infoStream.isEnabled("IFD")) {
>> infoStream.message("IFD", "delete new file \"" + fileName + "\"");
>> }
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.j
>> ava
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
>> he/lucene/index/IndexWriter.java?rev=1393408&r1=1393407&r2=1393408&vi
>> ew=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.j
>> ava (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.j
>> ava Wed Oct 3 11:01:13 2012
>> @@ -2145,7 +2145,7 @@ public class IndexWriter implements Clos
>> // Now build compound file
>> Collection<String> oldFiles = createCompoundFile(infoStream, directory,
>> MergeState.CheckAbort.NONE, newSegment.info, context);
>> newSegment.info.setUseCompoundFile(true);
>> -
>> +
>> synchronized(this) {
>> deleter.deleteNewFiles(oldFiles);
>> }
>> @@ -2321,31 +2321,62 @@ public class IndexWriter implements Clos
>> flush(false, true);
>>
>> List<SegmentInfoPerCommit> infos = new
>> ArrayList<SegmentInfoPerCommit>();
>> - for (Directory dir : dirs) {
>> - if (infoStream.isEnabled("IW")) {
>> - infoStream.message("IW", "addIndexes: process directory " + dir);
>> - }
>> - SegmentInfos sis = new SegmentInfos(); // read infos from dir
>> - sis.read(dir);
>> -
>> - for (SegmentInfoPerCommit info : sis) {
>> - assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" +
>> info.info.name;
>> -
>> - String newSegName = newSegmentName();
>> - String dsName = info.info.name;
>>
>> + boolean success = false;
>> + try {
>> + for (Directory dir : dirs) {
>> if (infoStream.isEnabled("IW")) {
>> - infoStream.message("IW", "addIndexes: process segment origName="
>> + info.info.name + " newName=" + newSegName + " dsName=" + dsName + "
>> info=" + info);
>> + infoStream.message("IW", "addIndexes: process directory " + dir);
>> }
>> + SegmentInfos sis = new SegmentInfos(); // read infos from dir
>> + sis.read(dir);
>> +
>> + for (SegmentInfoPerCommit info : sis) {
>> + assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" +
>> info.info.name;
>> +
>> + String newSegName = newSegmentName();
>> + String dsName = info.info.name;
>>
>> - IOContext context = new IOContext(new
>> MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
>> + if (infoStream.isEnabled("IW")) {
>> + infoStream.message("IW", "addIndexes: process segment origName="
>> + info.info.name + " newName=" + newSegName + " dsName=" + dsName + "
>> info=" + info);
>> + }
>> +
>> + IOContext context = new IOContext(new
>> MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
>>
>> - infos.add(copySegmentAsIs(info, newSegName, context));
>> + infos.add(copySegmentAsIs(info, newSegName, context));
>> + }
>> + }
>> + success = true;
>> + } finally {
>> + if (!success) {
>> + for(SegmentInfoPerCommit sipc : infos) {
>> + for(String file : sipc.files()) {
>> + try {
>> + directory.deleteFile(file);
>> + } catch (Throwable t) {
>> + }
>> + }
>> + }
>> }
>> }
>>
>> synchronized (this) {
>> - ensureOpen();
>> + success = false;
>> + try {
>> + ensureOpen();
>> + success = true;
>> + } finally {
>> + if (!success) {
>> + for(SegmentInfoPerCommit sipc : infos) {
>> + for(String file : sipc.files()) {
>> + try {
>> + directory.deleteFile(file);
>> + } catch (Throwable t) {
>> + }
>> + }
>> + }
>> + }
>> + }
>> segmentInfos.addAll(infos);
>> checkpoint();
>> }
>> @@ -2420,7 +2451,18 @@ public class IndexWriter implements Clos
>> merger.add(reader);
>> }
>>
>> - MergeState mergeState = merger.merge(); // merge 'em
>> + MergeState mergeState;
>> + boolean success = false;
>> + try {
>> + mergeState = merger.merge(); // merge 'em
>> + success = true;
>> + } finally {
>> + if (!success) {
>> + synchronized(this) {
>> + deleter.refresh(info.name);
>> + }
>> + }
>> + }
>>
>> SegmentInfoPerCommit infoPerCommit = new
>> SegmentInfoPerCommit(info, 0, -1L);
>>
>> @@ -2442,12 +2484,14 @@ public class IndexWriter implements Clos
>> // Now create the compound file if needed
>> if (useCompoundFile) {
>> Collection<String> filesToDelete = infoPerCommit.files();
>> - createCompoundFile(infoStream, directory,
>> MergeState.CheckAbort.NONE, info, context);
>> -
>> - // delete new non cfs files directly: they were never
>> - // registered with IFD
>> - synchronized(this) {
>> - deleter.deleteNewFiles(filesToDelete);
>> + try{
>> + createCompoundFile(infoStream, directory,
>> MergeState.CheckAbort.NONE, info, context);
>> + } finally {
>> + // delete new non cfs files directly: they were never
>> + // registered with IFD
>> + synchronized(this) {
>> + deleter.deleteNewFiles(filesToDelete);
>> + }
>> }
>> info.setUseCompoundFile(true);
>> }
>> @@ -2456,7 +2500,18 @@ public class IndexWriter implements Clos
>> // creating CFS so that 1) .si isn't slurped into CFS,
>> // and 2) .si reflects useCompoundFile=true change
>> // above:
>> - codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir,
>> info, mergeState.fieldInfos, context);
>> + success = false;
>> + try {
>> + codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir,
>> info, mergeState.fieldInfos, context);
>> + success = true;
>> + } finally {
>> + if (!success) {
>> + synchronized(this) {
>> + deleter.refresh(info.name);
>> + }
>> + }
>> + }
>> +
>> info.addFiles(trackingDir.getCreatedFiles());
>>
>> // Register the new segment
>> @@ -2512,23 +2567,38 @@ public class IndexWriter implements Clos
>> // We must rewrite the SI file because it references segment name in its list
>> of files, etc
>> TrackingDirectoryWrapper trackingDir = new
>> TrackingDirectoryWrapper(directory);
>>
>> -
>> newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackin
>> gDir, newInfo, fis, context);
>> + boolean success = false;
>>
>> - final Collection<String> siFiles = trackingDir.getCreatedFiles();
>> + try {
>>
>> - // Copy the segment's files
>> - for (String file: info.files()) {
>> +
>> newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackin
>> gDir, newInfo, fis, context);
>>
>> - final String newFileName = segName +
>> IndexFileNames.stripSegmentName(file);
>> + final Collection<String> siFiles = trackingDir.getCreatedFiles();
>>
>> - if (siFiles.contains(newFileName)) {
>> - // We already rewrote this above
>> - continue;
>> - }
>> + // Copy the segment's files
>> + for (String file: info.files()) {
>> +
>> + final String newFileName = segName +
>> IndexFileNames.stripSegmentName(file);
>>
>> - assert !directory.fileExists(newFileName): "file \"" + newFileName + "\"
>> already exists; siFiles=" + siFiles;
>> + if (siFiles.contains(newFileName)) {
>> + // We already rewrote this above
>> + continue;
>> + }
>>
>> - info.info.dir.copy(directory, file, newFileName, context);
>> + assert !directory.fileExists(newFileName): "file \"" + newFileName + "\"
>> already exists; siFiles=" + siFiles;
>> +
>> + info.info.dir.copy(directory, file, newFileName, context);
>> + }
>> + success = true;
>> + } finally {
>> + if (!success) {
>> + for(String file : newInfo.files()) {
>> + try {
>> + directory.deleteFile(file);
>> + } catch (Throwable t) {
>> + }
>> + }
>> + }
>> }
>>
>> return newInfoPerCommit;
>> @@ -3091,6 +3161,7 @@ public class IndexWriter implements Clos
>> if (infoStream.isEnabled("IW")) {
>> infoStream.message("IW", "commitMerge: skip: it was aborted");
>> }
>> + deleter.deleteNewFiles(merge.info.files());
>> return false;
>> }
>>
>> @@ -3136,6 +3207,11 @@ public class IndexWriter implements Clos
>> }
>> }
>>
>> + if (dropSegment) {
>> + assert !segmentInfos.contains(merge.info);
>> + deleter.deleteNewFiles(merge.info.files());
>> + }
>> +
>> // Must close before checkpoint, otherwise IFD won't be
>> // able to delete the held-open files from the merge
>> // readers:
>> @@ -3407,7 +3483,7 @@ public class IndexWriter implements Clos
>> setDiagnostics(si, "merge", details);
>>
>> if (infoStream.isEnabled("IW")) {
>> - infoStream.message("IW", "merge seg=" + merge.info.info.name);
>> + infoStream.message("IW", "merge seg=" + merge.info.info.name + " " +
>> segString(merge.segments));
>> }
>>
>> assert merge.estimatedMergeBytes == 0;
>> @@ -4116,7 +4192,22 @@ public class IndexWriter implements Clos
>> } catch(IOException ex) {
>> prior = ex;
>> } finally {
>> - IOUtils.closeWhileHandlingException(prior, cfsDir);
>> + boolean success = false;
>> + try {
>> + IOUtils.closeWhileHandlingException(prior, cfsDir);
>> + success = true;
>> + } finally {
>> + if (!success) {
>> + try {
>> + directory.deleteFile(fileName);
>> + } catch (Throwable t) {
>> + }
>> + try {
>> + directory.deleteFile(IndexFileNames.segmentFileName(info.name, "",
>> IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
>> + } catch (Throwable t) {
>> + }
>> + }
>> + }
>> }
>>
>> // Replace all previous files with the CFS/CFE files:
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWri
>> terConfig.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
>> he/lucene/index/LiveIndexWriterConfig.java?rev=1393408&r1=1393407&r2=13
>> 93408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWri
>> terConfig.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWri
>> terConfig.java Wed Oct 3 11:01:13 2012
>> @@ -547,7 +547,7 @@ public class LiveIndexWriterConfig {
>>
>> sb.append("ramBufferSizeMB=").append(getRAMBufferSizeMB()).append("\n");
>>
>> sb.append("maxBufferedDocs=").append(getMaxBufferedDocs()).append("\n");
>>
>> sb.append("maxBufferedDeleteTerms=").append(getMaxBufferedDeleteTerms(
>> )).append("\n");
>> -
>> sb.append("mergedSegmentWarmer=").append(getMergeScheduler()).append(
>> "\n");
>> +
>> sb.append("mergedSegmentWarmer=").append(getMergedSegmentWarmer())
>> .append("\n");
>>
>> sb.append("readerTermsIndexDivisor=").append(getReaderTermsIndexDivisor()
>> ).append("\n");
>>
>> sb.append("termIndexInterval=").append(getTermIndexInterval()).append("\n");
>> // TODO: this should be private to the codec, not settable here
>>
>> sb.append("delPolicy=").append(getIndexDeletionPolicy().getClass().getName())
>> .append("\n");
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfo
>> PerCommit.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
>> he/lucene/index/SegmentInfoPerCommit.java?rev=1393408&r1=1393407&r2=
>> 1393408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfo
>> PerCommit.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfo
>> PerCommit.java Wed Oct 3 11:01:13 2012
>> @@ -79,6 +79,7 @@ public class SegmentInfoPerCommit {
>>
>> /** Returns all files in use by this segment. */
>> public Collection<String> files() throws IOException {
>> + // Start from the wrapped info's files:
>> Collection<String> files = new HashSet<String>(info.files());
>>
>> // Must separately add any live docs files:
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.jav
>> a
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
>> he/lucene/store/Directory.java?rev=1393408&r1=1393407&r2=1393408&view
>> =diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.jav
>> a (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.jav
>> a Wed Oct 3 11:01:13 2012
>> @@ -201,7 +201,18 @@ public abstract class Directory implemen
>> } catch (IOException ioe) {
>> priorException = ioe;
>> } finally {
>> - IOUtils.closeWhileHandlingException(priorException, os, is);
>> + boolean success = false;
>> + try {
>> + IOUtils.closeWhileHandlingException(priorException, os, is);
>> + success = true;
>> + } finally {
>> + if (!success) {
>> + try {
>> + to.deleteFile(dest);
>> + } catch (Throwable t) {
>> + }
>> + }
>> + }
>> }
>> }
>>
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/Tes
>> tPerFieldPostingsFormat.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/codecs/perfield/TestPerFieldPostingsFormat.java?rev=1393408&r1=
>> 1393407&r2=1393408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/Tes
>> tPerFieldPostingsFormat.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/Tes
>> tPerFieldPostingsFormat.java Wed Oct 3 11:01:13 2012
>> @@ -33,7 +33,7 @@ public class TestPerFieldPostingsFormat
>> @Override
>> public void setUp() throws Exception {
>> super.setUp();
>> - codec = new RandomCodec(new Random(random().nextLong()),
>> Collections.EMPTY_SET);
>> + codec = new RandomCodec(new Random(random().nextLong()),
>> Collections.<String>emptySet());
>> }
>>
>> @Override
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.jav
>> a
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestCrash.java?rev=1393408&r1=1393407&r2=1393408&view
>> =diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.jav
>> a (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.jav
>> a Wed Oct 3 11:01:13 2012
>> @@ -20,12 +20,13 @@ package org.apache.lucene.index;
>> import java.io.IOException;
>> import java.util.Random;
>>
>> +import org.apache.lucene.analysis.MockAnalyzer;
>> +import org.apache.lucene.document.Document;
>> import org.apache.lucene.document.Field;
>> -import org.apache.lucene.util.LuceneTestCase;
>> +import org.apache.lucene.store.Directory;
>> import org.apache.lucene.store.MockDirectoryWrapper;
>> import org.apache.lucene.store.NoLockFactory;
>> -import org.apache.lucene.analysis.MockAnalyzer;
>> -import org.apache.lucene.document.Document;
>> +import org.apache.lucene.util.LuceneTestCase;
>>
>> public class TestCrash extends LuceneTestCase {
>>
>> @@ -67,20 +68,42 @@ public class TestCrash extends LuceneTes
>> // before any documents were added.
>> IndexWriter writer = initIndex(random(), true);
>> MockDirectoryWrapper dir = (MockDirectoryWrapper)
>> writer.getDirectory();
>> +
>> + // We create leftover files because merging could be
>> + // running when we crash:
>> + dir.setAssertNoUnrefencedFilesOnClose(false);
>> +
>> crash(writer);
>> +
>> IndexReader reader = DirectoryReader.open(dir);
>> assertTrue(reader.numDocs() < 157);
>> reader.close();
>> +
>> + // Make a new dir, copying from the crashed dir, and
>> + // open IW on it, to confirm IW "recovers" after a
>> + // crash:
>> + Directory dir2 = newDirectory(dir);
>> dir.close();
>> +
>> + new RandomIndexWriter(random(), dir2).close();
>> + dir2.close();
>> }
>>
>> public void testWriterAfterCrash() throws IOException {
>> // This test relies on being able to open a reader before any commit
>> // happened, so we must create an initial commit just to allow that, but
>> // before any documents were added.
>> + System.out.println("TEST: initIndex");
>> IndexWriter writer = initIndex(random(), true);
>> + System.out.println("TEST: done initIndex");
>> MockDirectoryWrapper dir = (MockDirectoryWrapper)
>> writer.getDirectory();
>> +
>> + // We create leftover files because merging could be
>> + // running / store files could be open when we crash:
>> + dir.setAssertNoUnrefencedFilesOnClose(false);
>> +
>> dir.setPreventDoubleWrite(false);
>> + System.out.println("TEST: now crash");
>> crash(writer);
>> writer = initIndex(random(), dir, false);
>> writer.close();
>> @@ -88,12 +111,25 @@ public class TestCrash extends LuceneTes
>> IndexReader reader = DirectoryReader.open(dir);
>> assertTrue(reader.numDocs() < 314);
>> reader.close();
>> +
>> + // Make a new dir, copying from the crashed dir, and
>> + // open IW on it, to confirm IW "recovers" after a
>> + // crash:
>> + Directory dir2 = newDirectory(dir);
>> dir.close();
>> +
>> + new RandomIndexWriter(random(), dir2).close();
>> + dir2.close();
>> }
>>
>> public void testCrashAfterReopen() throws IOException {
>> IndexWriter writer = initIndex(random(), false);
>> MockDirectoryWrapper dir = (MockDirectoryWrapper)
>> writer.getDirectory();
>> +
>> + // We create leftover files because merging could be
>> + // running when we crash:
>> + dir.setAssertNoUnrefencedFilesOnClose(false);
>> +
>> writer.close();
>> writer = initIndex(random(), dir, false);
>> assertEquals(314, writer.maxDoc());
>> @@ -111,7 +147,15 @@ public class TestCrash extends LuceneTes
>> IndexReader reader = DirectoryReader.open(dir);
>> assertTrue(reader.numDocs() >= 157);
>> reader.close();
>> +
>> + // Make a new dir, copying from the crashed dir, and
>> + // open IW on it, to confirm IW "recovers" after a
>> + // crash:
>> + Directory dir2 = newDirectory(dir);
>> dir.close();
>> +
>> + new RandomIndexWriter(random(), dir2).close();
>> + dir2.close();
>> }
>>
>> public void testCrashAfterClose() throws IOException {
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
>> Reader.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestDirectoryReader.java?rev=1393408&r1=1393407&r2=139
>> 3408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
>> Reader.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
>> Reader.java Wed Oct 3 11:01:13 2012
>> @@ -39,6 +39,7 @@ import org.apache.lucene.index.IndexWrit
>> import org.apache.lucene.search.DocIdSetIterator;
>> import org.apache.lucene.search.FieldCache;
>> import org.apache.lucene.store.Directory;
>> +import org.apache.lucene.store.MockDirectoryWrapper;
>> import org.apache.lucene.store.NoSuchDirectoryException;
>> import org.apache.lucene.util.Bits;
>> import org.apache.lucene.util.BytesRef;
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
>> ReaderReopen.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestDirectoryReaderReopen.java?rev=1393408&r1=1393407&
>> r2=1393408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
>> ReaderReopen.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
>> ReaderReopen.java Wed Oct 3 11:01:13 2012
>> @@ -38,6 +38,7 @@ import org.apache.lucene.search.IndexSea
>> import org.apache.lucene.search.ScoreDoc;
>> import org.apache.lucene.search.TermQuery;
>> import org.apache.lucene.store.Directory;
>> +import org.apache.lucene.store.MockDirectoryWrapper;
>> import org.apache.lucene.util.LuceneTestCase;
>> import org.apache.lucene.util._TestUtil;
>>
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestDoc.java?rev=1393408&r1=1393407&r2=1393408&view=
>> diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
>> (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
>> Wed Oct 3 11:01:13 2012
>> @@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexWrit
>> import org.apache.lucene.search.DocIdSetIterator;
>> import org.apache.lucene.store.Directory;
>> import org.apache.lucene.store.IOContext;
>> +import org.apache.lucene.store.MockDirectoryWrapper;
>> import org.apache.lucene.store.TrackingDirectoryWrapper;
>> import org.apache.lucene.util.Constants;
>> import org.apache.lucene.util.InfoStream;
>> @@ -112,6 +113,13 @@ public class TestDoc extends LuceneTestC
>> PrintWriter out = new PrintWriter(sw, true);
>>
>> Directory directory = newFSDirectory(indexDir, null);
>> +
>> + if (directory instanceof MockDirectoryWrapper) {
>> + // We create unreferenced files (we don't even write
>> + // a segments file):
>> + ((MockDirectoryWrapper)
>> directory).setAssertNoUnrefencedFilesOnClose(false);
>> + }
>> +
>> IndexWriter writer = new IndexWriter(
>> directory,
>> newIndexWriterConfig(TEST_VERSION_CURRENT, new
>> MockAnalyzer(random())).
>> @@ -147,6 +155,13 @@ public class TestDoc extends LuceneTestC
>> out = new PrintWriter(sw, true);
>>
>> directory = newFSDirectory(indexDir, null);
>> +
>> + if (directory instanceof MockDirectoryWrapper) {
>> + // We create unreferenced files (we don't even write
>> + // a segments file):
>> + ((MockDirectoryWrapper)
>> directory).setAssertNoUnrefencedFilesOnClose(false);
>> + }
>> +
>> writer = new IndexWriter(
>> directory,
>> newIndexWriterConfig(TEST_VERSION_CURRENT, new
>> MockAnalyzer(random())).
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
>> erExceptions.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestIndexWriterExceptions.java?rev=1393408&r1=1393407&r
>> 2=1393408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
>> erExceptions.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
>> erExceptions.java Wed Oct 3 11:01:13 2012
>> @@ -1070,6 +1070,9 @@ public class TestIndexWriterExceptions e
>> fail("segmentInfos failed to retry fallback to correct segments_N file");
>> }
>> reader.close();
>> +
>> + // should remove the corrumpted segments_N
>> + new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
>> null)).close();
>> dir.close();
>> }
>>
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
>> erWithThreads.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestIndexWriterWithThreads.java?rev=1393408&r1=1393407
>> &r2=1393408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
>> erWithThreads.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
>> erWithThreads.java Wed Oct 3 11:01:13 2012
>> @@ -284,6 +284,9 @@ public class TestIndexWriterWithThreads
>> failure.clearDoFail();
>> writer.close(false);
>> }
>> + if (VERBOSE) {
>> + System.out.println("TEST: success=" + success);
>> + }
>>
>> if (success) {
>> IndexReader reader = DirectoryReader.open(dir);
>> @@ -341,6 +344,12 @@ public class TestIndexWriterWithThreads
>> }
>> @Override
>> public void eval(MockDirectoryWrapper dir) throws IOException {
>> +
>> + // Since we throw exc during abort, eg when IW is
>> + // attempting to delete files, we will leave
>> + // leftovers:
>> + dir.setAssertNoUnrefencedFilesOnClose(false);
>> +
>> if (doFail) {
>> StackTraceElement[] trace = new Exception().getStackTrace();
>> boolean sawAbortOrFlushDoc = false;
>> @@ -405,6 +414,8 @@ public class TestIndexWriterWithThreads
>> if ("flush".equals(trace[i].getMethodName()) &&
>> "org.apache.lucene.index.DocFieldProcessor".equals(trace[i].getClassName())) {
>> if (onlyOnce)
>> doFail = false;
>> + //System.out.println(Thread.currentThread().getName() + ": NOW FAIL:
>> onlyOnce=" + onlyOnce);
>> + //new Throwable().printStackTrace(System.out);
>> throw new IOException("now failing on purpose");
>> }
>> }
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeleti
>> onPolicy.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestNoDeletionPolicy.java?rev=1393408&r1=1393407&r2=139
>> 3408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeleti
>> onPolicy.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeleti
>> onPolicy.java Wed Oct 3 11:01:13 2012
>> @@ -21,10 +21,12 @@ import java.lang.reflect.Constructor;
>> import java.lang.reflect.Method;
>> import java.lang.reflect.Modifier;
>> import java.util.Arrays;
>> +
>> import org.apache.lucene.analysis.MockAnalyzer;
>> import org.apache.lucene.document.Document;
>> import org.apache.lucene.document.Field;
>> import org.apache.lucene.store.Directory;
>> +import org.apache.lucene.store.MockDirectoryWrapper;
>> import org.apache.lucene.util.LuceneTestCase;
>> import org.junit.Test;
>>
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistent
>> SnapshotDeletionPolicy.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestPersistentSnapshotDeletionPolicy.java?rev=1393408&r1=1
>> 393407&r2=1393408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistent
>> SnapshotDeletionPolicy.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistent
>> SnapshotDeletionPolicy.java Wed Oct 3 11:01:13 2012
>> @@ -25,6 +25,7 @@ import org.apache.lucene.document.Docume
>> import org.apache.lucene.index.IndexWriterConfig.OpenMode;
>> import org.apache.lucene.store.Directory;
>> import org.apache.lucene.store.LockObtainFailedException;
>> +import org.apache.lucene.store.MockDirectoryWrapper;
>> import org.junit.After;
>> import org.junit.Before;
>> import org.junit.Test;
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUp
>> dates.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestRollingUpdates.java?rev=1393408&r1=1393407&r2=1393
>> 408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUp
>> dates.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUp
>> dates.java Wed Oct 3 11:01:13 2012
>> @@ -39,7 +39,6 @@ public class TestRollingUpdates extends
>> public void testRollingUpdates() throws Exception {
>> Random random = new Random(random().nextLong());
>> final BaseDirectoryWrapper dir = newDirectory();
>> - dir.setCheckIndexOnClose(false); // we use a custom codec provider
>> final LineFileDocs docs = new LineFileDocs(random, true);
>>
>> //provider.register(new MemoryCodec());
>> @@ -130,10 +129,11 @@ public class TestRollingUpdates extends
>> assertEquals(SIZE, w.numDocs());
>>
>> w.close();
>> +
>> + TestIndexWriter.assertNoUnreferencedFiles(dir, "leftover files after rolling
>> updates");
>> +
>> docs.close();
>>
>> - _TestUtil.checkIndex(dir);
>> -
>> // LUCENE-4455:
>> SegmentInfos infos = new SegmentInfos();
>> infos.read(dir);
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshot
>> DeletionPolicy.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestSnapshotDeletionPolicy.java?rev=1393408&r1=1393407&r
>> 2=1393408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshot
>> DeletionPolicy.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshot
>> DeletionPolicy.java Wed Oct 3 11:01:13 2012
>> @@ -17,17 +17,18 @@ package org.apache.lucene.index;
>> * limitations under the License.
>> */
>>
>> +import java.io.IOException;
>> import java.util.Collection;
>> import java.util.Map;
>> import java.util.Random;
>> -import java.io.IOException;
>>
>> +import org.apache.lucene.analysis.MockAnalyzer;
>> import org.apache.lucene.document.Document;
>> import org.apache.lucene.document.FieldType;
>> import org.apache.lucene.document.TextField;
>> import org.apache.lucene.store.Directory;
>> import org.apache.lucene.store.IndexInput;
>> -import org.apache.lucene.analysis.MockAnalyzer;
>> +import org.apache.lucene.store.MockDirectoryWrapper;
>> import org.apache.lucene.util.LuceneTestCase;
>> import org.apache.lucene.util.ThreadInterruptedException;
>> import org.junit.Test;
>> @@ -313,6 +314,7 @@ public class TestSnapshotDeletionPolicy
>> public void testRollbackToOldSnapshot() throws Exception {
>> int numSnapshots = 2;
>> Directory dir = newDirectory();
>> +
>> SnapshotDeletionPolicy sdp = getDeletionPolicy();
>> IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp));
>> prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
>> @@ -325,10 +327,11 @@ public class TestSnapshotDeletionPolicy
>> writer.deleteUnusedFiles();
>> assertSnapshotExists(dir, sdp, numSnapshots - 1);
>> writer.close();
>> -
>> +
>> // but 'snapshot1' files will still exist (need to release snapshot before they
>> can be deleted).
>> String segFileName = sdp.getSnapshot("snapshot1").getSegmentsFileName();
>> assertTrue("snapshot files should exist in the directory: " + segFileName,
>> dir.fileExists(segFileName));
>> +
>> dir.close();
>> }
>>
>> @@ -385,6 +388,7 @@ public class TestSnapshotDeletionPolicy
>> @Test
>> public void testSnapshotLastCommitTwice() throws Exception {
>> Directory dir = newDirectory();
>> +
>> SnapshotDeletionPolicy sdp = getDeletionPolicy();
>> IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp));
>> writer.addDocument(new Document());
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
>> onRollback.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestTransactionRollback.java?rev=1393408&r1=1393407&r2=
>> 1393408&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
>> onRollback.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
>> onRollback.java Wed Oct 3 11:01:13 2012
>> @@ -21,17 +21,18 @@ package org.apache.lucene.index;
>> import java.io.IOException;
>> import java.util.BitSet;
>> import java.util.Collection;
>> +import java.util.HashMap;
>> import java.util.Iterator;
>> import java.util.List;
>> import java.util.Map;
>> -import java.util.HashMap;
>>
>> -import org.apache.lucene.document.Field;
>> -import org.apache.lucene.util.LuceneTestCase;
>> import org.apache.lucene.analysis.MockAnalyzer;
>> import org.apache.lucene.document.Document;
>> +import org.apache.lucene.document.Field;
>> import org.apache.lucene.store.Directory;
>> +import org.apache.lucene.store.MockDirectoryWrapper;
>> import org.apache.lucene.util.Bits;
>> +import org.apache.lucene.util.LuceneTestCase;
>>
>> /**
>> * Test class to illustrate using IndexDeletionPolicy to provide multi-level
>> rollback capability.
>> @@ -55,13 +56,16 @@ public class TestTransactionRollback ext
>> for (Iterator<IndexCommit> iterator = commits.iterator();
>> iterator.hasNext();) {
>> IndexCommit commit = iterator.next();
>> Map<String,String> ud=commit.getUserData();
>> - if (ud.size() > 0)
>> - if (ud.get("index").endsWith(ids))
>> - last=commit;
>> + if (ud.size() > 0) {
>> + if (ud.get("index").endsWith(ids)) {
>> + last = commit;
>> + }
>> + }
>> }
>>
>> - if (last==null)
>> + if (last==null) {
>> throw new RuntimeException("Couldn't find commit point "+id);
>> + }
>>
>> IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
>> TEST_VERSION_CURRENT, new
>> MockAnalyzer(random())).setIndexDeletionPolicy(
>> @@ -74,13 +78,13 @@ public class TestTransactionRollback ext
>>
>> public void testRepeatedRollBacks() throws Exception {
>>
>> - int expectedLastRecordId=100;
>> + int expectedLastRecordId = 100;
>> while (expectedLastRecordId>10) {
>> - expectedLastRecordId -=10;
>> + expectedLastRecordId -= 10;
>> rollBackLast(expectedLastRecordId);
>>
>> BitSet expecteds = new BitSet(100);
>> - expecteds.set(1,(expectedLastRecordId+1),true);
>> + expecteds.set(1, (expectedLastRecordId+1), true);
>> checkExpecteds(expecteds);
>> }
>> }
>> @@ -125,6 +129,7 @@ public class TestTransactionRollback ext
>> public void setUp() throws Exception {
>> super.setUp();
>> dir = newDirectory();
>> +
>> //Build index, of records 1 to 100, committing after each batch of 10
>> IndexDeletionPolicy sdp=new KeepAllDeletionPolicy();
>> IndexWriter w=new IndexWriter(dir, newIndexWriterConfig(
>> TEST_VERSION_CURRENT, new
>> MockAnalyzer(random())).setIndexDeletionPolicy(sdp));
>> @@ -199,6 +204,7 @@ public class TestTransactionRollback ext
>> }
>>
>> public void testRollbackDeletionPolicy() throws Exception {
>> +
>> for(int i=0;i<2;i++) {
>> // Unless you specify a prior commit point, rollback
>> // should not work:
>>
>> Modified:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
>> ons.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/index/TestTransactions.java?rev=1393408&r1=1393407&r2=139340
>> 8&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
>> ons.java (original)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
>> ons.java Wed Oct 3 11:01:13 2012
>> @@ -215,6 +215,11 @@ public class TestTransactions extends Lu
>> dir1.setFailOnOpenInput(false);
>> dir2.setFailOnOpenInput(false);
>>
>> + // We throw exceptions in deleteFile, which creates
>> + // leftover files:
>> + dir1.setAssertNoUnrefencedFilesOnClose(false);
>> + dir2.setAssertNoUnrefencedFilesOnClose(false);
>> +
>> initIndex(dir1);
>> initIndex(dir2);
>>
>>
>> Added:
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/Te
>> stFailIfUnreferencedFiles.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
>> he/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java?rev=1393408&vie
>> w=auto
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/Te
>> stFailIfUnreferencedFiles.java (added)
>> +++
>> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/Te
>> stFailIfUnreferencedFiles.java Wed Oct 3 11:01:13 2012
>> @@ -0,0 +1,58 @@
>> +package org.apache.lucene.util.junitcompat;
>> +
>> +/*
>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>> + * contributor license agreements. See the NOTICE file distributed with
>> + * this work for additional information regarding copyright ownership.
>> + * The ASF licenses this file to You under the Apache License, Version 2.0
>> + * (the "License"); you may not use this file except in compliance with
>> + * the License. You may obtain a copy of the License at
>> + *
>> + * http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing, software
>> + * distributed under the License is distributed on an "AS IS" BASIS,
>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>> implied.
>> + * See the License for the specific language governing permissions and
>> + * limitations under the License.
>> + */
>> +
>> +import java.util.Collections;
>> +
>> +import org.apache.lucene.document.Document;
>> +import org.apache.lucene.index.IndexWriter;
>> +import org.apache.lucene.index.IndexWriterConfig;
>> +import org.apache.lucene.store.Directory;
>> +import org.apache.lucene.store.IOContext;
>> +import org.apache.lucene.store.IndexOutput;
>> +import org.junit.Assert;
>> +import org.junit.Test;
>> +import org.junit.runner.JUnitCore;
>> +import org.junit.runner.Result;
>> +
>> +// LUCENE-4456: Test that we fail if there are unreferenced files
>> +public class TestFailIfUnreferencedFiles extends WithNestedTests {
>> + public TestFailIfUnreferencedFiles() {
>> + super(true);
>> + }
>> +
>> + public static class Nested1 extends WithNestedTests.AbstractNestedTest {
>> + public void testDummy() throws Exception {
>> + Directory dir = newMockDirectory();
>> + IndexWriter iw = new IndexWriter(dir, new
>> IndexWriterConfig(TEST_VERSION_CURRENT, null));
>> + iw.addDocument(new Document());
>> + iw.close();
>> + IndexOutput output = dir.createOutput("_hello.world",
>> IOContext.DEFAULT);
>> + output.writeString("i am unreferenced!");
>> + output.close();
>> + dir.sync(Collections.singleton("_hello.world"));
>> + dir.close();
>> + }
>> + }
>> +
>> + @Test
>> + public void testFailIfUnreferencedFiles() {
>> + Result r = JUnitCore.runClasses(Nested1.class);
>> + Assert.assertEquals(1, r.getFailureCount());
>> + }
>> +}
>>
>> Modified:
>> lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplit
>> ter.java
>> URL:
>> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apac
>> he/lucene/index/TestIndexSplitter.java?rev=1393408&r1=1393407&r2=139340
>> 8&view=diff
>> ================================================================
>> ==============
>> ---
>> lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplit
>> ter.java (original)
>> +++
>> lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplit
>> ter.java Wed Oct 3 11:01:13 2012
>> @@ -22,6 +22,7 @@ import org.apache.lucene.analysis.MockAn
>> import org.apache.lucene.document.Document;
>> import org.apache.lucene.index.IndexWriterConfig.OpenMode;
>> import org.apache.lucene.store.Directory;
>> +import org.apache.lucene.store.MockDirectoryWrapper;
>> import org.apache.lucene.util.LuceneTestCase;
>> import org.apache.lucene.util._TestUtil;
>>
>> @@ -34,6 +35,11 @@ public class TestIndexSplitter extends L
>> _TestUtil.rmDir(destDir);
>> destDir.mkdirs();
>> Directory fsDir = newFSDirectory(dir);
>> + // IndexSplitter.split makes its own commit directly with SIPC/SegmentInfos,
>> + // so the unreferenced files are expected.
>> + if (fsDir instanceof MockDirectoryWrapper) {
>> +
>> ((MockDirectoryWrapper)fsDir).setAssertNoUnrefencedFilesOnClose(false);
>> + }
>>
>> LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
>> mergePolicy.setNoCFSRatio(1.0);
>>
>> Modified: lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1
>> 393408&r1=1393407&r2=1393408&view=diff
>> ================================================================
>> ==============
>> --- lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
>> (original)
>> +++ lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Wed
>> Oct 3 11:01:13 2012
>> @@ -402,10 +402,16 @@ public class RandomIndexWriter implement
>> final int segCount = w.getSegmentCount();
>> if (r.nextBoolean() || segCount == 0) {
>> // full forceMerge
>> + if (LuceneTestCase.VERBOSE) {
>> + System.out.println("RIW: doRandomForceMerge(1)");
>> + }
>> w.forceMerge(1);
>> } else {
>> // partial forceMerge
>> final int limit = _TestUtil.nextInt(r, 1, segCount);
>> + if (LuceneTestCase.VERBOSE) {
>> + System.out.println("RIW: doRandomForceMerge(" + limit + ")");
>> + }
>> w.forceMerge(limit);
>> assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit:
>> "limit=" + limit + " actual=" + w.getSegmentCount();
>> }
>>
>> Modified: lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev
>> =1393408&r1=1393407&r2=1393408&view=diff
>> ================================================================
>> ==============
>> --- lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
>> (original)
>> +++ lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
>> Wed Oct 3 11:01:13 2012
>> @@ -28,14 +28,18 @@ import java.util.HashMap;
>> import java.util.HashSet;
>> import java.util.IdentityHashMap;
>> import java.util.Iterator;
>> +import java.util.List;
>> import java.util.Map;
>> import java.util.Random;
>> import java.util.Set;
>> +import java.util.TreeSet;
>> import java.util.concurrent.atomic.AtomicInteger;
>>
>> import org.apache.lucene.index.DirectoryReader;
>> import org.apache.lucene.index.IndexWriter;
>> import org.apache.lucene.index.IndexWriterConfig;
>> +import org.apache.lucene.index.NoDeletionPolicy;
>> +import org.apache.lucene.index.SegmentInfos;
>> import org.apache.lucene.util.LuceneTestCase;
>> import org.apache.lucene.util._TestUtil;
>>
>> @@ -204,15 +208,17 @@ public class MockDirectoryWrapper extend
>> // first force-close all files, so we can corrupt on windows etc.
>> // clone the file map, as these guys want to remove themselves on close.
>> Map<Closeable,Exception> m = new
>> IdentityHashMap<Closeable,Exception>(openFileHandles);
>> - for (Closeable f : m.keySet())
>> + for (Closeable f : m.keySet()) {
>> try {
>> f.close();
>> } catch (Exception ignored) {}
>> + }
>>
>> while(it.hasNext()) {
>> String name = it.next();
>> int damage = randomState.nextInt(5);
>> String action = null;
>> +
>> if (damage == 0) {
>> action = "deleted";
>> deleteFile(name, true);
>> @@ -544,6 +550,9 @@ public class MockDirectoryWrapper extend
>>
>> @Override
>> public synchronized void close() throws IOException {
>> + // files that we tried to delete, but couldn't because readers were open.
>> + // all that matters is that we tried! (they will eventually go away)
>> + Set<String> pendingDeletions = new HashSet<String>(openFilesDeleted);
>> maybeYield();
>> if (openFiles == null) {
>> openFiles = new HashMap<String,Integer>();
>> @@ -574,17 +583,91 @@ public class MockDirectoryWrapper extend
>> }
>> _TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose());
>>
>> + // TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
>> if (assertNoUnreferencedFilesOnClose) {
>> - // now look for unreferenced files:
>> - String[] startFiles = listAll();
>> - new IndexWriter(this, new
>> IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null)).rollback();
>> - String[] endFiles = listAll();
>> + // now look for unreferenced files: discount ones that we tried to delete
>> but could not
>> + Set<String> allFiles = new HashSet<String>(Arrays.asList(listAll()));
>> + allFiles.removeAll(pendingDeletions);
>> + String[] startFiles = allFiles.toArray(new String[0]);
>> + IndexWriterConfig iwc = new
>> IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null);
>> + iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
>> + new IndexWriter(delegate, iwc).rollback();
>> + String[] endFiles = delegate.listAll();
>> +
>> + Set<String> startSet = new TreeSet<String>(Arrays.asList(startFiles));
>> + Set<String> endSet = new TreeSet<String>(Arrays.asList(endFiles));
>> +
>> + if (pendingDeletions.contains("segments.gen") &&
>> endSet.contains("segments.gen")) {
>> + // this is possible if we hit an exception while writing segments.gen, we
>> try to delete it
>> + // and it ends out in pendingDeletions (but IFD wont remove this).
>> + startSet.add("segments.gen");
>> + if (LuceneTestCase.VERBOSE) {
>> + System.out.println("MDW: Unreferenced check: Ignoring
>> segments.gen that we could not delete.");
>> + }
>> + }
>> +
>> + // its possible we cannot delete the segments_N on windows if someone
>> has it open and
>> + // maybe other files too, depending on timing. normally someone on
>> windows wouldnt have
>> + // an issue (IFD would nuke this stuff eventually), but we pass
>> NoDeletionPolicy...
>> + for (String file : pendingDeletions) {
>> + if (file.startsWith("segments") && !file.equals("segments.gen") &&
>> endSet.contains(file)) {
>> + startSet.add(file);
>> + if (LuceneTestCase.VERBOSE) {
>> + System.out.println("MDW: Unreferenced check: Ignoring segments
>> file: " + file + " that we could not delete.");
>> + }
>> + try {
>> + SegmentInfos sis = new SegmentInfos();
>> + sis.read(delegate, file);
>> + Set<String> ghosts = new HashSet<String>(sis.files(delegate, false));
>> + for (String s : ghosts) {
>> + if (endSet.contains(s) && !startSet.contains(s)) {
>> + assert pendingDeletions.contains(s);
>> + if (LuceneTestCase.VERBOSE) {
>> + System.out.println("MDW: Unreferenced check: Ignoring
>> referenced file: " + s + " " +
>> + "from " + file + " that we could not delete.");
>> + }
>> + startSet.add(s);
>> + }
>> + }
>> + } catch (Throwable ignore) {}
>> + }
>> + }
>>
>> - Arrays.sort(startFiles);
>> - Arrays.sort(endFiles);
>> + startFiles = startSet.toArray(new String[0]);
>> + endFiles = endSet.toArray(new String[0]);
>>
>> if (!Arrays.equals(startFiles, endFiles)) {
>> - assert false : "unreferenced files: before delete:\n " +
>> Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles);
>> + StringBuilder sb = new StringBuilder();
>> + List<String> removed = new ArrayList<String>();
>> + for(String fileName : startFiles) {
>> + if (!endSet.contains(fileName)) {
>> + removed.add(fileName);
>> + }
>> + }
>> +
>> + List<String> added = new ArrayList<String>();
>> + for(String fileName : endFiles) {
>> + if (!startSet.contains(fileName)) {
>> + added.add(fileName);
>> + }
>> + }
>> +
>> + String extras;
>> + if (removed.size() != 0) {
>> + extras = "\n\nThese files were removed: " + removed;
>> + } else {
>> + extras = "";
>> + }
>> +
>> + if (added.size() != 0) {
>> + extras += "\n\nThese files were added (waaaaaaaaaat!): " + added;
>> + }
>> +
>> + if (pendingDeletions.size() != 0) {
>> + extras += "\n\nThese files we had previously tried to delete, but
>> couldn't: " + pendingDeletions;
>> + }
>> +
>> + assert false : "unreferenced files: before delete:\n " +
>> Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles) +
>> extras;
>> }
>>
>> DirectoryReader ir1 = DirectoryReader.open(this);
>> @@ -607,7 +690,6 @@ public class MockDirectoryWrapper extend
>> if (v != null) {
>> if (v.intValue() == 1) {
>> openFiles.remove(name);
>> - openFilesDeleted.remove(name);
>> } else {
>> v = Integer.valueOf(v.intValue()-1);
>> openFiles.put(name, v);
>>
>> Modified: lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/util/LuceneTestCase.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=139340
>> 8&r1=1393407&r2=1393408&view=diff
>> ================================================================
>> ==============
>> --- lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
>> +++ lucene/dev/trunk/lucene/test-
>> framework/src/java/org/apache/lucene/util/LuceneTestCase.java Wed Oct 3
>> 11:01:13 2012
>> @@ -948,6 +948,7 @@ public abstract class LuceneTestCase ext
>> if (rarely(random)) {
>> directory = new NRTCachingDirectory(directory, random.nextDouble(),
>> random.nextDouble());
>> }
>> +
>> if (bare) {
>> BaseDirectoryWrapper base = new BaseDirectoryWrapper(directory);
>> closeAfterSuite(new CloseableDirectory(base, suiteFailureMarker));
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
codecs/src/java/org/apache/lucene/codecs/simpletext/ core/src/java/org/apache/lucene/index/
core/src/java/org/apache/lucene/store/ core/src/test/org/apache/lucene/codecs/perfield/
core/src/te
Posted by Michael McCandless <lu...@mikemccandless.com>.
On Wed, Oct 3, 2012 at 12:33 PM, Uwe Schindler <uw...@thetaphi.de> wrote:
> Then we should nuke it. The commits are not lost, you can always look at them after deletion. Just open your repository browser and enter an older revision number! I just don’t want to make the "play-around" branches be first-class citicens of the Lucene / Solr repository.
OK I agree: I'll go nuke it.
Mike McCandless
http://blog.mikemccandless.com
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
RE: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./ codecs/src/java/org/apache/lucene/codecs/simpletext/ core/src/java/org/apache/lucene/index/ core/src/java/org/apache/lucene/store/ core/src/test/org/apache/lucene/codecs/perfield/ core/src/te
Posted by Uwe Schindler <uw...@thetaphi.de>.
Then we should nuke it. The commits are not lost, you can always look at them after deletion. Just open your repository browser and enter an older revision number! I just don’t want to make the "play-around" branches be first-class citicens of the Lucene / Solr repository.
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: Robert Muir [mailto:rcmuir@gmail.com]
> Sent: Wednesday, October 03, 2012 6:14 PM
> To: dev@lucene.apache.org
> Subject: Re: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
> codecs/src/java/org/apache/lucene/codecs/simpletext/
> core/src/java/org/apache/lucene/index/
> core/src/java/org/apache/lucene/store/
> core/src/test/org/apache/lucene/codecs/perfield/ core/src/te
>
> I created a patch because the branch had lots of back-and-forth (essentially
> noise) from mike and I.
>
> We also dont move anything around or anyting like that (its just fixing
> bugs/tests really) so a patch to me made more sense...
>
> The branch was just so we could easily collaborate and set my jenkins on it, etc,
> etc.
>
> On Wed, Oct 3, 2012 at 12:10 PM, Uwe Schindler <uw...@thetaphi.de> wrote:
> > Why did you not merge-reintegrate?
> >
> > -----
> > Uwe Schindler
> > H.-H.-Meier-Allee 63, D-28213 Bremen
> > http://www.thetaphi.de
> > eMail: uwe@thetaphi.de
> >
> >
> >> -----Original Message-----
> >> From: Michael McCandless [mailto:lucene@mikemccandless.com]
> >> Sent: Wednesday, October 03, 2012 6:04 PM
> >> To: dev@lucene.apache.org
> >> Subject: Re: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
> >> codecs/src/java/org/apache/lucene/codecs/simpletext/
> >> core/src/java/org/apache/lucene/index/
> >> core/src/java/org/apache/lucene/store/
> >> core/src/test/org/apache/lucene/codecs/perfield/ core/src/te
> >>
> >> On Wed, Oct 3, 2012 at 8:29 AM, Uwe Schindler <uw...@thetaphi.de> wrote:
> >> > Yes :)
> >> > It is still there and can be viewed (using @rev in URL), but it
> >> > should not be
> >> visible as "active". We already have way too many branches, which
> >> seem to be dead, we should cleanup.
> >>
> >> Hmm, though, I'm nervous destroying commit history ... I don't think
> >> in this case it'd be useful ... but maybe it could be?
> >>
> >> I don't think we have THAT many branches :)
> >>
> >> Mike McCandless
> >>
> >> http://blog.mikemccandless.com
> >>
> >> ---------------------------------------------------------------------
> >> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For
> >> additional commands, e-mail: dev-help@lucene.apache.org
> >
> >
> > ---------------------------------------------------------------------
> > To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For
> > additional commands, e-mail: dev-help@lucene.apache.org
> >
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For additional
> commands, e-mail: dev-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
codecs/src/java/org/apache/lucene/codecs/simpletext/ core/src/java/org/apache/lucene/index/
core/src/java/org/apache/lucene/store/ core/src/test/org/apache/lucene/codecs/perfield/
core/src/te
Posted by Robert Muir <rc...@gmail.com>.
I created a patch because the branch had lots of back-and-forth
(essentially noise) from mike and I.
We also dont move anything around or anyting like that (its just
fixing bugs/tests really) so a patch to me made more sense...
The branch was just so we could easily collaborate and set my jenkins
on it, etc, etc.
On Wed, Oct 3, 2012 at 12:10 PM, Uwe Schindler <uw...@thetaphi.de> wrote:
> Why did you not merge-reintegrate?
>
> -----
> Uwe Schindler
> H.-H.-Meier-Allee 63, D-28213 Bremen
> http://www.thetaphi.de
> eMail: uwe@thetaphi.de
>
>
>> -----Original Message-----
>> From: Michael McCandless [mailto:lucene@mikemccandless.com]
>> Sent: Wednesday, October 03, 2012 6:04 PM
>> To: dev@lucene.apache.org
>> Subject: Re: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
>> codecs/src/java/org/apache/lucene/codecs/simpletext/
>> core/src/java/org/apache/lucene/index/
>> core/src/java/org/apache/lucene/store/
>> core/src/test/org/apache/lucene/codecs/perfield/ core/src/te
>>
>> On Wed, Oct 3, 2012 at 8:29 AM, Uwe Schindler <uw...@thetaphi.de> wrote:
>> > Yes :)
>> > It is still there and can be viewed (using @rev in URL), but it should not be
>> visible as "active". We already have way too many branches, which seem to be
>> dead, we should cleanup.
>>
>> Hmm, though, I'm nervous destroying commit history ... I don't think in this
>> case it'd be useful ... but maybe it could be?
>>
>> I don't think we have THAT many branches :)
>>
>> Mike McCandless
>>
>> http://blog.mikemccandless.com
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For additional
>> commands, e-mail: dev-help@lucene.apache.org
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
RE: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./ codecs/src/java/org/apache/lucene/codecs/simpletext/ core/src/java/org/apache/lucene/index/ core/src/java/org/apache/lucene/store/ core/src/test/org/apache/lucene/codecs/perfield/ core/src/te
Posted by Uwe Schindler <uw...@thetaphi.de>.
Why did you not merge-reintegrate?
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: Michael McCandless [mailto:lucene@mikemccandless.com]
> Sent: Wednesday, October 03, 2012 6:04 PM
> To: dev@lucene.apache.org
> Subject: Re: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
> codecs/src/java/org/apache/lucene/codecs/simpletext/
> core/src/java/org/apache/lucene/index/
> core/src/java/org/apache/lucene/store/
> core/src/test/org/apache/lucene/codecs/perfield/ core/src/te
>
> On Wed, Oct 3, 2012 at 8:29 AM, Uwe Schindler <uw...@thetaphi.de> wrote:
> > Yes :)
> > It is still there and can be viewed (using @rev in URL), but it should not be
> visible as "active". We already have way too many branches, which seem to be
> dead, we should cleanup.
>
> Hmm, though, I'm nervous destroying commit history ... I don't think in this
> case it'd be useful ... but maybe it could be?
>
> I don't think we have THAT many branches :)
>
> Mike McCandless
>
> http://blog.mikemccandless.com
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For additional
> commands, e-mail: dev-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
codecs/src/java/org/apache/lucene/codecs/simpletext/ core/src/java/org/apache/lucene/index/
core/src/java/org/apache/lucene/store/ core/src/test/org/apache/lucene/codecs/perfield/
core/src/te
Posted by Michael McCandless <lu...@mikemccandless.com>.
On Wed, Oct 3, 2012 at 8:29 AM, Uwe Schindler <uw...@thetaphi.de> wrote:
> Yes :)
> It is still there and can be viewed (using @rev in URL), but it should not be visible as "active". We already have way too many branches, which seem to be dead, we should cleanup.
Hmm, though, I'm nervous destroying commit history ... I don't think
in this case it'd be useful ... but maybe it could be?
I don't think we have THAT many branches :)
Mike McCandless
http://blog.mikemccandless.com
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
RE: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./ codecs/src/java/org/apache/lucene/codecs/simpletext/ core/src/java/org/apache/lucene/index/ core/src/java/org/apache/lucene/store/ core/src/test/org/apache/lucene/codecs/perfield/ core/src/te
Posted by Uwe Schindler <uw...@thetaphi.de>.
Yes :)
It is still there and can be viewed (using @rev in URL), but it should not be visible as "active". We already have way too many branches, which seem to be dead, we should cleanup.
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: Michael McCandless [mailto:lucene@mikemccandless.com]
> Sent: Wednesday, October 03, 2012 2:24 PM
> To: dev@lucene.apache.org
> Subject: Re: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
> codecs/src/java/org/apache/lucene/codecs/simpletext/
> core/src/java/org/apache/lucene/index/
> core/src/java/org/apache/lucene/store/
> core/src/test/org/apache/lucene/codecs/perfield/ core/src/te
>
> I didn't re-integrate; I just committed the patch.
>
> But I can remove the branch anyway if you want...?
>
> Mike McCandless
>
> http://blog.mikemccandless.com
>
> On Wed, Oct 3, 2012 at 8:19 AM, Uwe Schindler <uw...@thetaphi.de> wrote:
> > Can you remove the branch after reintegrate?
> >
> > -----
> > Uwe Schindler
> > H.-H.-Meier-Allee 63, D-28213 Bremen
> > http://www.thetaphi.de
> > eMail: uwe@thetaphi.de
> >
> >
> >> -----Original Message-----
> >> From: mikemccand@apache.org [mailto:mikemccand@apache.org]
> >> Sent: Wednesday, October 03, 2012 1:01 PM
> >> To: commits@lucene.apache.org
> >> Subject: svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./
> >> codecs/src/java/org/apache/lucene/codecs/simpletext/
> >> core/src/java/org/apache/lucene/index/
> >> core/src/java/org/apache/lucene/store/
> >> core/src/test/org/apache/lucene/codecs/perfield/ core/src/test/org...
> >>
> >> Author: mikemccand
> >> Date: Wed Oct 3 11:01:13 2012
> >> New Revision: 1393408
> >>
> >> URL: http://svn.apache.org/viewvc?rev=1393408&view=rev
> >> Log:
> >> LUCENE-4456: fix various bugs (see lucene/CHANGES.txt)
> >>
> >> Added:
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/Te
> >> stFailIfUnreferencedFiles.java (with props)
> >> Modified:
> >> lucene/dev/trunk/lucene/CHANGES.txt
> >>
> >>
> lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletex
> >> t/SimpleTextDocValuesConsumer.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsW
> >> riterPerThread.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDele
> >> ter.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.j
> >> ava
> >>
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWri
> >> terConfig.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfo
> >> PerCommit.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.jav
> >> a
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/Tes
> >> tPerFieldPostingsFormat.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.jav
> >> a
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
> >> Reader.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
> >> ReaderReopen.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
> >> erExceptions.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
> >> erWithThreads.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeleti
> >> onPolicy.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistent
> >> SnapshotDeletionPolicy.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUp
> >> dates.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshot
> >> DeletionPolicy.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
> >> onRollback.java
> >>
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
> >> ons.java
> >>
> >>
> lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplit
> >> ter.java
> >> lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
> >> lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
> >> lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/util/LuceneTestCase.java
> >>
> >> Modified: lucene/dev/trunk/lucene/CHANGES.txt
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1393
> >> 408&r1=1393407&r2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> --- lucene/dev/trunk/lucene/CHANGES.txt (original)
> >> +++ lucene/dev/trunk/lucene/CHANGES.txt Wed Oct 3 11:01:13 2012
> >> @@ -226,6 +226,14 @@ Bug Fixes
> >> CheckIndex that would report no deletions when a segment has
> >> deletions, and vice/versa. (Uwe Schindler, Robert Muir, Mike
> McCandless)
> >>
> >> +* LUCENE-4456: Fixed double-counting sizeInBytes for a segment
> >> + (affects how merge policies pick merges); fixed CheckIndex's
> >> + incorrect reporting of whether a segment has deletions; fixed case
> >> + where on abort Lucene could remove files it didn't create; fixed
> >> + many cases where IndexWriter could leave leftover files (on
> >> + exception in various places, on reuse of a segment name after crash
> >> + and recovery. (Uwe Schindler, Robert Muir, Mike McCandless)
> >> +
> >> Optimizations
> >>
> >> * LUCENE-4322: Decrease lucene-core JAR size. The core JAR size had
> >> increased a
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletex
> >> t/SimpleTextDocValuesConsumer.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/ap
> >>
> ache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java?rev=1393
> >> 408&r1=1393407&r2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletex
> >> t/SimpleTextDocValuesConsumer.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletex
> >> t/SimpleTextDocValuesConsumer.java Wed Oct 3 11:01:13 2012
> >> @@ -198,6 +198,7 @@ public class SimpleTextDocValuesConsumer
> >> IOUtils.close(output);
> >> } else {
> >> IOUtils.closeWhileHandlingException(output);
> >> + dir.deleteFile(fileName);
> >> }
> >> }
> >> }
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsW
> >> riterPerThread.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
> >>
> he/lucene/index/DocumentsWriterPerThread.java?rev=1393408&r1=1393407&
> >> r2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsW
> >> riterPerThread.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsW
> >> riterPerThread.java Wed Oct 3 11:01:13 2012
> >> @@ -137,6 +137,7 @@ class DocumentsWriterPerThread {
> >> * currently buffered docs. This resets our state,
> >> * discarding any docs added since last flush. */
> >> void abort() {
> >> + //System.out.println(Thread.currentThread().getName() + ": now abort
> >> seg=" + segmentInfo.name);
> >> hasAborted = aborting = true;
> >> try {
> >> if (infoStream.isEnabled("DWPT")) {
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDele
> >> ter.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
> >>
> he/lucene/index/IndexFileDeleter.java?rev=1393408&r1=1393407&r2=139340
> >> 8&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDele
> >> ter.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDele
> >> ter.java Wed Oct 3 11:01:13 2012
> >> @@ -333,12 +333,16 @@ final class IndexFileDeleter {
> >> segmentPrefix2 = null;
> >> }
> >>
> >> + Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
> >> +
> >> for(int i=0;i<files.length;i++) {
> >> String fileName = files[i];
> >> + m.reset(fileName);
> >> if ((segmentName == null || fileName.startsWith(segmentPrefix1) ||
> >> fileName.startsWith(segmentPrefix2)) &&
> >> !fileName.endsWith("write.lock") &&
> >> !refCounts.containsKey(fileName) &&
> >> - !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
> >> + !fileName.equals(IndexFileNames.SEGMENTS_GEN) &&
> >> + (m.matches() || fileName.startsWith(IndexFileNames.SEGMENTS))) {
> >> // Unreferenced file, so remove it
> >> if (infoStream.isEnabled("IFD")) {
> >> infoStream.message("IFD", "refresh [prefix=" + segmentName + "]:
> >> removing newly created unreferenced file \"" + fileName + "\"");
> >> @@ -554,7 +558,13 @@ final class IndexFileDeleter {
> >> void deleteNewFiles(Collection<String> files) throws IOException {
> >> assert locked();
> >> for (final String fileName: files) {
> >> - if (!refCounts.containsKey(fileName)) {
> >> + // NOTE: it's very unusual yet possible for the
> >> + // refCount to be present and 0: it can happen if you
> >> + // open IW on a crashed index, and it removes a bunch
> >> + // of unref'd files, and then you add new docs / do
> >> + // merging, and it reuses that segment name.
> >> + // TestCrash.testCrashAfterReopen can hit this:
> >> + if (!refCounts.containsKey(fileName) || refCounts.get(fileName).count
> ==
> >> 0) {
> >> if (infoStream.isEnabled("IFD")) {
> >> infoStream.message("IFD", "delete new file \"" + fileName + "\"");
> >> }
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.j
> >> ava
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
> >>
> he/lucene/index/IndexWriter.java?rev=1393408&r1=1393407&r2=1393408&vi
> >> ew=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.j
> >> ava (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.j
> >> ava Wed Oct 3 11:01:13 2012
> >> @@ -2145,7 +2145,7 @@ public class IndexWriter implements Clos
> >> // Now build compound file
> >> Collection<String> oldFiles = createCompoundFile(infoStream,
> directory,
> >> MergeState.CheckAbort.NONE, newSegment.info, context);
> >> newSegment.info.setUseCompoundFile(true);
> >> -
> >> +
> >> synchronized(this) {
> >> deleter.deleteNewFiles(oldFiles);
> >> }
> >> @@ -2321,31 +2321,62 @@ public class IndexWriter implements Clos
> >> flush(false, true);
> >>
> >> List<SegmentInfoPerCommit> infos = new
> >> ArrayList<SegmentInfoPerCommit>();
> >> - for (Directory dir : dirs) {
> >> - if (infoStream.isEnabled("IW")) {
> >> - infoStream.message("IW", "addIndexes: process directory " + dir);
> >> - }
> >> - SegmentInfos sis = new SegmentInfos(); // read infos from dir
> >> - sis.read(dir);
> >> -
> >> - for (SegmentInfoPerCommit info : sis) {
> >> - assert !infos.contains(info): "dup info dir=" + info.info.dir + " name="
> +
> >> info.info.name;
> >> -
> >> - String newSegName = newSegmentName();
> >> - String dsName = info.info.name;
> >>
> >> + boolean success = false;
> >> + try {
> >> + for (Directory dir : dirs) {
> >> if (infoStream.isEnabled("IW")) {
> >> - infoStream.message("IW", "addIndexes: process segment
> origName="
> >> + info.info.name + " newName=" + newSegName + " dsName=" + dsName +
> "
> >> info=" + info);
> >> + infoStream.message("IW", "addIndexes: process directory " + dir);
> >> }
> >> + SegmentInfos sis = new SegmentInfos(); // read infos from dir
> >> + sis.read(dir);
> >> +
> >> + for (SegmentInfoPerCommit info : sis) {
> >> + assert !infos.contains(info): "dup info dir=" + info.info.dir + "
> name=" +
> >> info.info.name;
> >> +
> >> + String newSegName = newSegmentName();
> >> + String dsName = info.info.name;
> >>
> >> - IOContext context = new IOContext(new
> >> MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
> >> + if (infoStream.isEnabled("IW")) {
> >> + infoStream.message("IW", "addIndexes: process segment
> origName="
> >> + info.info.name + " newName=" + newSegName + " dsName=" + dsName +
> "
> >> info=" + info);
> >> + }
> >> +
> >> + IOContext context = new IOContext(new
> >> MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
> >>
> >> - infos.add(copySegmentAsIs(info, newSegName, context));
> >> + infos.add(copySegmentAsIs(info, newSegName, context));
> >> + }
> >> + }
> >> + success = true;
> >> + } finally {
> >> + if (!success) {
> >> + for(SegmentInfoPerCommit sipc : infos) {
> >> + for(String file : sipc.files()) {
> >> + try {
> >> + directory.deleteFile(file);
> >> + } catch (Throwable t) {
> >> + }
> >> + }
> >> + }
> >> }
> >> }
> >>
> >> synchronized (this) {
> >> - ensureOpen();
> >> + success = false;
> >> + try {
> >> + ensureOpen();
> >> + success = true;
> >> + } finally {
> >> + if (!success) {
> >> + for(SegmentInfoPerCommit sipc : infos) {
> >> + for(String file : sipc.files()) {
> >> + try {
> >> + directory.deleteFile(file);
> >> + } catch (Throwable t) {
> >> + }
> >> + }
> >> + }
> >> + }
> >> + }
> >> segmentInfos.addAll(infos);
> >> checkpoint();
> >> }
> >> @@ -2420,7 +2451,18 @@ public class IndexWriter implements Clos
> >> merger.add(reader);
> >> }
> >>
> >> - MergeState mergeState = merger.merge(); // merge 'em
> >> + MergeState mergeState;
> >> + boolean success = false;
> >> + try {
> >> + mergeState = merger.merge(); // merge 'em
> >> + success = true;
> >> + } finally {
> >> + if (!success) {
> >> + synchronized(this) {
> >> + deleter.refresh(info.name);
> >> + }
> >> + }
> >> + }
> >>
> >> SegmentInfoPerCommit infoPerCommit = new
> >> SegmentInfoPerCommit(info, 0, -1L);
> >>
> >> @@ -2442,12 +2484,14 @@ public class IndexWriter implements Clos
> >> // Now create the compound file if needed
> >> if (useCompoundFile) {
> >> Collection<String> filesToDelete = infoPerCommit.files();
> >> - createCompoundFile(infoStream, directory,
> >> MergeState.CheckAbort.NONE, info, context);
> >> -
> >> - // delete new non cfs files directly: they were never
> >> - // registered with IFD
> >> - synchronized(this) {
> >> - deleter.deleteNewFiles(filesToDelete);
> >> + try{
> >> + createCompoundFile(infoStream, directory,
> >> MergeState.CheckAbort.NONE, info, context);
> >> + } finally {
> >> + // delete new non cfs files directly: they were never
> >> + // registered with IFD
> >> + synchronized(this) {
> >> + deleter.deleteNewFiles(filesToDelete);
> >> + }
> >> }
> >> info.setUseCompoundFile(true);
> >> }
> >> @@ -2456,7 +2500,18 @@ public class IndexWriter implements Clos
> >> // creating CFS so that 1) .si isn't slurped into CFS,
> >> // and 2) .si reflects useCompoundFile=true change
> >> // above:
> >> - codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir,
> >> info, mergeState.fieldInfos, context);
> >> + success = false;
> >> + try {
> >> +
> codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir,
> >> info, mergeState.fieldInfos, context);
> >> + success = true;
> >> + } finally {
> >> + if (!success) {
> >> + synchronized(this) {
> >> + deleter.refresh(info.name);
> >> + }
> >> + }
> >> + }
> >> +
> >> info.addFiles(trackingDir.getCreatedFiles());
> >>
> >> // Register the new segment
> >> @@ -2512,23 +2567,38 @@ public class IndexWriter implements Clos
> >> // We must rewrite the SI file because it references segment name in its
> list
> >> of files, etc
> >> TrackingDirectoryWrapper trackingDir = new
> >> TrackingDirectoryWrapper(directory);
> >>
> >> -
> >>
> newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackin
> >> gDir, newInfo, fis, context);
> >> + boolean success = false;
> >>
> >> - final Collection<String> siFiles = trackingDir.getCreatedFiles();
> >> + try {
> >>
> >> - // Copy the segment's files
> >> - for (String file: info.files()) {
> >> +
> >>
> newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackin
> >> gDir, newInfo, fis, context);
> >>
> >> - final String newFileName = segName +
> >> IndexFileNames.stripSegmentName(file);
> >> + final Collection<String> siFiles = trackingDir.getCreatedFiles();
> >>
> >> - if (siFiles.contains(newFileName)) {
> >> - // We already rewrote this above
> >> - continue;
> >> - }
> >> + // Copy the segment's files
> >> + for (String file: info.files()) {
> >> +
> >> + final String newFileName = segName +
> >> IndexFileNames.stripSegmentName(file);
> >>
> >> - assert !directory.fileExists(newFileName): "file \"" + newFileName + "\"
> >> already exists; siFiles=" + siFiles;
> >> + if (siFiles.contains(newFileName)) {
> >> + // We already rewrote this above
> >> + continue;
> >> + }
> >>
> >> - info.info.dir.copy(directory, file, newFileName, context);
> >> + assert !directory.fileExists(newFileName): "file \"" + newFileName +
> "\"
> >> already exists; siFiles=" + siFiles;
> >> +
> >> + info.info.dir.copy(directory, file, newFileName, context);
> >> + }
> >> + success = true;
> >> + } finally {
> >> + if (!success) {
> >> + for(String file : newInfo.files()) {
> >> + try {
> >> + directory.deleteFile(file);
> >> + } catch (Throwable t) {
> >> + }
> >> + }
> >> + }
> >> }
> >>
> >> return newInfoPerCommit;
> >> @@ -3091,6 +3161,7 @@ public class IndexWriter implements Clos
> >> if (infoStream.isEnabled("IW")) {
> >> infoStream.message("IW", "commitMerge: skip: it was aborted");
> >> }
> >> + deleter.deleteNewFiles(merge.info.files());
> >> return false;
> >> }
> >>
> >> @@ -3136,6 +3207,11 @@ public class IndexWriter implements Clos
> >> }
> >> }
> >>
> >> + if (dropSegment) {
> >> + assert !segmentInfos.contains(merge.info);
> >> + deleter.deleteNewFiles(merge.info.files());
> >> + }
> >> +
> >> // Must close before checkpoint, otherwise IFD won't be
> >> // able to delete the held-open files from the merge
> >> // readers:
> >> @@ -3407,7 +3483,7 @@ public class IndexWriter implements Clos
> >> setDiagnostics(si, "merge", details);
> >>
> >> if (infoStream.isEnabled("IW")) {
> >> - infoStream.message("IW", "merge seg=" + merge.info.info.name);
> >> + infoStream.message("IW", "merge seg=" + merge.info.info.name + " " +
> >> segString(merge.segments));
> >> }
> >>
> >> assert merge.estimatedMergeBytes == 0;
> >> @@ -4116,7 +4192,22 @@ public class IndexWriter implements Clos
> >> } catch(IOException ex) {
> >> prior = ex;
> >> } finally {
> >> - IOUtils.closeWhileHandlingException(prior, cfsDir);
> >> + boolean success = false;
> >> + try {
> >> + IOUtils.closeWhileHandlingException(prior, cfsDir);
> >> + success = true;
> >> + } finally {
> >> + if (!success) {
> >> + try {
> >> + directory.deleteFile(fileName);
> >> + } catch (Throwable t) {
> >> + }
> >> + try {
> >> + directory.deleteFile(IndexFileNames.segmentFileName(info.name,
> "",
> >> IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
> >> + } catch (Throwable t) {
> >> + }
> >> + }
> >> + }
> >> }
> >>
> >> // Replace all previous files with the CFS/CFE files:
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWri
> >> terConfig.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
> >>
> he/lucene/index/LiveIndexWriterConfig.java?rev=1393408&r1=1393407&r2=13
> >> 93408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWri
> >> terConfig.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWri
> >> terConfig.java Wed Oct 3 11:01:13 2012
> >> @@ -547,7 +547,7 @@ public class LiveIndexWriterConfig {
> >>
> >>
> sb.append("ramBufferSizeMB=").append(getRAMBufferSizeMB()).append("\n");
> >>
> >>
> sb.append("maxBufferedDocs=").append(getMaxBufferedDocs()).append("\n");
> >>
> >>
> sb.append("maxBufferedDeleteTerms=").append(getMaxBufferedDeleteTerms(
> >> )).append("\n");
> >> -
> >>
> sb.append("mergedSegmentWarmer=").append(getMergeScheduler()).append(
> >> "\n");
> >> +
> >>
> sb.append("mergedSegmentWarmer=").append(getMergedSegmentWarmer())
> >> .append("\n");
> >>
> >>
> sb.append("readerTermsIndexDivisor=").append(getReaderTermsIndexDivisor()
> >> ).append("\n");
> >>
> >>
> sb.append("termIndexInterval=").append(getTermIndexInterval()).append("\n");
> >> // TODO: this should be private to the codec, not settable here
> >>
> >>
> sb.append("delPolicy=").append(getIndexDeletionPolicy().getClass().getName())
> >> .append("\n");
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfo
> >> PerCommit.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
> >>
> he/lucene/index/SegmentInfoPerCommit.java?rev=1393408&r1=1393407&r2=
> >> 1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfo
> >> PerCommit.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfo
> >> PerCommit.java Wed Oct 3 11:01:13 2012
> >> @@ -79,6 +79,7 @@ public class SegmentInfoPerCommit {
> >>
> >> /** Returns all files in use by this segment. */
> >> public Collection<String> files() throws IOException {
> >> + // Start from the wrapped info's files:
> >> Collection<String> files = new HashSet<String>(info.files());
> >>
> >> // Must separately add any live docs files:
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.jav
> >> a
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apac
> >>
> he/lucene/store/Directory.java?rev=1393408&r1=1393407&r2=1393408&view
> >> =diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.jav
> >> a (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.jav
> >> a Wed Oct 3 11:01:13 2012
> >> @@ -201,7 +201,18 @@ public abstract class Directory implemen
> >> } catch (IOException ioe) {
> >> priorException = ioe;
> >> } finally {
> >> - IOUtils.closeWhileHandlingException(priorException, os, is);
> >> + boolean success = false;
> >> + try {
> >> + IOUtils.closeWhileHandlingException(priorException, os, is);
> >> + success = true;
> >> + } finally {
> >> + if (!success) {
> >> + try {
> >> + to.deleteFile(dest);
> >> + } catch (Throwable t) {
> >> + }
> >> + }
> >> + }
> >> }
> >> }
> >>
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/Tes
> >> tPerFieldPostingsFormat.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/codecs/perfield/TestPerFieldPostingsFormat.java?rev=1393408&r1=
> >> 1393407&r2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/Tes
> >> tPerFieldPostingsFormat.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/Tes
> >> tPerFieldPostingsFormat.java Wed Oct 3 11:01:13 2012
> >> @@ -33,7 +33,7 @@ public class TestPerFieldPostingsFormat
> >> @Override
> >> public void setUp() throws Exception {
> >> super.setUp();
> >> - codec = new RandomCodec(new Random(random().nextLong()),
> >> Collections.EMPTY_SET);
> >> + codec = new RandomCodec(new Random(random().nextLong()),
> >> Collections.<String>emptySet());
> >> }
> >>
> >> @Override
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.jav
> >> a
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestCrash.java?rev=1393408&r1=1393407&r2=1393408&view
> >> =diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.jav
> >> a (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.jav
> >> a Wed Oct 3 11:01:13 2012
> >> @@ -20,12 +20,13 @@ package org.apache.lucene.index;
> >> import java.io.IOException;
> >> import java.util.Random;
> >>
> >> +import org.apache.lucene.analysis.MockAnalyzer;
> >> +import org.apache.lucene.document.Document;
> >> import org.apache.lucene.document.Field;
> >> -import org.apache.lucene.util.LuceneTestCase;
> >> +import org.apache.lucene.store.Directory;
> >> import org.apache.lucene.store.MockDirectoryWrapper;
> >> import org.apache.lucene.store.NoLockFactory;
> >> -import org.apache.lucene.analysis.MockAnalyzer;
> >> -import org.apache.lucene.document.Document;
> >> +import org.apache.lucene.util.LuceneTestCase;
> >>
> >> public class TestCrash extends LuceneTestCase {
> >>
> >> @@ -67,20 +68,42 @@ public class TestCrash extends LuceneTes
> >> // before any documents were added.
> >> IndexWriter writer = initIndex(random(), true);
> >> MockDirectoryWrapper dir = (MockDirectoryWrapper)
> >> writer.getDirectory();
> >> +
> >> + // We create leftover files because merging could be
> >> + // running when we crash:
> >> + dir.setAssertNoUnrefencedFilesOnClose(false);
> >> +
> >> crash(writer);
> >> +
> >> IndexReader reader = DirectoryReader.open(dir);
> >> assertTrue(reader.numDocs() < 157);
> >> reader.close();
> >> +
> >> + // Make a new dir, copying from the crashed dir, and
> >> + // open IW on it, to confirm IW "recovers" after a
> >> + // crash:
> >> + Directory dir2 = newDirectory(dir);
> >> dir.close();
> >> +
> >> + new RandomIndexWriter(random(), dir2).close();
> >> + dir2.close();
> >> }
> >>
> >> public void testWriterAfterCrash() throws IOException {
> >> // This test relies on being able to open a reader before any commit
> >> // happened, so we must create an initial commit just to allow that, but
> >> // before any documents were added.
> >> + System.out.println("TEST: initIndex");
> >> IndexWriter writer = initIndex(random(), true);
> >> + System.out.println("TEST: done initIndex");
> >> MockDirectoryWrapper dir = (MockDirectoryWrapper)
> >> writer.getDirectory();
> >> +
> >> + // We create leftover files because merging could be
> >> + // running / store files could be open when we crash:
> >> + dir.setAssertNoUnrefencedFilesOnClose(false);
> >> +
> >> dir.setPreventDoubleWrite(false);
> >> + System.out.println("TEST: now crash");
> >> crash(writer);
> >> writer = initIndex(random(), dir, false);
> >> writer.close();
> >> @@ -88,12 +111,25 @@ public class TestCrash extends LuceneTes
> >> IndexReader reader = DirectoryReader.open(dir);
> >> assertTrue(reader.numDocs() < 314);
> >> reader.close();
> >> +
> >> + // Make a new dir, copying from the crashed dir, and
> >> + // open IW on it, to confirm IW "recovers" after a
> >> + // crash:
> >> + Directory dir2 = newDirectory(dir);
> >> dir.close();
> >> +
> >> + new RandomIndexWriter(random(), dir2).close();
> >> + dir2.close();
> >> }
> >>
> >> public void testCrashAfterReopen() throws IOException {
> >> IndexWriter writer = initIndex(random(), false);
> >> MockDirectoryWrapper dir = (MockDirectoryWrapper)
> >> writer.getDirectory();
> >> +
> >> + // We create leftover files because merging could be
> >> + // running when we crash:
> >> + dir.setAssertNoUnrefencedFilesOnClose(false);
> >> +
> >> writer.close();
> >> writer = initIndex(random(), dir, false);
> >> assertEquals(314, writer.maxDoc());
> >> @@ -111,7 +147,15 @@ public class TestCrash extends LuceneTes
> >> IndexReader reader = DirectoryReader.open(dir);
> >> assertTrue(reader.numDocs() >= 157);
> >> reader.close();
> >> +
> >> + // Make a new dir, copying from the crashed dir, and
> >> + // open IW on it, to confirm IW "recovers" after a
> >> + // crash:
> >> + Directory dir2 = newDirectory(dir);
> >> dir.close();
> >> +
> >> + new RandomIndexWriter(random(), dir2).close();
> >> + dir2.close();
> >> }
> >>
> >> public void testCrashAfterClose() throws IOException {
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
> >> Reader.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestDirectoryReader.java?rev=1393408&r1=1393407&r2=139
> >> 3408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
> >> Reader.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
> >> Reader.java Wed Oct 3 11:01:13 2012
> >> @@ -39,6 +39,7 @@ import org.apache.lucene.index.IndexWrit
> >> import org.apache.lucene.search.DocIdSetIterator;
> >> import org.apache.lucene.search.FieldCache;
> >> import org.apache.lucene.store.Directory;
> >> +import org.apache.lucene.store.MockDirectoryWrapper;
> >> import org.apache.lucene.store.NoSuchDirectoryException;
> >> import org.apache.lucene.util.Bits;
> >> import org.apache.lucene.util.BytesRef;
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
> >> ReaderReopen.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestDirectoryReaderReopen.java?rev=1393408&r1=1393407&
> >> r2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
> >> ReaderReopen.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectory
> >> ReaderReopen.java Wed Oct 3 11:01:13 2012
> >> @@ -38,6 +38,7 @@ import org.apache.lucene.search.IndexSea
> >> import org.apache.lucene.search.ScoreDoc;
> >> import org.apache.lucene.search.TermQuery;
> >> import org.apache.lucene.store.Directory;
> >> +import org.apache.lucene.store.MockDirectoryWrapper;
> >> import org.apache.lucene.util.LuceneTestCase;
> >> import org.apache.lucene.util._TestUtil;
> >>
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestDoc.java?rev=1393408&r1=1393407&r2=1393408&view=
> >> diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
> >> (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
> >> Wed Oct 3 11:01:13 2012
> >> @@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexWrit
> >> import org.apache.lucene.search.DocIdSetIterator;
> >> import org.apache.lucene.store.Directory;
> >> import org.apache.lucene.store.IOContext;
> >> +import org.apache.lucene.store.MockDirectoryWrapper;
> >> import org.apache.lucene.store.TrackingDirectoryWrapper;
> >> import org.apache.lucene.util.Constants;
> >> import org.apache.lucene.util.InfoStream;
> >> @@ -112,6 +113,13 @@ public class TestDoc extends LuceneTestC
> >> PrintWriter out = new PrintWriter(sw, true);
> >>
> >> Directory directory = newFSDirectory(indexDir, null);
> >> +
> >> + if (directory instanceof MockDirectoryWrapper) {
> >> + // We create unreferenced files (we don't even write
> >> + // a segments file):
> >> + ((MockDirectoryWrapper)
> >> directory).setAssertNoUnrefencedFilesOnClose(false);
> >> + }
> >> +
> >> IndexWriter writer = new IndexWriter(
> >> directory,
> >> newIndexWriterConfig(TEST_VERSION_CURRENT, new
> >> MockAnalyzer(random())).
> >> @@ -147,6 +155,13 @@ public class TestDoc extends LuceneTestC
> >> out = new PrintWriter(sw, true);
> >>
> >> directory = newFSDirectory(indexDir, null);
> >> +
> >> + if (directory instanceof MockDirectoryWrapper) {
> >> + // We create unreferenced files (we don't even write
> >> + // a segments file):
> >> + ((MockDirectoryWrapper)
> >> directory).setAssertNoUnrefencedFilesOnClose(false);
> >> + }
> >> +
> >> writer = new IndexWriter(
> >> directory,
> >> newIndexWriterConfig(TEST_VERSION_CURRENT, new
> >> MockAnalyzer(random())).
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
> >> erExceptions.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestIndexWriterExceptions.java?rev=1393408&r1=1393407&r
> >> 2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
> >> erExceptions.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
> >> erExceptions.java Wed Oct 3 11:01:13 2012
> >> @@ -1070,6 +1070,9 @@ public class TestIndexWriterExceptions e
> >> fail("segmentInfos failed to retry fallback to correct segments_N file");
> >> }
> >> reader.close();
> >> +
> >> + // should remove the corrumpted segments_N
> >> + new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
> >> null)).close();
> >> dir.close();
> >> }
> >>
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
> >> erWithThreads.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestIndexWriterWithThreads.java?rev=1393408&r1=1393407
> >> &r2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
> >> erWithThreads.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWrit
> >> erWithThreads.java Wed Oct 3 11:01:13 2012
> >> @@ -284,6 +284,9 @@ public class TestIndexWriterWithThreads
> >> failure.clearDoFail();
> >> writer.close(false);
> >> }
> >> + if (VERBOSE) {
> >> + System.out.println("TEST: success=" + success);
> >> + }
> >>
> >> if (success) {
> >> IndexReader reader = DirectoryReader.open(dir);
> >> @@ -341,6 +344,12 @@ public class TestIndexWriterWithThreads
> >> }
> >> @Override
> >> public void eval(MockDirectoryWrapper dir) throws IOException {
> >> +
> >> + // Since we throw exc during abort, eg when IW is
> >> + // attempting to delete files, we will leave
> >> + // leftovers:
> >> + dir.setAssertNoUnrefencedFilesOnClose(false);
> >> +
> >> if (doFail) {
> >> StackTraceElement[] trace = new Exception().getStackTrace();
> >> boolean sawAbortOrFlushDoc = false;
> >> @@ -405,6 +414,8 @@ public class TestIndexWriterWithThreads
> >> if ("flush".equals(trace[i].getMethodName()) &&
> >>
> "org.apache.lucene.index.DocFieldProcessor".equals(trace[i].getClassName())) {
> >> if (onlyOnce)
> >> doFail = false;
> >> + //System.out.println(Thread.currentThread().getName() + ": NOW
> FAIL:
> >> onlyOnce=" + onlyOnce);
> >> + //new Throwable().printStackTrace(System.out);
> >> throw new IOException("now failing on purpose");
> >> }
> >> }
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeleti
> >> onPolicy.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestNoDeletionPolicy.java?rev=1393408&r1=1393407&r2=139
> >> 3408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeleti
> >> onPolicy.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeleti
> >> onPolicy.java Wed Oct 3 11:01:13 2012
> >> @@ -21,10 +21,12 @@ import java.lang.reflect.Constructor;
> >> import java.lang.reflect.Method;
> >> import java.lang.reflect.Modifier;
> >> import java.util.Arrays;
> >> +
> >> import org.apache.lucene.analysis.MockAnalyzer;
> >> import org.apache.lucene.document.Document;
> >> import org.apache.lucene.document.Field;
> >> import org.apache.lucene.store.Directory;
> >> +import org.apache.lucene.store.MockDirectoryWrapper;
> >> import org.apache.lucene.util.LuceneTestCase;
> >> import org.junit.Test;
> >>
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistent
> >> SnapshotDeletionPolicy.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestPersistentSnapshotDeletionPolicy.java?rev=1393408&r1=1
> >> 393407&r2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistent
> >> SnapshotDeletionPolicy.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistent
> >> SnapshotDeletionPolicy.java Wed Oct 3 11:01:13 2012
> >> @@ -25,6 +25,7 @@ import org.apache.lucene.document.Docume
> >> import org.apache.lucene.index.IndexWriterConfig.OpenMode;
> >> import org.apache.lucene.store.Directory;
> >> import org.apache.lucene.store.LockObtainFailedException;
> >> +import org.apache.lucene.store.MockDirectoryWrapper;
> >> import org.junit.After;
> >> import org.junit.Before;
> >> import org.junit.Test;
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUp
> >> dates.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestRollingUpdates.java?rev=1393408&r1=1393407&r2=1393
> >> 408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUp
> >> dates.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUp
> >> dates.java Wed Oct 3 11:01:13 2012
> >> @@ -39,7 +39,6 @@ public class TestRollingUpdates extends
> >> public void testRollingUpdates() throws Exception {
> >> Random random = new Random(random().nextLong());
> >> final BaseDirectoryWrapper dir = newDirectory();
> >> - dir.setCheckIndexOnClose(false); // we use a custom codec provider
> >> final LineFileDocs docs = new LineFileDocs(random, true);
> >>
> >> //provider.register(new MemoryCodec());
> >> @@ -130,10 +129,11 @@ public class TestRollingUpdates extends
> >> assertEquals(SIZE, w.numDocs());
> >>
> >> w.close();
> >> +
> >> + TestIndexWriter.assertNoUnreferencedFiles(dir, "leftover files after
> rolling
> >> updates");
> >> +
> >> docs.close();
> >>
> >> - _TestUtil.checkIndex(dir);
> >> -
> >> // LUCENE-4455:
> >> SegmentInfos infos = new SegmentInfos();
> >> infos.read(dir);
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshot
> >> DeletionPolicy.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestSnapshotDeletionPolicy.java?rev=1393408&r1=1393407&r
> >> 2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshot
> >> DeletionPolicy.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshot
> >> DeletionPolicy.java Wed Oct 3 11:01:13 2012
> >> @@ -17,17 +17,18 @@ package org.apache.lucene.index;
> >> * limitations under the License.
> >> */
> >>
> >> +import java.io.IOException;
> >> import java.util.Collection;
> >> import java.util.Map;
> >> import java.util.Random;
> >> -import java.io.IOException;
> >>
> >> +import org.apache.lucene.analysis.MockAnalyzer;
> >> import org.apache.lucene.document.Document;
> >> import org.apache.lucene.document.FieldType;
> >> import org.apache.lucene.document.TextField;
> >> import org.apache.lucene.store.Directory;
> >> import org.apache.lucene.store.IndexInput;
> >> -import org.apache.lucene.analysis.MockAnalyzer;
> >> +import org.apache.lucene.store.MockDirectoryWrapper;
> >> import org.apache.lucene.util.LuceneTestCase;
> >> import org.apache.lucene.util.ThreadInterruptedException;
> >> import org.junit.Test;
> >> @@ -313,6 +314,7 @@ public class TestSnapshotDeletionPolicy
> >> public void testRollbackToOldSnapshot() throws Exception {
> >> int numSnapshots = 2;
> >> Directory dir = newDirectory();
> >> +
> >> SnapshotDeletionPolicy sdp = getDeletionPolicy();
> >> IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp));
> >> prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
> >> @@ -325,10 +327,11 @@ public class TestSnapshotDeletionPolicy
> >> writer.deleteUnusedFiles();
> >> assertSnapshotExists(dir, sdp, numSnapshots - 1);
> >> writer.close();
> >> -
> >> +
> >> // but 'snapshot1' files will still exist (need to release snapshot before
> they
> >> can be deleted).
> >> String segFileName =
> sdp.getSnapshot("snapshot1").getSegmentsFileName();
> >> assertTrue("snapshot files should exist in the directory: " + segFileName,
> >> dir.fileExists(segFileName));
> >> +
> >> dir.close();
> >> }
> >>
> >> @@ -385,6 +388,7 @@ public class TestSnapshotDeletionPolicy
> >> @Test
> >> public void testSnapshotLastCommitTwice() throws Exception {
> >> Directory dir = newDirectory();
> >> +
> >> SnapshotDeletionPolicy sdp = getDeletionPolicy();
> >> IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp));
> >> writer.addDocument(new Document());
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
> >> onRollback.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestTransactionRollback.java?rev=1393408&r1=1393407&r2=
> >> 1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
> >> onRollback.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
> >> onRollback.java Wed Oct 3 11:01:13 2012
> >> @@ -21,17 +21,18 @@ package org.apache.lucene.index;
> >> import java.io.IOException;
> >> import java.util.BitSet;
> >> import java.util.Collection;
> >> +import java.util.HashMap;
> >> import java.util.Iterator;
> >> import java.util.List;
> >> import java.util.Map;
> >> -import java.util.HashMap;
> >>
> >> -import org.apache.lucene.document.Field;
> >> -import org.apache.lucene.util.LuceneTestCase;
> >> import org.apache.lucene.analysis.MockAnalyzer;
> >> import org.apache.lucene.document.Document;
> >> +import org.apache.lucene.document.Field;
> >> import org.apache.lucene.store.Directory;
> >> +import org.apache.lucene.store.MockDirectoryWrapper;
> >> import org.apache.lucene.util.Bits;
> >> +import org.apache.lucene.util.LuceneTestCase;
> >>
> >> /**
> >> * Test class to illustrate using IndexDeletionPolicy to provide multi-level
> >> rollback capability.
> >> @@ -55,13 +56,16 @@ public class TestTransactionRollback ext
> >> for (Iterator<IndexCommit> iterator = commits.iterator();
> >> iterator.hasNext();) {
> >> IndexCommit commit = iterator.next();
> >> Map<String,String> ud=commit.getUserData();
> >> - if (ud.size() > 0)
> >> - if (ud.get("index").endsWith(ids))
> >> - last=commit;
> >> + if (ud.size() > 0) {
> >> + if (ud.get("index").endsWith(ids)) {
> >> + last = commit;
> >> + }
> >> + }
> >> }
> >>
> >> - if (last==null)
> >> + if (last==null) {
> >> throw new RuntimeException("Couldn't find commit point "+id);
> >> + }
> >>
> >> IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
> >> TEST_VERSION_CURRENT, new
> >> MockAnalyzer(random())).setIndexDeletionPolicy(
> >> @@ -74,13 +78,13 @@ public class TestTransactionRollback ext
> >>
> >> public void testRepeatedRollBacks() throws Exception {
> >>
> >> - int expectedLastRecordId=100;
> >> + int expectedLastRecordId = 100;
> >> while (expectedLastRecordId>10) {
> >> - expectedLastRecordId -=10;
> >> + expectedLastRecordId -= 10;
> >> rollBackLast(expectedLastRecordId);
> >>
> >> BitSet expecteds = new BitSet(100);
> >> - expecteds.set(1,(expectedLastRecordId+1),true);
> >> + expecteds.set(1, (expectedLastRecordId+1), true);
> >> checkExpecteds(expecteds);
> >> }
> >> }
> >> @@ -125,6 +129,7 @@ public class TestTransactionRollback ext
> >> public void setUp() throws Exception {
> >> super.setUp();
> >> dir = newDirectory();
> >> +
> >> //Build index, of records 1 to 100, committing after each batch of 10
> >> IndexDeletionPolicy sdp=new KeepAllDeletionPolicy();
> >> IndexWriter w=new IndexWriter(dir, newIndexWriterConfig(
> >> TEST_VERSION_CURRENT, new
> >> MockAnalyzer(random())).setIndexDeletionPolicy(sdp));
> >> @@ -199,6 +204,7 @@ public class TestTransactionRollback ext
> >> }
> >>
> >> public void testRollbackDeletionPolicy() throws Exception {
> >> +
> >> for(int i=0;i<2;i++) {
> >> // Unless you specify a prior commit point, rollback
> >> // should not work:
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
> >> ons.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/index/TestTransactions.java?rev=1393408&r1=1393407&r2=139340
> >> 8&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
> >> ons.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransacti
> >> ons.java Wed Oct 3 11:01:13 2012
> >> @@ -215,6 +215,11 @@ public class TestTransactions extends Lu
> >> dir1.setFailOnOpenInput(false);
> >> dir2.setFailOnOpenInput(false);
> >>
> >> + // We throw exceptions in deleteFile, which creates
> >> + // leftover files:
> >> + dir1.setAssertNoUnrefencedFilesOnClose(false);
> >> + dir2.setAssertNoUnrefencedFilesOnClose(false);
> >> +
> >> initIndex(dir1);
> >> initIndex(dir2);
> >>
> >>
> >> Added:
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/Te
> >> stFailIfUnreferencedFiles.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apac
> >>
> he/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java?rev=1393408&vie
> >> w=auto
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/Te
> >> stFailIfUnreferencedFiles.java (added)
> >> +++
> >>
> lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/Te
> >> stFailIfUnreferencedFiles.java Wed Oct 3 11:01:13 2012
> >> @@ -0,0 +1,58 @@
> >> +package org.apache.lucene.util.junitcompat;
> >> +
> >> +/*
> >> + * Licensed to the Apache Software Foundation (ASF) under one or more
> >> + * contributor license agreements. See the NOTICE file distributed with
> >> + * this work for additional information regarding copyright ownership.
> >> + * The ASF licenses this file to You under the Apache License, Version 2.0
> >> + * (the "License"); you may not use this file except in compliance with
> >> + * the License. You may obtain a copy of the License at
> >> + *
> >> + * http://www.apache.org/licenses/LICENSE-2.0
> >> + *
> >> + * Unless required by applicable law or agreed to in writing, software
> >> + * distributed under the License is distributed on an "AS IS" BASIS,
> >> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> >> implied.
> >> + * See the License for the specific language governing permissions and
> >> + * limitations under the License.
> >> + */
> >> +
> >> +import java.util.Collections;
> >> +
> >> +import org.apache.lucene.document.Document;
> >> +import org.apache.lucene.index.IndexWriter;
> >> +import org.apache.lucene.index.IndexWriterConfig;
> >> +import org.apache.lucene.store.Directory;
> >> +import org.apache.lucene.store.IOContext;
> >> +import org.apache.lucene.store.IndexOutput;
> >> +import org.junit.Assert;
> >> +import org.junit.Test;
> >> +import org.junit.runner.JUnitCore;
> >> +import org.junit.runner.Result;
> >> +
> >> +// LUCENE-4456: Test that we fail if there are unreferenced files
> >> +public class TestFailIfUnreferencedFiles extends WithNestedTests {
> >> + public TestFailIfUnreferencedFiles() {
> >> + super(true);
> >> + }
> >> +
> >> + public static class Nested1 extends WithNestedTests.AbstractNestedTest {
> >> + public void testDummy() throws Exception {
> >> + Directory dir = newMockDirectory();
> >> + IndexWriter iw = new IndexWriter(dir, new
> >> IndexWriterConfig(TEST_VERSION_CURRENT, null));
> >> + iw.addDocument(new Document());
> >> + iw.close();
> >> + IndexOutput output = dir.createOutput("_hello.world",
> >> IOContext.DEFAULT);
> >> + output.writeString("i am unreferenced!");
> >> + output.close();
> >> + dir.sync(Collections.singleton("_hello.world"));
> >> + dir.close();
> >> + }
> >> + }
> >> +
> >> + @Test
> >> + public void testFailIfUnreferencedFiles() {
> >> + Result r = JUnitCore.runClasses(Nested1.class);
> >> + Assert.assertEquals(1, r.getFailureCount());
> >> + }
> >> +}
> >>
> >> Modified:
> >>
> lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplit
> >> ter.java
> >> URL:
> >>
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apac
> >>
> he/lucene/index/TestIndexSplitter.java?rev=1393408&r1=1393407&r2=139340
> >> 8&view=diff
> >>
> ================================================================
> >> ==============
> >> ---
> >>
> lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplit
> >> ter.java (original)
> >> +++
> >>
> lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplit
> >> ter.java Wed Oct 3 11:01:13 2012
> >> @@ -22,6 +22,7 @@ import org.apache.lucene.analysis.MockAn
> >> import org.apache.lucene.document.Document;
> >> import org.apache.lucene.index.IndexWriterConfig.OpenMode;
> >> import org.apache.lucene.store.Directory;
> >> +import org.apache.lucene.store.MockDirectoryWrapper;
> >> import org.apache.lucene.util.LuceneTestCase;
> >> import org.apache.lucene.util._TestUtil;
> >>
> >> @@ -34,6 +35,11 @@ public class TestIndexSplitter extends L
> >> _TestUtil.rmDir(destDir);
> >> destDir.mkdirs();
> >> Directory fsDir = newFSDirectory(dir);
> >> + // IndexSplitter.split makes its own commit directly with
> SIPC/SegmentInfos,
> >> + // so the unreferenced files are expected.
> >> + if (fsDir instanceof MockDirectoryWrapper) {
> >> +
> >> ((MockDirectoryWrapper)fsDir).setAssertNoUnrefencedFilesOnClose(false);
> >> + }
> >>
> >> LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
> >> mergePolicy.setNoCFSRatio(1.0);
> >>
> >> Modified: lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
> >> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-
> >>
> framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1
> >> 393408&r1=1393407&r2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> --- lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
> >> (original)
> >> +++ lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
> Wed
> >> Oct 3 11:01:13 2012
> >> @@ -402,10 +402,16 @@ public class RandomIndexWriter implement
> >> final int segCount = w.getSegmentCount();
> >> if (r.nextBoolean() || segCount == 0) {
> >> // full forceMerge
> >> + if (LuceneTestCase.VERBOSE) {
> >> + System.out.println("RIW: doRandomForceMerge(1)");
> >> + }
> >> w.forceMerge(1);
> >> } else {
> >> // partial forceMerge
> >> final int limit = _TestUtil.nextInt(r, 1, segCount);
> >> + if (LuceneTestCase.VERBOSE) {
> >> + System.out.println("RIW: doRandomForceMerge(" + limit + ")");
> >> + }
> >> w.forceMerge(limit);
> >> assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit:
> >> "limit=" + limit + " actual=" + w.getSegmentCount();
> >> }
> >>
> >> Modified: lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
> >> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-
> >>
> framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev
> >> =1393408&r1=1393407&r2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> --- lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
> >> (original)
> >> +++ lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
> >> Wed Oct 3 11:01:13 2012
> >> @@ -28,14 +28,18 @@ import java.util.HashMap;
> >> import java.util.HashSet;
> >> import java.util.IdentityHashMap;
> >> import java.util.Iterator;
> >> +import java.util.List;
> >> import java.util.Map;
> >> import java.util.Random;
> >> import java.util.Set;
> >> +import java.util.TreeSet;
> >> import java.util.concurrent.atomic.AtomicInteger;
> >>
> >> import org.apache.lucene.index.DirectoryReader;
> >> import org.apache.lucene.index.IndexWriter;
> >> import org.apache.lucene.index.IndexWriterConfig;
> >> +import org.apache.lucene.index.NoDeletionPolicy;
> >> +import org.apache.lucene.index.SegmentInfos;
> >> import org.apache.lucene.util.LuceneTestCase;
> >> import org.apache.lucene.util._TestUtil;
> >>
> >> @@ -204,15 +208,17 @@ public class MockDirectoryWrapper extend
> >> // first force-close all files, so we can corrupt on windows etc.
> >> // clone the file map, as these guys want to remove themselves on close.
> >> Map<Closeable,Exception> m = new
> >> IdentityHashMap<Closeable,Exception>(openFileHandles);
> >> - for (Closeable f : m.keySet())
> >> + for (Closeable f : m.keySet()) {
> >> try {
> >> f.close();
> >> } catch (Exception ignored) {}
> >> + }
> >>
> >> while(it.hasNext()) {
> >> String name = it.next();
> >> int damage = randomState.nextInt(5);
> >> String action = null;
> >> +
> >> if (damage == 0) {
> >> action = "deleted";
> >> deleteFile(name, true);
> >> @@ -544,6 +550,9 @@ public class MockDirectoryWrapper extend
> >>
> >> @Override
> >> public synchronized void close() throws IOException {
> >> + // files that we tried to delete, but couldn't because readers were open.
> >> + // all that matters is that we tried! (they will eventually go away)
> >> + Set<String> pendingDeletions = new HashSet<String>(openFilesDeleted);
> >> maybeYield();
> >> if (openFiles == null) {
> >> openFiles = new HashMap<String,Integer>();
> >> @@ -574,17 +583,91 @@ public class MockDirectoryWrapper extend
> >> }
> >> _TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose());
> >>
> >> + // TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
> >> if (assertNoUnreferencedFilesOnClose) {
> >> - // now look for unreferenced files:
> >> - String[] startFiles = listAll();
> >> - new IndexWriter(this, new
> >> IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT,
> null)).rollback();
> >> - String[] endFiles = listAll();
> >> + // now look for unreferenced files: discount ones that we tried to
> delete
> >> but could not
> >> + Set<String> allFiles = new HashSet<String>(Arrays.asList(listAll()));
> >> + allFiles.removeAll(pendingDeletions);
> >> + String[] startFiles = allFiles.toArray(new String[0]);
> >> + IndexWriterConfig iwc = new
> >> IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null);
> >> + iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
> >> + new IndexWriter(delegate, iwc).rollback();
> >> + String[] endFiles = delegate.listAll();
> >> +
> >> + Set<String> startSet = new TreeSet<String>(Arrays.asList(startFiles));
> >> + Set<String> endSet = new TreeSet<String>(Arrays.asList(endFiles));
> >> +
> >> + if (pendingDeletions.contains("segments.gen") &&
> >> endSet.contains("segments.gen")) {
> >> + // this is possible if we hit an exception while writing segments.gen,
> we
> >> try to delete it
> >> + // and it ends out in pendingDeletions (but IFD wont remove this).
> >> + startSet.add("segments.gen");
> >> + if (LuceneTestCase.VERBOSE) {
> >> + System.out.println("MDW: Unreferenced check: Ignoring
> >> segments.gen that we could not delete.");
> >> + }
> >> + }
> >> +
> >> + // its possible we cannot delete the segments_N on windows if
> someone
> >> has it open and
> >> + // maybe other files too, depending on timing. normally someone on
> >> windows wouldnt have
> >> + // an issue (IFD would nuke this stuff eventually), but we pass
> >> NoDeletionPolicy...
> >> + for (String file : pendingDeletions) {
> >> + if (file.startsWith("segments") && !file.equals("segments.gen") &&
> >> endSet.contains(file)) {
> >> + startSet.add(file);
> >> + if (LuceneTestCase.VERBOSE) {
> >> + System.out.println("MDW: Unreferenced check: Ignoring
> segments
> >> file: " + file + " that we could not delete.");
> >> + }
> >> + try {
> >> + SegmentInfos sis = new SegmentInfos();
> >> + sis.read(delegate, file);
> >> + Set<String> ghosts = new HashSet<String>(sis.files(delegate,
> false));
> >> + for (String s : ghosts) {
> >> + if (endSet.contains(s) && !startSet.contains(s)) {
> >> + assert pendingDeletions.contains(s);
> >> + if (LuceneTestCase.VERBOSE) {
> >> + System.out.println("MDW: Unreferenced check: Ignoring
> >> referenced file: " + s + " " +
> >> + "from " + file + " that we could not delete.");
> >> + }
> >> + startSet.add(s);
> >> + }
> >> + }
> >> + } catch (Throwable ignore) {}
> >> + }
> >> + }
> >>
> >> - Arrays.sort(startFiles);
> >> - Arrays.sort(endFiles);
> >> + startFiles = startSet.toArray(new String[0]);
> >> + endFiles = endSet.toArray(new String[0]);
> >>
> >> if (!Arrays.equals(startFiles, endFiles)) {
> >> - assert false : "unreferenced files: before delete:\n " +
> >> Arrays.toString(startFiles) + "\n after delete:\n " +
> Arrays.toString(endFiles);
> >> + StringBuilder sb = new StringBuilder();
> >> + List<String> removed = new ArrayList<String>();
> >> + for(String fileName : startFiles) {
> >> + if (!endSet.contains(fileName)) {
> >> + removed.add(fileName);
> >> + }
> >> + }
> >> +
> >> + List<String> added = new ArrayList<String>();
> >> + for(String fileName : endFiles) {
> >> + if (!startSet.contains(fileName)) {
> >> + added.add(fileName);
> >> + }
> >> + }
> >> +
> >> + String extras;
> >> + if (removed.size() != 0) {
> >> + extras = "\n\nThese files were removed: " + removed;
> >> + } else {
> >> + extras = "";
> >> + }
> >> +
> >> + if (added.size() != 0) {
> >> + extras += "\n\nThese files were added (waaaaaaaaaat!): " +
> added;
> >> + }
> >> +
> >> + if (pendingDeletions.size() != 0) {
> >> + extras += "\n\nThese files we had previously tried to delete, but
> >> couldn't: " + pendingDeletions;
> >> + }
> >> +
> >> + assert false : "unreferenced files: before delete:\n " +
> >> Arrays.toString(startFiles) + "\n after delete:\n " +
> Arrays.toString(endFiles) +
> >> extras;
> >> }
> >>
> >> DirectoryReader ir1 = DirectoryReader.open(this);
> >> @@ -607,7 +690,6 @@ public class MockDirectoryWrapper extend
> >> if (v != null) {
> >> if (v.intValue() == 1) {
> >> openFiles.remove(name);
> >> - openFilesDeleted.remove(name);
> >> } else {
> >> v = Integer.valueOf(v.intValue()-1);
> >> openFiles.put(name, v);
> >>
> >> Modified: lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/util/LuceneTestCase.java
> >> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-
> >>
> framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=139340
> >> 8&r1=1393407&r2=1393408&view=diff
> >>
> ================================================================
> >> ==============
> >> --- lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
> >> +++ lucene/dev/trunk/lucene/test-
> >> framework/src/java/org/apache/lucene/util/LuceneTestCase.java Wed Oct
> 3
> >> 11:01:13 2012
> >> @@ -948,6 +948,7 @@ public abstract class LuceneTestCase ext
> >> if (rarely(random)) {
> >> directory = new NRTCachingDirectory(directory, random.nextDouble(),
> >> random.nextDouble());
> >> }
> >> +
> >> if (bare) {
> >> BaseDirectoryWrapper base = new BaseDirectoryWrapper(directory);
> >> closeAfterSuite(new CloseableDirectory(base, suiteFailureMarker));
> >
> >
> >
> > ---------------------------------------------------------------------
> > To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> > For additional commands, e-mail: dev-help@lucene.apache.org
> >
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org