You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2012/12/12 19:41:01 UTC
svn commit: r1420907 - in /lucene/dev/trunk/lucene:
core/src/java/org/apache/lucene/codecs/compressing/
core/src/test/org/apache/lucene/codecs/compressing/
test-framework/src/java/org/apache/lucene/codecs/compressing/
Author: jpountz
Date: Wed Dec 12 18:40:58 2012
New Revision: 1420907
URL: http://svn.apache.org/viewvc?rev=1420907&view=rev
Log:
LUCENE-4613: CompressingStoredFieldsFormat: fix .abort() when the segment suffix is not empty.
Modified:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/DummyCompressingCodec.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java?rev=1420907&r1=1420906&r2=1420907&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java Wed Dec 12 18:40:58 2012
@@ -48,16 +48,30 @@ import org.apache.lucene.store.IOContext
public class CompressingStoredFieldsFormat extends StoredFieldsFormat {
private final String formatName;
+ private final String segmentSuffix;
private final CompressionMode compressionMode;
private final int chunkSize;
/**
+ * Create a new {@link CompressingStoredFieldsFormat} with an empty segment
+ * suffix.
+ *
+ * @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(String, String, CompressionMode, int)
+ */
+ public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode, int chunkSize) {
+ this(formatName, "", compressionMode, chunkSize);
+ }
+
+ /**
* Create a new {@link CompressingStoredFieldsFormat}.
* <p>
* <code>formatName</code> is the name of the format. This name will be used
* in the file formats to perform
* {@link CodecUtil#checkHeader(org.apache.lucene.store.DataInput, String, int, int) codec header checks}.
* <p>
+ * <code>segmentSuffix</code> is the segment suffix. This suffix is added to
+ * the result file name only if it's not the empty string.
+ * <p>
* The <code>compressionMode</code> parameter allows you to choose between
* compression algorithms that have various compression and decompression
* speeds so that you can pick the one that best fits your indexing and
@@ -81,25 +95,29 @@ public class CompressingStoredFieldsForm
* @param chunkSize the minimum number of bytes of a single chunk of stored documents
* @see CompressionMode
*/
- public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode, int chunkSize) {
+ public CompressingStoredFieldsFormat(String formatName, String segmentSuffix,
+ CompressionMode compressionMode, int chunkSize) {
this.formatName = formatName;
+ this.segmentSuffix = segmentSuffix;
this.compressionMode = compressionMode;
if (chunkSize < 1) {
throw new IllegalArgumentException("chunkSize must be >= 1");
}
this.chunkSize = chunkSize;
+
}
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si,
FieldInfos fn, IOContext context) throws IOException {
- return new CompressingStoredFieldsReader(directory, si, "", fn, context, formatName, compressionMode);
+ return new CompressingStoredFieldsReader(directory, si, segmentSuffix, fn,
+ context, formatName, compressionMode);
}
@Override
public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si,
IOContext context) throws IOException {
- return new CompressingStoredFieldsWriter(directory, si, "", context,
+ return new CompressingStoredFieldsWriter(directory, si, segmentSuffix, context,
formatName, compressionMode, chunkSize);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1420907&r1=1420906&r2=1420907&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Wed Dec 12 18:40:58 2012
@@ -70,6 +70,7 @@ public final class CompressingStoredFiel
private final Directory directory;
private final String segment;
+ private final String segmentSuffix;
private CompressingStoredFieldsIndexWriter indexWriter;
private IndexOutput fieldsStream;
@@ -89,6 +90,7 @@ public final class CompressingStoredFiel
assert directory != null;
this.directory = directory;
this.segment = si.name;
+ this.segmentSuffix = segmentSuffix;
this.compressionMode = compressionMode;
this.compressor = compressionMode.newCompressor();
this.chunkSize = chunkSize;
@@ -287,8 +289,8 @@ public final class CompressingStoredFiel
public void abort() {
IOUtils.closeWhileHandlingException(this);
IOUtils.deleteFilesIgnoringExceptions(directory,
- IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION),
- IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION));
+ IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION),
+ IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION));
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java?rev=1420907&r1=1420906&r2=1420907&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java Wed Dec 12 18:40:58 2012
@@ -54,6 +54,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
+import org.junit.Test;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@@ -362,5 +363,41 @@ public class TestCompressingStoredFields
}
rd.close();
}
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testDeletePartiallyWrittenFilesIfAbort() throws IOException {
+ final Document validDoc = new Document();
+ validDoc.add(new IntField("id", 0, Store.YES));
+ iw.addDocument(validDoc);
+ iw.commit();
+
+ // make sure that #writeField will fail to trigger an abort
+ final Document invalidDoc = new Document();
+ FieldType fieldType = new FieldType();
+ fieldType.setStored(true);
+ invalidDoc.add(new Field("invalid", fieldType) {
+
+ @Override
+ public String stringValue() {
+ return null;
+ }
+
+ });
+
+ try {
+ iw.addDocument(invalidDoc);
+ iw.commit();
+ }
+ finally {
+ int counter = 0;
+ for (String fileName : dir.listAll()) {
+ if (fileName.endsWith(".fdt") || fileName.endsWith(".fdx")) {
+ counter++;
+ }
+ }
+ // Only one .fdt and one .fdx files must have been found
+ assertEquals(2, counter);
+ }
+ }
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java?rev=1420907&r1=1420906&r2=1420907&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java Wed Dec 12 18:40:58 2012
@@ -34,30 +34,51 @@ public abstract class CompressingCodec e
/**
* Create a random instance.
*/
- public static CompressingCodec randomInstance(Random random, int chunkSize) {
+ public static CompressingCodec randomInstance(Random random, int chunkSize, boolean withSegmentSuffix) {
switch (random.nextInt(4)) {
case 0:
- return new FastCompressingCodec(chunkSize);
+ return new FastCompressingCodec(chunkSize, withSegmentSuffix);
case 1:
- return new FastDecompressionCompressingCodec(chunkSize);
+ return new FastDecompressionCompressingCodec(chunkSize, withSegmentSuffix);
case 2:
- return new HighCompressionCompressingCodec(chunkSize);
+ return new HighCompressionCompressingCodec(chunkSize, withSegmentSuffix);
case 3:
- return new DummyCompressingCodec(chunkSize);
+ return new DummyCompressingCodec(chunkSize, withSegmentSuffix);
default:
throw new AssertionError();
}
}
+ /**
+ * Creates a random {@link CompressingCodec} that is using an empty segment
+ * suffix
+ */
public static CompressingCodec randomInstance(Random random) {
- return randomInstance(random, RandomInts.randomIntBetween(random, 1, 500));
+ return randomInstance(random, RandomInts.randomIntBetween(random, 1, 500), false);
+ }
+
+ /**
+ * Creates a random {@link CompressingCodec} that is using a segment suffix
+ */
+ public static CompressingCodec randomInstance(Random random, boolean withSegmentSuffix) {
+ return randomInstance(random, RandomInts.randomIntBetween(random, 1, 500), withSegmentSuffix);
}
private final CompressingStoredFieldsFormat storedFieldsFormat;
- public CompressingCodec(String name, CompressionMode compressionMode, int chunkSize) {
+ /**
+ * Creates a compressing codec with a given segment suffix
+ */
+ public CompressingCodec(String name, String segmentSuffix, CompressionMode compressionMode, int chunkSize) {
super(name, new Lucene41Codec());
- this.storedFieldsFormat = new CompressingStoredFieldsFormat(name, compressionMode, chunkSize);
+ this.storedFieldsFormat = new CompressingStoredFieldsFormat(name, segmentSuffix, compressionMode, chunkSize);
+ }
+
+ /**
+ * Creates a compressing codec with an empty segment suffix
+ */
+ public CompressingCodec(String name, CompressionMode compressionMode, int chunkSize) {
+ this(name, "", compressionMode, chunkSize);
}
@Override
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/DummyCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/DummyCompressingCodec.java?rev=1420907&r1=1420906&r2=1420907&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/DummyCompressingCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/DummyCompressingCodec.java Wed Dec 12 18:40:58 2012
@@ -82,13 +82,15 @@ public class DummyCompressingCodec exten
};
/** Constructor that allows to configure the chunk size. */
- public DummyCompressingCodec(int chunkSize) {
- super("DummyCompressingStoredFields", DUMMY, chunkSize);
+ public DummyCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+ super("DummyCompressingStoredFields",
+ withSegmentSuffix ? "DummyCompressingStoredFields" : "",
+ DUMMY, chunkSize);
}
/** Default constructor. */
public DummyCompressingCodec() {
- this(1 << 14);
+ this(1 << 14, false);
}
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java?rev=1420907&r1=1420906&r2=1420907&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java Wed Dec 12 18:40:58 2012
@@ -21,13 +21,15 @@ package org.apache.lucene.codecs.compres
public class FastCompressingCodec extends CompressingCodec {
/** Constructor that allows to configure the chunk size. */
- public FastCompressingCodec(int chunkSize) {
- super("FastCompressingStoredFields", CompressionMode.FAST, chunkSize);
+ public FastCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+ super("FastCompressingStoredFields",
+ withSegmentSuffix ? "FastCompressingStoredFields" : "",
+ CompressionMode.FAST, chunkSize);
}
/** Default constructor. */
public FastCompressingCodec() {
- this(1 << 14);
+ this(1 << 14, false);
}
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java?rev=1420907&r1=1420906&r2=1420907&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java Wed Dec 12 18:40:58 2012
@@ -21,13 +21,15 @@ package org.apache.lucene.codecs.compres
public class FastDecompressionCompressingCodec extends CompressingCodec {
/** Constructor that allows to configure the chunk size. */
- public FastDecompressionCompressingCodec(int chunkSize) {
- super("FastDecompressionCompressingStoredFields", CompressionMode.FAST_DECOMPRESSION, chunkSize);
+ public FastDecompressionCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+ super("FastDecompressionCompressingStoredFields",
+ withSegmentSuffix ? "FastDecompressionCompressingStoredFields" : "",
+ CompressionMode.FAST_DECOMPRESSION, chunkSize);
}
/** Default constructor. */
public FastDecompressionCompressingCodec() {
- this(1 << 14);
+ this(1 << 14, false);
}
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java?rev=1420907&r1=1420906&r2=1420907&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java Wed Dec 12 18:40:58 2012
@@ -21,13 +21,15 @@ package org.apache.lucene.codecs.compres
public class HighCompressionCompressingCodec extends CompressingCodec {
/** Constructor that allows to configure the chunk size. */
- public HighCompressionCompressingCodec(int chunkSize) {
- super("HighCompressionCompressingStoredFields", CompressionMode.HIGH_COMPRESSION, chunkSize);
+ public HighCompressionCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+ super("HighCompressionCompressingStoredFields",
+ withSegmentSuffix ? "HighCompressionCompressingStoredFields" : "",
+ CompressionMode.HIGH_COMPRESSION, chunkSize);
}
/** Default constructor. */
public HighCompressionCompressingCodec() {
- this(1 << 14);
+ this(1 << 14, false);
}
}