You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/05/20 14:25:43 UTC
svn commit: r1125348 - in
/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index: ./
codecs/ values/
Author: simonw
Date: Fri May 20 12:25:42 2011
New Revision: 1125348
URL: http://svn.apache.org/viewvc?rev=1125348&view=rev
Log:
LUCENE-3108: Closing previously opened files if not all DocValues can be loaded
Modified:
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1125348&r1=1125347&r2=1125348&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Fri May 20 12:25:42 2011
@@ -307,8 +307,16 @@ final class DocFieldProcessor extends Do
perDocConsumer = codec.docsConsumer(perDocWriteState);
perDocConsumers.put(Integer.valueOf(fieldInfo.getCodecId()), perDocConsumer);
}
- docValuesConsumer = perDocConsumer.addValuesField(fieldInfo);
- fieldInfo.commitDocValues();
+ boolean success = false;
+ try {
+ docValuesConsumer = perDocConsumer.addValuesField(fieldInfo);
+ fieldInfo.commitDocValues();
+ success = true;
+ } finally {
+ if (!success) {
+ fieldInfo.revertUncommitted();
+ }
+ }
docValues.put(fieldInfo.name, docValuesConsumer);
return docValuesConsumer;
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java?rev=1125348&r1=1125347&r2=1125348&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java Fri May 20 12:25:42 2011
@@ -294,13 +294,12 @@ final class PerFieldCodecWrapper extends
}
public void close() throws IOException {
- final Iterator<PerDocValues> it = codecs.values().iterator();
+ final Collection<PerDocValues> values = codecs.values();
IOException err = null;
- while (it.hasNext()) {
+ for (PerDocValues perDocValues : values) {
try {
- PerDocValues next = it.next();
- if (next != null) {
- next.close();
+ if (perDocValues != null) {
+ perDocValues.close();
}
} catch (IOException ioe) {
// keep first IOException we hit but keep
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java?rev=1125348&r1=1125347&r2=1125348&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java Fri May 20 12:25:42 2011
@@ -61,37 +61,33 @@ public class DefaultDocValuesConsumer ex
public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
Set<String> files) throws IOException {
FieldInfos fieldInfos = segmentInfo.getFieldInfos();
- boolean indexed = false;
for (FieldInfo fieldInfo : fieldInfos) {
- if (fieldInfo.getCodecId() == codecId) {
- indexed |= fieldInfo.isIndexed;
- if (fieldInfo.hasDocValues()) {
- String filename = docValuesId(segmentInfo.name, codecId, fieldInfo.number);
- switch (fieldInfo.getDocValues()) {
- case BYTES_FIXED_DEREF:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_STRAIGHT:
- files.add(IndexFileNames.segmentFileName(filename, "",
- Writer.INDEX_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
- Writer.INDEX_EXTENSION));
- // until here all types use an index
- case BYTES_FIXED_STRAIGHT:
- case FLOAT_32:
- case FLOAT_64:
- case INTS:
- files.add(IndexFileNames.segmentFileName(filename, "",
- Writer.DATA_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
- Writer.DATA_EXTENSION));
- break;
- default:
- assert false;
- }
+ if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) {
+ String filename = docValuesId(segmentInfo.name, codecId,
+ fieldInfo.number);
+ switch (fieldInfo.getDocValues()) {
+ case BYTES_FIXED_DEREF:
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ case BYTES_FIXED_SORTED:
+ case BYTES_VAR_STRAIGHT:
+ files.add(IndexFileNames.segmentFileName(filename, "",
+ Writer.INDEX_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
+ Writer.INDEX_EXTENSION));
+ // until here all types use an index
+ case BYTES_FIXED_STRAIGHT:
+ case FLOAT_32:
+ case FLOAT_64:
+ case INTS:
+ files.add(IndexFileNames.segmentFileName(filename, "",
+ Writer.DATA_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
+ Writer.DATA_EXTENSION));
+ break;
+ default:
+ assert false;
}
-
}
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java?rev=1125348&r1=1125347&r2=1125348&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java Fri May 20 12:25:42 2011
@@ -39,7 +39,7 @@ import org.apache.lucene.store.Directory
*/
public class DefaultDocValuesProducer extends PerDocValues {
- protected final TreeMap<String, DocValues> docValues = new TreeMap<String, DocValues>();
+ protected final TreeMap<String, DocValues> docValues;
/**
* Creates a new {@link DefaultDocValuesProducer} instance and loads all
@@ -58,7 +58,7 @@ public class DefaultDocValuesProducer ex
*/
public DefaultDocValuesProducer(SegmentInfo si, Directory dir,
FieldInfos fieldInfo, int codecId) throws IOException {
- load(fieldInfo, si.name, si.docCount, dir, codecId);
+ docValues = load(fieldInfo, si.name, si.docCount, dir, codecId);
}
/**
@@ -66,22 +66,37 @@ public class DefaultDocValuesProducer ex
* <code>null</code> if this field has no {@link DocValues}.
*/
@Override
- public DocValues docValues(String field) throws IOException {
+ public synchronized DocValues docValues(String field) throws IOException {
return docValues.get(field);
}
// Only opens files... doesn't actually load any values
- protected void load(FieldInfos fieldInfos, String segment, int docCount,
- Directory dir, int codecId) throws IOException {
- for (FieldInfo fieldInfo : fieldInfos) {
- if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) {
- final String field = fieldInfo.name;
- // TODO can we have a compound file per segment and codec for docvalues?
- final String id = DefaultDocValuesConsumer.docValuesId(segment, codecId, fieldInfo.number);
- docValues.put(field, loadDocValues(docCount, dir, id, fieldInfo
- .getDocValues()));
+ protected TreeMap<String, DocValues> load(FieldInfos fieldInfos,
+ String segment, int docCount, Directory dir, int codecId)
+ throws IOException {
+ TreeMap<String, DocValues> values = new TreeMap<String, DocValues>();
+ boolean success = false;
+ try {
+
+ for (FieldInfo fieldInfo : fieldInfos) {
+ if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) {
+ final String field = fieldInfo.name;
+ // TODO can we have a compound file per segment and codec for
+ // docvalues?
+ final String id = DefaultDocValuesConsumer.docValuesId(segment,
+ codecId, fieldInfo.number);
+ values.put(field,
+ loadDocValues(docCount, dir, id, fieldInfo.getDocValues()));
+ }
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ // if we fail we must close all opened resources if there are any
+ closeDocValues(values.values());
}
}
+ return values;
}
@@ -130,8 +145,12 @@ public class DefaultDocValuesProducer ex
}
}
- public void close() throws IOException {
- final Collection<DocValues> values = docValues.values();
+ public synchronized void close() throws IOException {
+ closeDocValues(docValues.values());
+ }
+
+ private void closeDocValues(final Collection<DocValues> values)
+ throws IOException {
IOException ex = null;
for (DocValues docValues : values) {
try {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1125348&r1=1125347&r2=1125348&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java Fri May 20 12:25:42 2011
@@ -434,6 +434,8 @@ public final class Bytes {
this.id = id;
datIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
Writer.DATA_EXTENSION));
+ boolean success = false;
+ try {
version = CodecUtil.checkHeader(datIn, codecName, maxVersion, maxVersion);
if (doIndex) {
idxIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
@@ -444,6 +446,12 @@ public final class Bytes {
} else {
idxIn = null;
}
+ success = true;
+ } finally {
+ if (!success) {
+ closeInternal();
+ }
+ }
}
/**
@@ -467,12 +475,16 @@ public final class Bytes {
try {
super.close();
} finally {
- try {
- datIn.close();
- } finally {
- if (idxIn != null) {
- idxIn.close();
- }
+ closeInternal();
+ }
+ }
+
+ private void closeInternal() throws IOException {
+ try {
+ datIn.close();
+ } finally {
+ if (idxIn != null) {
+ idxIn.close();
}
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java?rev=1125348&r1=1125347&r2=1125348&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java Fri May 20 12:25:42 2011
@@ -71,7 +71,6 @@ public class Floats {
}
abstract static class FloatsWriter extends Writer {
- private final Directory dir;
private final String id;
private FloatsRef floatsRef;
protected int lastDocId = -1;
@@ -81,20 +80,16 @@ public class Floats {
protected FloatsWriter(Directory dir, String id, int precision,
AtomicLong bytesUsed) throws IOException {
super(bytesUsed);
- this.dir = dir;
this.id = id;
this.precision = (byte) precision;
- initDatOut();
- }
-
- private void initDatOut() throws IOException {
datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
Writer.DATA_EXTENSION));
CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT);
assert datOut.getFilePointer() == CodecUtil.headerLength(CODEC_NAME);
- datOut.writeByte(precision);
+ datOut.writeByte(this.precision);
}
+
public long ramBytesUsed() {
return 0;
}
@@ -125,8 +120,6 @@ public class Floats {
assert reader.precisionBytes == (int) precision;
if (reader.maxDoc == 0)
return;
- if (datOut == null)
- initDatOut();
final int docBase = state.docBase;
if (docBase - lastDocId > 1) {
// fill with default values
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java?rev=1125348&r1=1125347&r2=1125348&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java Fri May 20 12:25:42 2011
@@ -60,13 +60,13 @@ class PackedIntsImpl {
protected IntsWriter(Directory dir, String id, AtomicLong bytesUsed)
throws IOException {
super(bytesUsed);
+ datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
+ DATA_EXTENSION));
+ CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT);
this.id = id;
docToValue = new long[1];
bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG); // TODO the bitset
// needs memory too
- datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
- DATA_EXTENSION));
- CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT);
}
@Override