You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2017/01/25 16:14:42 UTC
[2/2] lucene-solr:branch_6_4: LUCENE-7647:
CompressingStoredFieldsFormat should reclaim memory more aggressively.
LUCENE-7647: CompressingStoredFieldsFormat should reclaim memory more aggressively.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ff9cc1d8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ff9cc1d8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ff9cc1d8
Branch: refs/heads/branch_6_4
Commit: ff9cc1d8090c8e8cbc7ec22b50c156fafad8e6f3
Parents: bfb8a61
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed Jan 25 16:15:04 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed Jan 25 17:08:46 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 ++
.../CompressingStoredFieldsWriter.java | 5 +-
.../codecs/compressing/CompressionMode.java | 49 ++++++++++++++------
.../lucene/codecs/compressing/Compressor.java | 3 +-
.../dummy/DummyCompressingCodec.java | 3 ++
5 files changed, 46 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ff9cc1d8/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 37328aa..4b87106 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -17,6 +17,10 @@ Bug Fixes
* LUCENE-7657: Fixed potential memory leak in the case that a (Span)TermQuery
with a TermContext is cached. (Adrien Grand)
+* LUCENE-7647: Made stored fields reclaim native memory more aggressively when
+ configured with BEST_COMPRESSION. This could otherwise result in out-of-memory
+ issues. (Adrien Grand)
+
======================= Lucene 6.4.0 =======================
API Changes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ff9cc1d8/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
index cda855d..18b7559 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
@@ -76,7 +76,7 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
private CompressingStoredFieldsIndexWriter indexWriter;
private IndexOutput fieldsStream;
- private final Compressor compressor;
+ private Compressor compressor;
private final CompressionMode compressionMode;
private final int chunkSize;
private final int maxDocsPerChunk;
@@ -136,10 +136,11 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
@Override
public void close() throws IOException {
try {
- IOUtils.close(fieldsStream, indexWriter);
+ IOUtils.close(fieldsStream, indexWriter, compressor);
} finally {
fieldsStream = null;
indexWriter = null;
+ compressor = null;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ff9cc1d8/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
index 326eba3..53a84cb 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
@@ -164,6 +164,10 @@ public abstract class CompressionMode {
LZ4.compress(bytes, off, len, out, ht);
}
+ @Override
+ public void close() throws IOException {
+ // no-op
+ }
}
private static final class LZ4HighCompressor extends Compressor {
@@ -180,15 +184,17 @@ public abstract class CompressionMode {
LZ4.compressHC(bytes, off, len, out, ht);
}
+ @Override
+ public void close() throws IOException {
+ // no-op
+ }
}
private static final class DeflateDecompressor extends Decompressor {
- final Inflater decompressor;
byte[] compressed;
DeflateDecompressor() {
- decompressor = new Inflater(true);
compressed = new byte[0];
}
@@ -207,20 +213,24 @@ public abstract class CompressionMode {
in.readBytes(compressed, 0, compressedLength);
compressed[compressedLength] = 0; // explicitly set dummy byte to 0
- decompressor.reset();
- // extra "dummy byte"
- decompressor.setInput(compressed, 0, paddedLength);
-
- bytes.offset = bytes.length = 0;
- bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength);
+ final Inflater decompressor = new Inflater(true);
try {
- bytes.length = decompressor.inflate(bytes.bytes, bytes.length, originalLength);
- } catch (DataFormatException e) {
- throw new IOException(e);
- }
- if (!decompressor.finished()) {
- throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput()
- + ", needsDict=" + decompressor.needsDictionary(), in);
+ // extra "dummy byte"
+ decompressor.setInput(compressed, 0, paddedLength);
+
+ bytes.offset = bytes.length = 0;
+ bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength);
+ try {
+ bytes.length = decompressor.inflate(bytes.bytes, bytes.length, originalLength);
+ } catch (DataFormatException e) {
+ throw new IOException(e);
+ }
+ if (!decompressor.finished()) {
+ throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput()
+ + ", needsDict=" + decompressor.needsDictionary(), in);
+ }
+ } finally {
+ decompressor.end();
}
if (bytes.length != originalLength) {
throw new CorruptIndexException("Lengths mismatch: " + bytes.length + " != " + originalLength, in);
@@ -240,6 +250,7 @@ public abstract class CompressionMode {
final Deflater compressor;
byte[] compressed;
+ boolean closed;
DeflateCompressor(int level) {
compressor = new Deflater(level, true);
@@ -275,6 +286,14 @@ public abstract class CompressionMode {
out.writeBytes(compressed, totalCount);
}
+ @Override
+ public void close() throws IOException {
+ if (closed == false) {
+ compressor.end();
+ closed = true;
+ }
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ff9cc1d8/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
index bd2fadb..f95246c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
@@ -17,6 +17,7 @@
package org.apache.lucene.codecs.compressing;
+import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.store.DataOutput;
@@ -24,7 +25,7 @@ import org.apache.lucene.store.DataOutput;
/**
* A data compressor.
*/
-public abstract class Compressor {
+public abstract class Compressor implements Closeable {
/** Sole constructor, typically called from sub-classes. */
protected Compressor() {}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ff9cc1d8/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
index d15adad..167418e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
@@ -79,6 +79,9 @@ public class DummyCompressingCodec extends CompressingCodec {
out.writeBytes(bytes, off, len);
}
+ @Override
+ public void close() throws IOException {};
+
};
/** Constructor that allows to configure the chunk size. */