You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2017/02/06 10:58:00 UTC
[1/3] lucene-solr:branch_5_5: LUCENE-7657: Fixed potential memory
leak when a (Span)TermQuery that wraps a TermContext is cached.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_5_5 ae789c252 -> da641ba2d
LUCENE-7657: Fixed potential memory leak when a (Span)TermQuery that wraps a TermContext is cached.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/af77c60a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/af77c60a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/af77c60a
Branch: refs/heads/branch_5_5
Commit: af77c60a71f3d26848641104e4e3f44d23e4e4a9
Parents: ae789c2
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed Jan 25 16:10:47 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon Feb 6 10:35:39 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../apache/lucene/index/IndexReaderContext.java | 7 ++++++-
.../org/apache/lucene/index/TermContext.java | 21 ++++++++++++--------
.../apache/lucene/search/BlendedTermQuery.java | 12 ++++++-----
.../org/apache/lucene/search/TermQuery.java | 4 ++--
.../lucene/search/spans/SpanTermQuery.java | 4 ++--
.../lucene/search/TermAutomatonQuery.java | 2 +-
7 files changed, 34 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af77c60a/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d4253e7..b3f6938 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -11,6 +11,9 @@ Bug Fixes
trying to highlight a query containing a degenerate case of a MultiPhraseQuery with one
term. (Thomas Kappler via David Smiley)
+* LUCENE-7657: Fixed potential memory leak in the case that a (Span)TermQuery
+ with a TermContext is cached. (Adrien Grand)
+
Build
* LUCENE-7543: Make changes-to-html target an offline operation, by moving the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af77c60a/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java b/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java
index 247fa57..dada3ff 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java
@@ -32,7 +32,12 @@ public abstract class IndexReaderContext {
public final int docBaseInParent;
/** the ord for this reader in the parent, <tt>0</tt> if parent is null */
public final int ordInParent;
-
+
+ // An object that uniquely identifies this context without referencing
+ // segments. The goal is to make it fine to have references to this
+ // identity object, even after the index reader has been closed
+ final Object identity = new Object();
+
IndexReaderContext(CompositeReaderContext parent, int ordInParent, int docBaseInParent) {
if (!(this instanceof CompositeReaderContext || this instanceof LeafReaderContext))
throw new Error("This class should never be extended by custom code!");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af77c60a/lucene/core/src/java/org/apache/lucene/index/TermContext.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermContext.java b/lucene/core/src/java/org/apache/lucene/index/TermContext.java
index ee5163e..b893038 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermContext.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermContext.java
@@ -33,12 +33,8 @@ import java.util.Arrays;
*/
public final class TermContext {
- /** Holds the {@link IndexReaderContext} of the top-level
- * {@link IndexReader}, used internally only for
- * asserting.
- *
- * @lucene.internal */
- public final IndexReaderContext topReaderContext;
+ // Important: do NOT keep hard references to index readers
+ private final Object topReaderContextIdentity;
private final TermState[] states;
private int docFreq;
private long totalTermFreq;
@@ -50,7 +46,7 @@ public final class TermContext {
*/
public TermContext(IndexReaderContext context) {
assert context != null && context.isTopLevel;
- topReaderContext = context;
+ topReaderContextIdentity = context.identity;
docFreq = 0;
totalTermFreq = 0;
final int len;
@@ -61,7 +57,16 @@ public final class TermContext {
}
states = new TermState[len];
}
-
+
+ /**
+ * Expert: Return whether this {@link TermContext} was built for the given
+ * {@link IndexReaderContext}. This is typically used for assertions.
+ * @lucene.internal
+ */
+ public boolean wasBuiltFor(IndexReaderContext context) {
+ return topReaderContextIdentity == context.identity;
+ }
+
/**
* Creates a {@link TermContext} with an initial {@link TermState},
* {@link IndexReader} pair.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af77c60a/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java
index e407df8..971b941 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
@@ -271,7 +272,7 @@ public final class BlendedTermQuery extends Query {
}
final TermContext[] contexts = Arrays.copyOf(this.contexts, this.contexts.length);
for (int i = 0; i < contexts.length; ++i) {
- if (contexts[i] == null || contexts[i].topReaderContext != reader.getContext()) {
+ if (contexts[i] == null || contexts[i].wasBuiltFor(reader.getContext()) == false) {
contexts[i] = TermContext.build(reader.getContext(), terms[i]);
}
}
@@ -291,7 +292,7 @@ public final class BlendedTermQuery extends Query {
}
for (int i = 0; i < contexts.length; ++i) {
- contexts[i] = adjustFrequencies(contexts[i], df, ttf);
+ contexts[i] = adjustFrequencies(reader.getContext(), contexts[i], df, ttf);
}
Query[] termQueries = new Query[terms.length];
@@ -304,15 +305,16 @@ public final class BlendedTermQuery extends Query {
return rewriteMethod.rewrite(termQueries);
}
- private static TermContext adjustFrequencies(TermContext ctx, int artificialDf, long artificialTtf) {
- List<LeafReaderContext> leaves = ctx.topReaderContext.leaves();
+ private static TermContext adjustFrequencies(IndexReaderContext readerContext,
+ TermContext ctx, int artificialDf, long artificialTtf) {
+ List<LeafReaderContext> leaves = readerContext.leaves();
final int len;
if (leaves == null) {
len = 1;
} else {
len = leaves.size();
}
- TermContext newCtx = new TermContext(ctx.topReaderContext);
+ TermContext newCtx = new TermContext(readerContext);
for (int i = 0; i < len; ++i) {
TermState termState = ctx.get(i);
if (termState == null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af77c60a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
index 63a2eeb..b9d7a9b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
@@ -98,7 +98,7 @@ public class TermQuery extends Query {
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
- assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
+ assert termStates.wasBuiltFor(ReaderUtil.getTopLevelContext(context)) : "The top-reader used to create Weight is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
final TermsEnum termsEnum = getTermsEnum(context);
if (termsEnum == null) {
return null;
@@ -186,7 +186,7 @@ public class TermQuery extends Query {
final IndexReaderContext context = searcher.getTopReaderContext();
final TermContext termState;
if (perReaderTermState == null
- || perReaderTermState.topReaderContext != context) {
+ || perReaderTermState.wasBuiltFor(context) == false) {
// make TermQuery single-pass if we don't have a PRTS or if the context
// differs!
termState = TermContext.build(context, term);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af77c60a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
index b25983f..dd83871 100644
--- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
@@ -68,7 +68,7 @@ public class SpanTermQuery extends SpanQuery {
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
final TermContext context;
final IndexReaderContext topContext = searcher.getTopReaderContext();
- if (termContext == null || termContext.topReaderContext != topContext) {
+ if (termContext == null || termContext.wasBuiltFor(topContext) == false) {
context = TermContext.build(topContext, term);
}
else {
@@ -100,7 +100,7 @@ public class SpanTermQuery extends SpanQuery {
@Override
public Spans getSpans(final LeafReaderContext context, Postings requiredPostings) throws IOException {
- assert termContext.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termContext.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
+ assert termContext.wasBuiltFor(ReaderUtil.getTopLevelContext(context)) : "The top-reader used to create Weight is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
final TermState state = termContext.get(context.ord);
if (state == null) { // term is not present in that reader
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af77c60a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
index e9321df..0962184 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
@@ -386,7 +386,7 @@ public class TermAutomatonQuery extends Query {
boolean any = false;
for(Map.Entry<Integer,TermContext> ent : termStates.entrySet()) {
TermContext termContext = ent.getValue();
- assert termContext.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termContext.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
+ assert termContext.wasBuiltFor(ReaderUtil.getTopLevelContext(context)) : "The top-reader used to create Weight is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
BytesRef term = idToTerm.get(ent.getKey());
TermState state = termContext.get(context.ord);
if (state != null) {
[3/3] lucene-solr:branch_5_5: LUCENE-7562: don't throw NPE when
encountering a level 2 ghost field
Posted by jp...@apache.org.
LUCENE-7562: don't throw NPE when encountering a level 2 ghost field
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/da641ba2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/da641ba2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/da641ba2
Branch: refs/heads/branch_5_5
Commit: da641ba2da1675538176767d2d84bcf3cb5c4f21
Parents: 129624e
Author: Mike McCandless <mi...@apache.org>
Authored: Wed Nov 16 04:53:50 2016 -0500
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon Feb 6 11:34:43 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 ++
.../document/CompletionFieldsConsumer.java | 4 ++
.../document/TestPrefixCompletionQuery.java | 49 +++++++++++++++++---
3 files changed, 49 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da641ba2/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index dc28bfc..9ccfd30 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -18,6 +18,9 @@ Bug Fixes
configured with BEST_COMPRESSION. This could otherwise result in out-of-memory
issues. (Adrien Grand)
+* LUCENE-7562: CompletionFieldsConsumer sometimes throws
+ NullPointerException on ghost fields (Oliver Eilhard via Mike McCandless)
+
Build
* LUCENE-7543: Make changes-to-html target an offline operation, by moving the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da641ba2/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java
index 672c582..9df9d60 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java
@@ -86,6 +86,10 @@ final class CompletionFieldsConsumer extends FieldsConsumer {
for (String field : fields) {
CompletionTermWriter termWriter = new CompletionTermWriter();
Terms terms = fields.terms(field);
+ if (terms == null) {
+ // this can happen from ghost fields, where the incoming Fields iterator claims a field exists but it does not
+ continue;
+ }
TermsEnum termsEnum = terms.iterator();
// write terms
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da641ba2/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
index c37fa5a..76f3df8 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
@@ -24,9 +24,12 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SortedNumericDocValues;
@@ -37,7 +40,6 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.After;
import org.junit.Before;
-import org.junit.Test;
import static org.apache.lucene.search.suggest.document.TestSuggestField.Entry;
import static org.apache.lucene.search.suggest.document.TestSuggestField.assertSuggestions;
@@ -119,7 +121,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
dir.close();
}
- @Test
public void testSimple() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
@@ -147,7 +148,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
- @Test
public void testMostlyFilteredOutDocuments() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
@@ -194,7 +194,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
- @Test
public void testDocFiltering() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
@@ -236,7 +235,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
- @Test
public void testAnalyzerWithoutPreservePosAndSep() throws Exception {
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, false, false);
@@ -260,7 +258,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
- @Test
public void testAnalyzerWithSepAndNoPreservePos() throws Exception {
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, true, false);
@@ -284,7 +281,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
- @Test
public void testAnalyzerWithPreservePosAndNoSep() throws Exception {
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, false, true);
@@ -308,4 +304,43 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
+ public void testGhostField() throws Exception {
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriter iw = new IndexWriter(dir, iwcWithSuggestField(analyzer, "suggest_field", "suggest_field2", "suggest_field3"));
+
+ Document document = new Document();
+ document.add(new StringField("id", "0", Field.Store.NO));
+ document.add(new SuggestField("suggest_field", "apples", 3));
+ iw.addDocument(document);
+ // need another document so whole segment isn't deleted
+ iw.addDocument(new Document());
+ iw.commit();
+
+ document = new Document();
+ document.add(new StringField("id", "1", Field.Store.NO));
+ document.add(new SuggestField("suggest_field2", "apples", 3));
+ iw.addDocument(document);
+ iw.commit();
+
+ iw.deleteDocuments(new Term("id", "0"));
+ // first force merge is OK
+ iw.forceMerge(1);
+
+ // second force merge causes MultiFields to include "suggest_field" in its iteration, yet a null Terms is returned (no documents have
+ // this field anymore)
+ iw.addDocument(new Document());
+ iw.forceMerge(1);
+
+ DirectoryReader reader = DirectoryReader.open(iw);
+ SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
+
+ PrefixCompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field", "app"));
+ assertEquals(0, indexSearcher.suggest(query, 3).totalHits);
+
+ query = new PrefixCompletionQuery(analyzer, new Term("suggest_field2", "app"));
+ assertSuggestions(indexSearcher.suggest(query, 3), new Entry("apples", 3));
+
+ reader.close();
+ iw.close();
+ }
}
[2/3] lucene-solr:branch_5_5: LUCENE-7647:
CompressingStoredFieldsFormat should reclaim memory more aggressively.
Posted by jp...@apache.org.
LUCENE-7647: CompressingStoredFieldsFormat should reclaim memory more aggressively.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/129624e5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/129624e5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/129624e5
Branch: refs/heads/branch_5_5
Commit: 129624e5fbfdd6fa4a9904189caf416dbf6412ad
Parents: af77c60
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed Jan 25 16:15:04 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon Feb 6 10:41:54 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 ++
.../CompressingStoredFieldsWriter.java | 5 +-
.../codecs/compressing/CompressionMode.java | 49 ++++++++++++++------
.../lucene/codecs/compressing/Compressor.java | 3 +-
.../dummy/DummyCompressingCodec.java | 3 ++
5 files changed, 46 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/129624e5/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b3f6938..dc28bfc 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -14,6 +14,10 @@ Bug Fixes
* LUCENE-7657: Fixed potential memory leak in the case that a (Span)TermQuery
with a TermContext is cached. (Adrien Grand)
+* LUCENE-7647: Made stored fields reclaim native memory more aggressively when
+ configured with BEST_COMPRESSION. This could otherwise result in out-of-memory
+ issues. (Adrien Grand)
+
Build
* LUCENE-7543: Make changes-to-html target an offline operation, by moving the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/129624e5/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
index 79dfb27..2ad54b2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
@@ -75,7 +75,7 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
private CompressingStoredFieldsIndexWriter indexWriter;
private IndexOutput fieldsStream;
- private final Compressor compressor;
+ private Compressor compressor;
private final CompressionMode compressionMode;
private final int chunkSize;
private final int maxDocsPerChunk;
@@ -135,10 +135,11 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
@Override
public void close() throws IOException {
try {
- IOUtils.close(fieldsStream, indexWriter);
+ IOUtils.close(fieldsStream, indexWriter, compressor);
} finally {
fieldsStream = null;
indexWriter = null;
+ compressor = null;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/129624e5/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
index 326eba3..53a84cb 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
@@ -164,6 +164,10 @@ public abstract class CompressionMode {
LZ4.compress(bytes, off, len, out, ht);
}
+ @Override
+ public void close() throws IOException {
+ // no-op
+ }
}
private static final class LZ4HighCompressor extends Compressor {
@@ -180,15 +184,17 @@ public abstract class CompressionMode {
LZ4.compressHC(bytes, off, len, out, ht);
}
+ @Override
+ public void close() throws IOException {
+ // no-op
+ }
}
private static final class DeflateDecompressor extends Decompressor {
- final Inflater decompressor;
byte[] compressed;
DeflateDecompressor() {
- decompressor = new Inflater(true);
compressed = new byte[0];
}
@@ -207,20 +213,24 @@ public abstract class CompressionMode {
in.readBytes(compressed, 0, compressedLength);
compressed[compressedLength] = 0; // explicitly set dummy byte to 0
- decompressor.reset();
- // extra "dummy byte"
- decompressor.setInput(compressed, 0, paddedLength);
-
- bytes.offset = bytes.length = 0;
- bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength);
+ final Inflater decompressor = new Inflater(true);
try {
- bytes.length = decompressor.inflate(bytes.bytes, bytes.length, originalLength);
- } catch (DataFormatException e) {
- throw new IOException(e);
- }
- if (!decompressor.finished()) {
- throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput()
- + ", needsDict=" + decompressor.needsDictionary(), in);
+ // extra "dummy byte"
+ decompressor.setInput(compressed, 0, paddedLength);
+
+ bytes.offset = bytes.length = 0;
+ bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength);
+ try {
+ bytes.length = decompressor.inflate(bytes.bytes, bytes.length, originalLength);
+ } catch (DataFormatException e) {
+ throw new IOException(e);
+ }
+ if (!decompressor.finished()) {
+ throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput()
+ + ", needsDict=" + decompressor.needsDictionary(), in);
+ }
+ } finally {
+ decompressor.end();
}
if (bytes.length != originalLength) {
throw new CorruptIndexException("Lengths mismatch: " + bytes.length + " != " + originalLength, in);
@@ -240,6 +250,7 @@ public abstract class CompressionMode {
final Deflater compressor;
byte[] compressed;
+ boolean closed;
DeflateCompressor(int level) {
compressor = new Deflater(level, true);
@@ -275,6 +286,14 @@ public abstract class CompressionMode {
out.writeBytes(compressed, totalCount);
}
+ @Override
+ public void close() throws IOException {
+ if (closed == false) {
+ compressor.end();
+ closed = true;
+ }
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/129624e5/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
index bd2fadb..f95246c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
@@ -17,6 +17,7 @@
package org.apache.lucene.codecs.compressing;
+import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.store.DataOutput;
@@ -24,7 +25,7 @@ import org.apache.lucene.store.DataOutput;
/**
* A data compressor.
*/
-public abstract class Compressor {
+public abstract class Compressor implements Closeable {
/** Sole constructor, typically called from sub-classes. */
protected Compressor() {}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/129624e5/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
index d15adad..167418e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
@@ -79,6 +79,9 @@ public class DummyCompressingCodec extends CompressingCodec {
out.writeBytes(bytes, off, len);
}
+ @Override
+ public void close() throws IOException {};
+
};
/** Constructor that allows to configure the chunk size. */