You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/11/16 09:54:46 UTC
lucene-solr:branch_6x: LUCENE-7562: don't throw NPE when encountering
a level 2 ghost field
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 8c7decb4c -> 42459a036
LUCENE-7562: don't throw NPE when encountering a level 2 ghost field
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/42459a03
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/42459a03
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/42459a03
Branch: refs/heads/branch_6x
Commit: 42459a036d4fb5077b069797570f06318ce47929
Parents: 8c7decb
Author: Mike McCandless <mi...@apache.org>
Authored: Wed Nov 16 04:53:50 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Nov 16 04:54:16 2016 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 ++
.../document/CompletionFieldsConsumer.java | 4 ++
.../document/TestPrefixCompletionQuery.java | 49 +++++++++++++++++---
3 files changed, 49 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/42459a03/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f3f6169..f4479f8 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -14,6 +14,9 @@ Bug Fixes
* LUCENE-7547: JapaneseTokenizerFactory was failing to close the
dictionary file it opened (Markus via Mike McCandless)
+* LUCENE-7562: CompletionFieldsConsumer sometimes throws
+ NullPointerException on ghost fields (Oliver Eilhard via Mike McCandless)
+
Improvements
* LUCENE-7532: Add back lost codec file format documentation
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/42459a03/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java
index 672c582..9df9d60 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java
@@ -86,6 +86,10 @@ final class CompletionFieldsConsumer extends FieldsConsumer {
for (String field : fields) {
CompletionTermWriter termWriter = new CompletionTermWriter();
Terms terms = fields.terms(field);
+ if (terms == null) {
+ // this can happen from ghost fields, where the incoming Fields iterator claims a field exists but it does not
+ continue;
+ }
TermsEnum termsEnum = terms.iterator();
// write terms
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/42459a03/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
index 777c0c8..7ccfa34 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
@@ -24,9 +24,12 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SortedNumericDocValues;
@@ -37,7 +40,6 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.After;
import org.junit.Before;
-import org.junit.Test;
import static org.apache.lucene.search.suggest.document.TestSuggestField.Entry;
import static org.apache.lucene.search.suggest.document.TestSuggestField.assertSuggestions;
@@ -119,7 +121,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
dir.close();
}
- @Test
public void testSimple() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
@@ -148,7 +149,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
- @Test
public void testMostlyFilteredOutDocuments() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
@@ -195,7 +195,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
- @Test
public void testDocFiltering() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
@@ -237,7 +236,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
- @Test
public void testAnalyzerWithoutPreservePosAndSep() throws Exception {
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, false, false);
@@ -261,7 +259,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
- @Test
public void testAnalyzerWithSepAndNoPreservePos() throws Exception {
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, true, false);
@@ -285,7 +282,6 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
- @Test
public void testAnalyzerWithPreservePosAndNoSep() throws Exception {
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, false, true);
@@ -309,4 +305,43 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
iw.close();
}
+ public void testGhostField() throws Exception {
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriter iw = new IndexWriter(dir, iwcWithSuggestField(analyzer, "suggest_field", "suggest_field2", "suggest_field3"));
+
+ Document document = new Document();
+ document.add(new StringField("id", "0", Field.Store.NO));
+ document.add(new SuggestField("suggest_field", "apples", 3));
+ iw.addDocument(document);
+ // need another document so whole segment isn't deleted
+ iw.addDocument(new Document());
+ iw.commit();
+
+ document = new Document();
+ document.add(new StringField("id", "1", Field.Store.NO));
+ document.add(new SuggestField("suggest_field2", "apples", 3));
+ iw.addDocument(document);
+ iw.commit();
+
+ iw.deleteDocuments(new Term("id", "0"));
+ // first force merge is OK
+ iw.forceMerge(1);
+
+ // second force merge causes MultiFields to include "suggest_field" in its iteration, yet a null Terms is returned (no documents have
+ // this field anymore)
+ iw.addDocument(new Document());
+ iw.forceMerge(1);
+
+ DirectoryReader reader = DirectoryReader.open(iw);
+ SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
+
+ PrefixCompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field", "app"));
+ assertEquals(0, indexSearcher.suggest(query, 3).totalHits);
+
+ query = new PrefixCompletionQuery(analyzer, new Term("suggest_field2", "app"));
+ assertSuggestions(indexSearcher.suggest(query, 3), new Entry("apples", 3));
+
+ reader.close();
+ iw.close();
+ }
}