You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ds...@apache.org on 2016/07/25 14:00:22 UTC
lucene-solr:master: LUCENE-7391: Fix MemoryIndex fields() perf
regression
Repository: lucene-solr
Updated Branches:
refs/heads/master 2d07ffd97 -> dcc9a4b9a
LUCENE-7391: Fix MemoryIndex fields() perf regression
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/dcc9a4b9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/dcc9a4b9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/dcc9a4b9
Branch: refs/heads/master
Commit: dcc9a4b9a411a0436e5cb21e7d6251691640f3db
Parents: 2d07ffd
Author: David Smiley <ds...@apache.org>
Authored: Mon Jul 25 09:55:32 2016 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Mon Jul 25 09:59:00 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../apache/lucene/index/memory/MemoryIndex.java | 24 ++++++++++----------
.../lucene/index/memory/TestMemoryIndex.java | 14 ++++++++++++
3 files changed, 29 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dcc9a4b9/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 1edac7e..7b53017 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -79,6 +79,9 @@ Bug Fixes
* LUCENE-7389: Fix FieldType.setDimensions(...) validation for the dimensionNumBytes
parameter. (Martijn van Groningen)
+* LUCENE-7391: Fix performance regression in MemoryIndex's fields() introduced
+ in Lucene 6. (Steve Mason via David Smiley)
+
Improvements
* LUCENE-7323: Compound file writing now verifies the incoming
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dcc9a4b9/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index cdd53ed..9037dfa 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -21,11 +21,9 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
-import java.util.Locale;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
-import java.util.stream.Collectors;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -1045,6 +1043,7 @@ public class MemoryIndex {
private final class MemoryIndexReader extends LeafReader {
private final PointValues pointValues;
+ private Fields memoryFields = new MemoryFields(fields);
private MemoryIndexReader() {
super(); // avoid as much superclass baggage as possible
@@ -1198,13 +1197,7 @@ public class MemoryIndex {
@Override
public Fields fields() {
- Map<String, Info> filteredFields = fields.entrySet().stream()
- .filter(entry -> entry.getValue().numTokens > 0)
- .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
- (u,v) -> { throw new IllegalStateException(String.format(Locale.ROOT, "Duplicate key %s", u));},
- TreeMap::new
- ));
- return new MemoryFields(filteredFields );
+ return memoryFields;
}
private class MemoryFields extends Fields {
@@ -1217,13 +1210,16 @@ public class MemoryIndex {
@Override
public Iterator<String> iterator() {
- return fields.keySet().iterator();
+ return fields.entrySet().stream()
+ .filter(e -> e.getValue().numTokens > 0)
+ .map(Map.Entry::getKey)
+ .iterator();
}
@Override
public Terms terms(final String field) {
final Info info = fields.get(field);
- if (info == null) {
+ if (info == null || info.numTokens <= 0) {
return null;
}
@@ -1278,7 +1274,11 @@ public class MemoryIndex {
@Override
public int size() {
- return fields.size();
+ int size = 0;
+ for (String fieldName : this) {
+ size++;
+ }
+ return size;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dcc9a4b9/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index 2f95a4e..1d7c60e 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -45,6 +45,7 @@ import org.apache.lucene.document.TextField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
@@ -129,6 +130,19 @@ public class TestMemoryIndex extends LuceneTestCase {
assertEquals("be", terms.term().utf8ToString());
TestUtil.checkReader(reader);
}
+
+ public void testFieldsOnlyReturnsIndexedFields() throws IOException {
+ Document doc = new Document();
+
+ doc.add(new NumericDocValuesField("numeric", 29L));
+ doc.add(new TextField("text", "some text", Field.Store.NO));
+
+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
+ IndexSearcher searcher = mi.createSearcher();
+ IndexReader reader = searcher.getIndexReader();
+
+ assertEquals(reader.getTermVectors(0).size(), 1);
+ }
public void testReaderConsistency() throws IOException {
Analyzer analyzer = new MockPayloadAnalyzer();