You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ds...@apache.org on 2016/07/25 14:03:24 UTC
lucene-solr:branch_6x: LUCENE-7391: Fix MemoryIndex fields() perf
regression (cherry picked from commit dcc9a4b)
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 3daa82c65 -> d85a97185
LUCENE-7391: Fix MemoryIndex fields() perf regression
(cherry picked from commit dcc9a4b)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d85a9718
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d85a9718
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d85a9718
Branch: refs/heads/branch_6x
Commit: d85a9718564abf3dd33f66d057f8cfc835a56715
Parents: 3daa82c
Author: David Smiley <ds...@apache.org>
Authored: Mon Jul 25 09:55:32 2016 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Mon Jul 25 10:03:21 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../apache/lucene/index/memory/MemoryIndex.java | 24 ++++++++++----------
.../lucene/index/memory/TestMemoryIndex.java | 14 ++++++++++++
3 files changed, 29 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d85a9718/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 843b704..f58fd04 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -54,6 +54,9 @@ Bug Fixes
* LUCENE-7389: Fix FieldType.setDimensions(...) validation for the dimensionNumBytes
parameter. (Martijn van Groningen)
+* LUCENE-7391: Fix performance regression in MemoryIndex's fields() introduced
+ in Lucene 6. (Steve Mason via David Smiley)
+
Improvements
* LUCENE-7323: Compound file writing now verifies the incoming
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d85a9718/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index cdd53ed..9037dfa 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -21,11 +21,9 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
-import java.util.Locale;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
-import java.util.stream.Collectors;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -1045,6 +1043,7 @@ public class MemoryIndex {
private final class MemoryIndexReader extends LeafReader {
private final PointValues pointValues;
+ private Fields memoryFields = new MemoryFields(fields);
private MemoryIndexReader() {
super(); // avoid as much superclass baggage as possible
@@ -1198,13 +1197,7 @@ public class MemoryIndex {
@Override
public Fields fields() {
- Map<String, Info> filteredFields = fields.entrySet().stream()
- .filter(entry -> entry.getValue().numTokens > 0)
- .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
- (u,v) -> { throw new IllegalStateException(String.format(Locale.ROOT, "Duplicate key %s", u));},
- TreeMap::new
- ));
- return new MemoryFields(filteredFields );
+ return memoryFields;
}
private class MemoryFields extends Fields {
@@ -1217,13 +1210,16 @@ public class MemoryIndex {
@Override
public Iterator<String> iterator() {
- return fields.keySet().iterator();
+ return fields.entrySet().stream()
+ .filter(e -> e.getValue().numTokens > 0)
+ .map(Map.Entry::getKey)
+ .iterator();
}
@Override
public Terms terms(final String field) {
final Info info = fields.get(field);
- if (info == null) {
+ if (info == null || info.numTokens <= 0) {
return null;
}
@@ -1278,7 +1274,11 @@ public class MemoryIndex {
@Override
public int size() {
- return fields.size();
+ int size = 0;
+ for (String fieldName : this) {
+ size++;
+ }
+ return size;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d85a9718/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index 2f95a4e..1d7c60e 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -45,6 +45,7 @@ import org.apache.lucene.document.TextField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
@@ -129,6 +130,19 @@ public class TestMemoryIndex extends LuceneTestCase {
assertEquals("be", terms.term().utf8ToString());
TestUtil.checkReader(reader);
}
+
+ public void testFieldsOnlyReturnsIndexedFields() throws IOException {
+ Document doc = new Document();
+
+ doc.add(new NumericDocValuesField("numeric", 29L));
+ doc.add(new TextField("text", "some text", Field.Store.NO));
+
+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
+ IndexSearcher searcher = mi.createSearcher();
+ IndexReader reader = searcher.getIndexReader();
+
+ assertEquals(reader.getTermVectors(0).size(), 1);
+ }
public void testReaderConsistency() throws IOException {
Analyzer analyzer = new MockPayloadAnalyzer();