You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2014/12/05 16:24:13 UTC
svn commit: r1643326 [2/4] - in /lucene/dev/branches/lucene2878: ./ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/compressing/
lucene/core/src/java/org/apac...
Modified: lucene/dev/branches/lucene2878/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/lucene2878/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Fri Dec 5 15:24:11 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -36,6 +37,7 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
@@ -78,8 +80,20 @@ public class HighlighterTest extends Bas
Directory ramDir;
public IndexSearcher searcher = null;
int numHighlights = 0;
- Analyzer analyzer;
+ MockAnalyzer analyzer;
TopDocs hits;
+ FieldType fieldType;//see doc()
+
+ final FieldType FIELD_TYPE_TV;
+ {
+ FieldType fieldType = new FieldType(TextField.TYPE_STORED);
+ fieldType.setStoreTermVectors(true);
+ fieldType.setStoreTermVectorPositions(true);
+ fieldType.setStoreTermVectorPayloads(true);
+ fieldType.setStoreTermVectorOffsets(true);
+ fieldType.freeze();
+ FIELD_TYPE_TV = fieldType;
+ }
String[] texts = {
"Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
@@ -121,9 +135,8 @@ public class HighlighterTest extends Bas
}
public void testHighlightingCommonTermsQuery() throws Exception {
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 3);
- query.add(new Term(FIELD_NAME, "this"));
+ query.add(new Term(FIELD_NAME, "this"));//stop-word
query.add(new Term(FIELD_NAME, "long"));
query.add(new Term(FIELD_NAME, "very"));
@@ -141,7 +154,7 @@ public class HighlighterTest extends Bas
Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
highlighter.setTextFragmenter(fragmenter);
String fragment = highlighter.getBestFragment(stream, storedField);
- assertEquals("Hello <B>this</B> is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);
+ assertEquals("Hello this is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);
doc = searcher.doc(hits.scoreDocs[1].doc);
storedField = doc.get(FIELD_NAME);
@@ -150,7 +163,7 @@ public class HighlighterTest extends Bas
.getIndexReader(), hits.scoreDocs[1].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer));
fragment = highlighter.getBestFragment(stream, storedField);
- assertEquals("<B>This</B> piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>", fragment);
+ assertEquals("This piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>", fragment);
}
public void testHighlightUnknowQueryAfterRewrite() throws IOException, InvalidTokenOffsetsException {
@@ -159,7 +172,7 @@ public class HighlighterTest extends Bas
@Override
public Query rewrite(IndexReader reader) throws IOException {
CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 3);
- query.add(new Term(FIELD_NAME, "this"));
+ query.add(new Term(FIELD_NAME, "this"));//stop-word
query.add(new Term(FIELD_NAME, "long"));
query.add(new Term(FIELD_NAME, "very"));
return query;
@@ -180,9 +193,7 @@ public class HighlighterTest extends Bas
return super.equals(obj);
}
};
-
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
-
+
searcher = newSearcher(reader);
TopDocs hits = searcher.search(query, 10);
assertEquals(2, hits.totalHits);
@@ -197,7 +208,7 @@ public class HighlighterTest extends Bas
Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
highlighter.setTextFragmenter(fragmenter);
String fragment = highlighter.getBestFragment(stream, storedField);
- assertEquals("Hello <B>this</B> is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);
+ assertEquals("Hello this is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);
doc = searcher.doc(hits.scoreDocs[1].doc);
storedField = doc.get(FIELD_NAME);
@@ -206,7 +217,7 @@ public class HighlighterTest extends Bas
.getIndexReader(), hits.scoreDocs[1].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer));
fragment = highlighter.getBestFragment(stream, storedField);
- assertEquals("<B>This</B> piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>", fragment);
+ assertEquals("This piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>", fragment);
}
@@ -252,8 +263,7 @@ public class HighlighterTest extends Bas
*/
private String highlightField(Query query, String fieldName, String text)
throws IOException, InvalidTokenOffsetsException {
- TokenStream tokenStream = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)
- .tokenStream(fieldName, text);
+ TokenStream tokenStream = analyzer.tokenStream(fieldName, text);
// Assuming "<B>", "</B>" used to highlight
SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
QueryScorer scorer = new QueryScorer(query, fieldName, FIELD_NAME);
@@ -351,8 +361,9 @@ public class HighlighterTest extends Bas
Highlighter highlighter = new Highlighter(this, scorer);
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -380,8 +391,9 @@ public class HighlighterTest extends Bas
highlighter = new Highlighter(this, scorer);
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -409,8 +421,9 @@ public class HighlighterTest extends Bas
highlighter = new Highlighter(this, scorer);
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -434,8 +447,9 @@ public class HighlighterTest extends Bas
Highlighter highlighter = new Highlighter(this, scorer);
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -458,8 +472,9 @@ public class HighlighterTest extends Bas
Highlighter highlighter = new Highlighter(this, scorer);
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -482,8 +497,9 @@ public class HighlighterTest extends Bas
Highlighter highlighter = new Highlighter(this, scorer);
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -567,7 +583,8 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = "parent document";
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");
@@ -592,8 +609,9 @@ public class HighlighterTest extends Bas
highlighter.setTextFragmenter(new SimpleFragmenter(40));
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -614,8 +632,9 @@ public class HighlighterTest extends Bas
int maxNumFragmentsRequired = 2;
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -644,8 +663,9 @@ public class HighlighterTest extends Bas
Highlighter highlighter = new Highlighter(this, scorer);
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 5));
@@ -698,8 +718,9 @@ public class HighlighterTest extends Bas
Highlighter highlighter = new Highlighter(this,scorer);
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -769,8 +790,9 @@ public class HighlighterTest extends Bas
highlighter.setTextFragmenter(new SimpleFragmenter(40));
int maxNumFragmentsRequired = 2;
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -963,11 +985,11 @@ public class HighlighterTest extends Bas
hits = searcher.search(query, null, 1000);
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(HighlighterTest.FIELD_NAME);
- int maxNumFragmentsRequired = 2;
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer); int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
QueryScorer scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -987,11 +1009,12 @@ public class HighlighterTest extends Bas
numHighlights = 0;
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(HighlighterTest.FIELD_NAME);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
QueryScorer scorer = new QueryScorer(query, null);
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -1011,11 +1034,11 @@ public class HighlighterTest extends Bas
numHighlights = 0;
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(HighlighterTest.FIELD_NAME);
- int maxNumFragmentsRequired = 2;
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer); int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
QueryScorer scorer = new QueryScorer(query, "random_field", HighlighterTest.FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -1185,8 +1208,9 @@ public class HighlighterTest extends Bas
doSearching(new TermQuery(new Term(FIELD_NAME, "kennedy")));
numHighlights = 0;
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);
@@ -1199,21 +1223,25 @@ public class HighlighterTest extends Bas
numHighlights = 0;
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);
- highlighter.getBestFragment(analyzer, FIELD_NAME, text);
+ highlighter.getBestFragment(tokenStream, text);
}
assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
numHighlights == 4);
numHighlights = 0;
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);
- highlighter.getBestFragments(analyzer, FIELD_NAME, text, 10);
+ highlighter.getBestFragments(tokenStream, text, 10);
}
assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
numHighlights == 4);
@@ -1339,8 +1367,9 @@ public class HighlighterTest extends Bas
doSearching(new TermQuery(new Term(FIELD_NAME, "kennedy")));
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);// new Highlighter(this, new
@@ -1368,9 +1397,10 @@ public class HighlighterTest extends Bas
}
public void testMaxSizeHighlight() throws Exception {
- final MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
- // we disable MockTokenizer checks because we will forcefully limit the
+ // we disable MockTokenizer checks because we will forcefully limit the
// tokenstream and call end() before incrementToken() returns false.
+ // But we first need to clear the re-used tokenstream components that have enableChecks.
+ analyzer.getReuseStrategy().setReusableComponents(analyzer, FIELD_NAME, null);
analyzer.setEnableChecks(false);
TestHighlightRunner helper = new TestHighlightRunner() {
@@ -1471,8 +1501,7 @@ public class HighlighterTest extends Bas
numHighlights = 0;
// test to show how rewritten query can still be used
searcher = newSearcher(reader);
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
-
+
BooleanQuery query = new BooleanQuery();
query.add(new WildcardQuery(new Term(FIELD_NAME, "jf?")), Occur.SHOULD);
query.add(new WildcardQuery(new Term(FIELD_NAME, "kenned*")), Occur.SHOULD);
@@ -1491,8 +1520,9 @@ public class HighlighterTest extends Bas
int maxNumFragmentsRequired = 3;
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(FIELD_NAME);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
Highlighter highlighter = getHighlighter(query, FIELD_NAME, HighlighterTest.this, false);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -1823,12 +1853,6 @@ public class HighlighterTest extends Bas
searchIndex();
}
- private Document doc( String f, String v ){
- Document doc = new Document();
- doc.add( new TextField( f, v, Field.Store.YES));
- return doc;
- }
-
private void makeIndex() throws IOException {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
writer.addDocument( doc( "t_text1", "random words for highlighting tests del" ) );
@@ -1867,6 +1891,34 @@ public class HighlighterTest extends Bas
reader.close();
}
+ /** If we have term vectors, we can highlight based on payloads */
+ public void testPayloadQuery() throws IOException, InvalidTokenOffsetsException {
+ final String text = "random words and words";//"words" at positions 1 & 4
+
+ Analyzer analyzer = new MockPayloadAnalyzer();//sets payload to "pos: X" (where X is position #)
+ try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer))) {
+ writer.deleteAll();
+ Document doc = new Document();
+
+ doc.add(new Field(FIELD_NAME, text, FIELD_TYPE_TV));
+ writer.addDocument(doc);
+ writer.commit();
+ }
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ Query query = new SpanPayloadCheckQuery(new SpanTermQuery(new Term(FIELD_NAME, "words")),
+ Collections.singleton("pos: 1".getBytes("UTF-8")));//just match the first "word" occurrence
+ IndexSearcher searcher = newSearcher(reader);
+ Scorer scorer = new QueryScorer(query, searcher.getIndexReader(), FIELD_NAME);
+ Highlighter h = new Highlighter(scorer);
+
+ TopDocs hits = searcher.search(query, null, 10);
+ assertEquals(1, hits.scoreDocs.length);
+ TokenStream stream = TokenSources.getAnyTokenStream(searcher.getIndexReader(), 0, FIELD_NAME, analyzer);
+ String result = h.getBestFragment(stream, text);
+ assertEquals("random <B>words</B> and words", result);//only highlight first "word"
+ }
+ }
+
/*
*
* public void testBigramAnalyzer() throws IOException, ParseException {
@@ -1934,14 +1986,21 @@ public class HighlighterTest extends Bas
public void setUp() throws Exception {
super.setUp();
+ //Not many use this setup:
a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
- analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
dir = newDirectory();
+
+ //Most tests use this setup:
+ analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
ramDir = newDirectory();
- IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
+ fieldType = random().nextBoolean() ? FIELD_TYPE_TV : TextField.TYPE_STORED;
+ IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(analyzer));
+
for (String text : texts) {
- addDoc(writer, text);
+ writer.addDocument(doc(FIELD_NAME, text));
}
+
+ // a few tests need other docs...:
Document doc = new Document();
doc.add(new IntField(NUMERIC_FIELD_NAME, 1, Field.Store.NO));
doc.add(new StoredField(NUMERIC_FIELD_NAME, 1));
@@ -1969,6 +2028,8 @@ public class HighlighterTest extends Bas
writer.forceMerge(1);
writer.close();
reader = DirectoryReader.open(ramDir);
+
+ //Misc:
numHighlights = 0;
}
@@ -1979,13 +2040,11 @@ public class HighlighterTest extends Bas
ramDir.close();
super.tearDown();
}
- private void addDoc(IndexWriter writer, String text) throws IOException {
- Document d = new Document();
-
- Field f = new TextField(FIELD_NAME, text, Field.Store.YES);
- d.add(f);
- writer.addDocument(d);
+ private Document doc(String name, String value) {
+ Document d = new Document();
+ d.add(new Field(name, value, fieldType));//fieldType is randomly chosen for term vectors in setUp
+ return d;
}
private static Token createToken(String term, int start, int offset)
@@ -2164,11 +2223,13 @@ final class SynonymTokenizer extends Tok
throws Exception {
for (int i = 0; i < hits.totalHits; i++) {
- String text = searcher.doc(hits.scoreDocs[i].doc).get(HighlighterTest.FIELD_NAME);
+ final StoredDocument doc = searcher.doc(hits.scoreDocs[i].doc);
+ String text = doc.get(HighlighterTest.FIELD_NAME);
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
Scorer scorer = null;
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
+ TokenStream tokenStream = TokenSources.getAnyTokenStream(searcher.getIndexReader(),
+ hits.scoreDocs[i].doc, HighlighterTest.FIELD_NAME, doc, analyzer);
if (mode == QUERY) {
scorer = new QueryScorer(query);
} else if (mode == QUERY_TERM) {
@@ -2176,7 +2237,6 @@ final class SynonymTokenizer extends Tok
}
Highlighter highlighter = new Highlighter(formatter, scorer);
highlighter.setTextFragmenter(frag);
-
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
fragmentSeparator);
if (LuceneTestCase.VERBOSE) System.out.println("\t" + result);
Modified: lucene/dev/branches/lucene2878/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene2878/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Fri Dec 5 15:24:11 2014
@@ -18,13 +18,12 @@ package org.apache.lucene.index.memory;
*/
import java.io.IOException;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
-import java.util.NoSuchElementException;
+import java.util.SortedMap;
+import java.util.TreeMap;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -188,10 +187,7 @@ import org.apache.lucene.util.RecyclingI
public class MemoryIndex {
/** info for each field: Map<String fieldName, Info field> */
- private final HashMap<String,Info> fields = new HashMap<>();
-
- /** fields sorted ascending by fieldName; lazily computed on demand */
- private transient Map.Entry<String,Info>[] sortedFields;
+ private final SortedMap<String,Info> fields = new TreeMap<>();
private final boolean storeOffsets;
@@ -201,29 +197,12 @@ public class MemoryIndex {
private final IntBlockPool intBlockPool;
// private final IntBlockPool.SliceReader postingsReader;
private final IntBlockPool.SliceWriter postingsWriter;
-
- private HashMap<String,FieldInfo> fieldInfos = new HashMap<>();
private Counter bytesUsed;
private boolean frozen = false;
private Similarity normSimilarity = IndexSearcher.getDefaultSimilarity();
-
- /**
- * Sorts term entries into ascending order; also works for
- * Arrays.binarySearch() and Arrays.sort()
- */
- private static final Comparator<Object> termComparator = new Comparator<Object>() {
- @Override
- @SuppressWarnings({"unchecked","rawtypes"})
- public int compare(Object o1, Object o2) {
- if (o1 instanceof Map.Entry<?,?>) o1 = ((Map.Entry<?,?>) o1).getKey();
- if (o2 instanceof Map.Entry<?,?>) o2 = ((Map.Entry<?,?>) o2).getKey();
- if (o1 == o2) return 0;
- return ((Comparable) o1).compareTo((Comparable) o2);
- }
- };
/**
* Constructs an empty instance.
@@ -245,7 +224,6 @@ public class MemoryIndex {
*/
public MemoryIndex(boolean storeOffsets) {
this(storeOffsets, 0);
-
}
/**
@@ -294,7 +272,7 @@ public class MemoryIndex {
addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName));
}
-
+
/**
* Convenience method; Creates and returns a token stream that generates a
* token for each keyword in the given collection, "as is", without any
@@ -427,10 +405,12 @@ public class MemoryIndex {
int pos = -1;
final BytesRefHash terms;
final SliceByteStartArray sliceArray;
- Info info = null;
+ Info info;
long sumTotalTermFreq = 0;
int offset = 0;
+ FieldInfo fieldInfo;
if ((info = fields.get(fieldName)) != null) {
+ fieldInfo = info.fieldInfo;
numTokens = info.numTokens;
numOverlapTokens = info.numOverlapTokens;
pos = info.lastPosition + positionIncrementGap;
@@ -440,16 +420,13 @@ public class MemoryIndex {
sliceArray = info.sliceArray;
sumTotalTermFreq = info.sumTotalTermFreq;
} else {
+ fieldInfo = new FieldInfo(fieldName, fields.size(), false, false, false,
+ this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS,
+ DocValuesType.NONE, -1, null);
sliceArray = new SliceByteStartArray(BytesRefHash.DEFAULT_CAPACITY);
terms = new BytesRefHash(byteBlockPool, BytesRefHash.DEFAULT_CAPACITY, sliceArray);
}
- if (!fieldInfos.containsKey(fieldName)) {
- fieldInfos.put(fieldName,
- new FieldInfo(fieldName, fieldInfos.size(), false, false, false,
- this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS,
- DocValuesType.NONE, -1, null));
- }
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
@@ -486,8 +463,7 @@ public class MemoryIndex {
// ensure infos.numTokens > 0 invariant; needed for correct operation of terms()
if (numTokens > 0) {
- fields.put(fieldName, new Info(terms, sliceArray, numTokens, numOverlapTokens, boost, pos, offsetAtt.endOffset() + offset, sumTotalTermFreq));
- sortedFields = null; // invalidate sorted view, if any
+ fields.put(fieldName, new Info(fieldInfo, terms, sliceArray, numTokens, numOverlapTokens, boost, pos, offsetAtt.endOffset() + offset, sumTotalTermFreq));
}
} catch (Exception e) { // can never happen
throw new RuntimeException(e);
@@ -508,7 +484,13 @@ public class MemoryIndex {
public void setSimilarity(Similarity similarity) {
if (frozen)
throw new IllegalArgumentException("Cannot set Similarity when MemoryIndex is frozen");
+ if (this.normSimilarity == similarity)
+ return;
this.normSimilarity = similarity;
+ //invalidate any cached norms that may exist
+ for (Info info : fields.values()) {
+ info.norms = null;
+ }
}
/**
@@ -526,17 +508,16 @@ public class MemoryIndex {
/**
* Prepares the MemoryIndex for querying in a non-lazy way.
- *
+ * <p>
* After calling this you can query the MemoryIndex from multiple threads, but you
* cannot subsequently add new data.
*/
public void freeze() {
this.frozen = true;
- sortFields();
- for (Map.Entry<String,Info> info : sortedFields) {
- info.getValue().sortTerms();
+ for (Info info : fields.values()) {
+ info.sortTerms();
+ info.getNormDocValues();//lazily computed
}
- calculateNormValues();
}
/**
@@ -587,7 +568,7 @@ public class MemoryIndex {
* NOT close the index reader!!! This avoids all sorts of
* unnecessary baggage and locking in the Lucene IndexReader
* superclass, all of which is completely unnecessary for this main
- * memory index data structure without thread-safety claims.
+ * memory index data structure.
*
* Wishing IndexReader would be an interface...
*
@@ -598,26 +579,6 @@ public class MemoryIndex {
}
}
- /** sorts into ascending order (on demand), reusing memory along the way */
- private void sortFields() {
- if (sortedFields == null) sortedFields = sort(fields);
- }
-
- /** returns a view of the given map's entries, sorted ascending by key */
- private static <K,V> Map.Entry<K,V>[] sort(HashMap<K,V> map) {
- int size = map.size();
- @SuppressWarnings("unchecked")
- Map.Entry<K,V>[] entries = new Map.Entry[size];
-
- Iterator<Map.Entry<K,V>> iter = map.entrySet().iterator();
- for (int i=0; i < size; i++) {
- entries[i] = iter.next();
- }
-
- if (size > 1) ArrayUtil.introSort(entries, termComparator);
- return entries;
- }
-
/**
* Returns a String representation of the index data for debugging purposes.
*
@@ -625,13 +586,11 @@ public class MemoryIndex {
*/
@Override
public String toString() {
- StringBuilder result = new StringBuilder(256);
- sortFields();
+ StringBuilder result = new StringBuilder(256);
int sumPositions = 0;
int sumTerms = 0;
final BytesRef spare = new BytesRef();
- for (int i=0; i < sortedFields.length; i++) {
- Map.Entry<String,Info> entry = sortedFields[i];
+ for (Map.Entry<String, Info> entry : fields.entrySet()) {
String fieldName = entry.getKey();
Info info = entry.getValue();
info.sortTerms();
@@ -639,20 +598,20 @@ public class MemoryIndex {
SliceByteStartArray sliceArray = info.sliceArray;
int numPositions = 0;
SliceReader postingsReader = new SliceReader(intBlockPool);
- for (int j=0; j < info.terms.size(); j++) {
+ for (int j = 0; j < info.terms.size(); j++) {
int ord = info.sortedTerms[j];
info.terms.get(ord, spare);
int freq = sliceArray.freq[ord];
result.append("\t'" + spare + "':" + freq + ":");
postingsReader.reset(sliceArray.start[ord], sliceArray.end[ord]);
result.append(" [");
- final int iters = storeOffsets ? 3 : 1;
- while(!postingsReader.endOfSlice()) {
+ final int iters = storeOffsets ? 3 : 1;
+ while (!postingsReader.endOfSlice()) {
result.append("(");
-
+
for (int k = 0; k < iters; k++) {
result.append(postingsReader.readInt());
- if (k < iters-1) {
+ if (k < iters - 1) {
result.append(", ");
}
}
@@ -660,13 +619,13 @@ public class MemoryIndex {
if (!postingsReader.endOfSlice()) {
result.append(",");
}
-
+
}
result.append("]");
result.append("\n");
numPositions += freq;
}
-
+
result.append("\tterms=" + info.terms.size());
result.append(", positions=" + numPositions);
result.append("\n");
@@ -674,26 +633,31 @@ public class MemoryIndex {
sumTerms += info.terms.size();
}
- result.append("\nfields=" + sortedFields.length);
+ result.append("\nfields=" + fields.size());
result.append(", terms=" + sumTerms);
result.append(", positions=" + sumPositions);
return result.toString();
}
/**
- * Index data structure for a field; Contains the tokenized term texts and
+ * Index data structure for a field; contains the tokenized term texts and
* their positions.
*/
- private static final class Info {
-
+ private final class Info {
+
+ private final FieldInfo fieldInfo;
+
+ /** The norms for this field; computed on demand. */
+ private transient NumericDocValues norms;
+
/**
* Term strings and their positions for this field: Map <String
* termText, ArrayIntList positions>
*/
- private final BytesRefHash terms;
+ private final BytesRefHash terms; // note unfortunate variable name class with Terms type
private final SliceByteStartArray sliceArray;
-
+
/** Terms sorted ascending by term text; computed on demand */
private transient int[] sortedTerms;
@@ -714,7 +678,8 @@ public class MemoryIndex {
/** the last offset encountered in this field for multi field support*/
private final int lastOffset;
- public Info(BytesRefHash terms, SliceByteStartArray sliceArray, int numTokens, int numOverlapTokens, float boost, int lastPosition, int lastOffset, long sumTotalTermFreq) {
+ public Info(FieldInfo fieldInfo, BytesRefHash terms, SliceByteStartArray sliceArray, int numTokens, int numOverlapTokens, float boost, int lastPosition, int lastOffset, long sumTotalTermFreq) {
+ this.fieldInfo = fieldInfo;
this.terms = terms;
this.sliceArray = sliceArray;
this.numTokens = numTokens;
@@ -725,10 +690,6 @@ public class MemoryIndex {
this.lastOffset = lastOffset;
}
- public long getSumTotalTermFreq() {
- return sumTotalTermFreq;
- }
-
/**
* Sorts hashed terms into ascending order, reusing memory along the
* way. Note that sorting is lazily delayed until required (often it's
@@ -738,12 +699,30 @@ public class MemoryIndex {
* apart from more sophisticated Tries / prefix trees).
*/
public void sortTerms() {
- if (sortedTerms == null)
+ if (sortedTerms == null) {
sortedTerms = terms.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
+ }
}
-
- public float getBoost() {
- return boost;
+
+ public NumericDocValues getNormDocValues() {
+ if (norms == null) {
+ FieldInvertState invertState = new FieldInvertState(fieldInfo.name, fieldInfo.number,
+ numTokens, numOverlapTokens, 0, boost);
+ final long value = normSimilarity.computeNorm(invertState);
+ if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldInfo.name + ":" + value + ":" + numTokens);
+ norms = new NumericDocValues() {
+
+ @Override
+ public long get(int docID) {
+ if (docID != 0)
+ throw new IndexOutOfBoundsException();
+ else
+ return value;
+ }
+
+ };
+ }
+ return norms;
}
}
@@ -774,10 +753,6 @@ public class MemoryIndex {
private Info getInfo(String fieldName) {
return fields.get(fieldName);
}
-
- private Info getInfo(int pos) {
- return sortedFields[pos].getValue();
- }
@Override
public Bits getLiveDocs() {
@@ -786,7 +761,12 @@ public class MemoryIndex {
@Override
public FieldInfos getFieldInfos() {
- return new FieldInfos(fieldInfos.values().toArray(new FieldInfo[fieldInfos.size()]));
+ FieldInfo[] fieldInfos = new FieldInfo[fields.size()];
+ int i = 0;
+ for (Info info : fields.values()) {
+ fieldInfos[i++] = info.fieldInfo;
+ }
+ return new FieldInfos(fieldInfos);
}
@Override
@@ -827,98 +807,72 @@ public class MemoryIndex {
private class MemoryFields extends Fields {
@Override
public Iterator<String> iterator() {
- return new Iterator<String>() {
- int upto = -1;
+ return fields.keySet().iterator();
+ }
+ @Override
+ public Terms terms(final String field) {
+ final Info info = fields.get(field);
+ if (info == null)
+ return null;
+
+ return new Terms() {
@Override
- public String next() {
- upto++;
- if (upto >= sortedFields.length) {
- throw new NoSuchElementException();
- }
- return sortedFields[upto].getKey();
+ public TermsEnum iterator(TermsEnum reuse) {
+ return new MemoryTermsEnum(info);
}
@Override
- public boolean hasNext() {
- return upto+1 < sortedFields.length;
+ public long size() {
+ return info.terms.size();
}
@Override
- public void remove() {
- throw new UnsupportedOperationException();
+ public long getSumTotalTermFreq() {
+ return info.sumTotalTermFreq;
}
- };
- }
-
- @Override
- public Terms terms(final String field) {
- int i = Arrays.binarySearch(sortedFields, field, termComparator);
- if (i < 0) {
- return null;
- } else {
- final Info info = getInfo(i);
- info.sortTerms();
- return new Terms() {
- @Override
- public TermsEnum iterator(TermsEnum reuse) {
- return new MemoryTermsEnum(info);
- }
-
- @Override
- public long size() {
- return info.terms.size();
- }
-
- @Override
- public long getSumTotalTermFreq() {
- return info.getSumTotalTermFreq();
- }
+ @Override
+ public long getSumDocFreq() {
+ // each term has df=1
+ return info.terms.size();
+ }
- @Override
- public long getSumDocFreq() {
- // each term has df=1
- return info.terms.size();
- }
+ @Override
+ public int getDocCount() {
+ return size() > 0 ? 1 : 0;
+ }
- @Override
- public int getDocCount() {
- return info.terms.size() > 0 ? 1 : 0;
- }
+ @Override
+ public boolean hasFreqs() {
+ return true;
+ }
- @Override
- public boolean hasFreqs() {
- return true;
- }
+ @Override
+ public boolean hasOffsets() {
+ return storeOffsets;
+ }
- @Override
- public boolean hasOffsets() {
- return storeOffsets;
- }
+ @Override
+ public boolean hasPositions() {
+ return true;
+ }
- @Override
- public boolean hasPositions() {
- return true;
- }
-
- @Override
- public boolean hasPayloads() {
- return false;
- }
- };
- }
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
+ };
}
@Override
public int size() {
- return sortedFields.length;
+ return fields.size();
}
}
@Override
public Fields fields() {
- sortFields();
return new MemoryFields();
}
@@ -1172,44 +1126,20 @@ public class MemoryIndex {
@Override
public NumericDocValues getNormValues(String field) {
- if (norms == null)
- return calculateFieldNormValue(field);
- return norms.get(field);
+ Info info = fields.get(field);
+ if (info == null) {
+ return null;
+ }
+ return info.getNormDocValues();
}
}
- private Map<String, NumericDocValues> norms = null;
-
- private NumericDocValues calculateFieldNormValue(String field) {
- FieldInfo fieldInfo = fieldInfos.get(field);
- if (fieldInfo == null)
- return null;
- Info info = fields.get(field);
- int numTokens = info != null ? info.numTokens : 0;
- int numOverlapTokens = info != null ? info.numOverlapTokens : 0;
- float boost = info != null ? info.getBoost() : 1.0f;
- FieldInvertState invertState = new FieldInvertState(field, 0, numTokens, numOverlapTokens, 0, boost);
- long value = normSimilarity.computeNorm(invertState);
- if (DEBUG) System.err.println("MemoryIndexReader.norms: " + field + ":" + value + ":" + numTokens);
- return new MemoryIndexNormDocValues(value);
- }
-
- private void calculateNormValues() {
- norms = new HashMap<>();
- for (String field : fieldInfos.keySet()) {
- norms.put(field, calculateFieldNormValue(field));
- }
- }
-
/**
* Resets the {@link MemoryIndex} to its initial state and recycles all internal buffers.
*/
public void reset() {
- this.fieldInfos.clear();
- this.fields.clear();
- this.sortedFields = null;
- this.norms = null;
+ fields.clear();
this.normSimilarity = IndexSearcher.getDefaultSimilarity();
byteBlockPool.reset(false, false); // no need to 0-fill the buffers
intBlockPool.reset(true, false); // here must must 0-fill since we use slices
Modified: lucene/dev/branches/lucene2878/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java (original)
+++ lucene/dev/branches/lucene2878/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java Fri Dec 5 15:24:11 2014
@@ -100,7 +100,7 @@ public class TestMemoryIndex extends Luc
LeafReader reader = (LeafReader) searcher.getIndexReader();
float n1 = reader.getNormValues("f1").get(0);
- // Norms aren't cached, so we can change the Similarity
+ // Norms are re-computed when we change the Similarity
mi.setSimilarity(new DefaultSimilarity() {
@Override
public float lengthNorm(FieldInvertState state) {
Modified: lucene/dev/branches/lucene2878/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java (original)
+++ lucene/dev/branches/lucene2878/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java Fri Dec 5 15:24:11 2014
@@ -24,6 +24,7 @@ import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -137,7 +138,7 @@ public class IndexSplitter {
SegmentInfo info = infoPerCommit.info;
// Same info just changing the dir:
SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.name, info.getDocCount(),
- info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics(), info.getId());
+ info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics(), info.getId(), new HashMap<>());
destInfos.add(new SegmentCommitInfo(newInfo, infoPerCommit.getDelCount(),
infoPerCommit.getDelGen(), infoPerCommit.getFieldInfosGen(),
infoPerCommit.getDocValuesGen()));
Modified: lucene/dev/branches/lucene2878/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java (original)
+++ lucene/dev/branches/lucene2878/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java Fri Dec 5 15:24:11 2014
@@ -166,6 +166,7 @@ public class NativeUnixDirectory extends
private boolean isOpen;
public NativeUnixIndexOutput(Path path, int bufferSize) throws IOException {
+ super("NativeUnixIndexOutput(path=\"" + path.toString() + "\")");
//this.path = path;
final FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), false);
fos = new FileOutputStream(fd);
Modified: lucene/dev/branches/lucene2878/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java (original)
+++ lucene/dev/branches/lucene2878/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java Fri Dec 5 15:24:11 2014
@@ -33,10 +33,12 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* Constructs a filter for docs matching any of the terms added to this class.
@@ -45,7 +47,9 @@ import org.apache.lucene.util.BytesRef;
* a choice of "category" labels picked by the end user. As a filter, this is much faster than the
* equivalent query (a BooleanQuery with many "should" TermQueries)
*/
-public final class TermsFilter extends Filter {
+public final class TermsFilter extends Filter implements Accountable {
+
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(TermsFilter.class);
/*
* this class is often used for large number of terms in a single field.
@@ -178,7 +182,14 @@ public final class TermsFilter extends F
this.hashCode = hash;
}
-
+
+ @Override
+ public long ramBytesUsed() {
+ return BASE_RAM_BYTES_USED
+ + RamUsageEstimator.sizeOf(termsAndFields)
+ + RamUsageEstimator.sizeOf(termsBytes)
+ + RamUsageEstimator.sizeOf(offsets);
+ }
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
@@ -254,7 +265,13 @@ public final class TermsFilter extends F
return builder.toString();
}
- private static final class TermsAndField {
+ private static final class TermsAndField implements Accountable {
+
+ private static final long BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(TermsAndField.class)
+ + RamUsageEstimator.shallowSizeOfInstance(String.class)
+ + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; // header of the array held by the String
+
final int start;
final int end;
final String field;
@@ -268,6 +285,13 @@ public final class TermsFilter extends F
}
@Override
+ public long ramBytesUsed() {
+ // this is an approximation since we don't actually know how strings store
+ // their data, which can be JVM-dependent
+ return BASE_RAM_BYTES_USED + field.length() * RamUsageEstimator.NUM_BYTES_CHAR;
+ }
+
+ @Override
public int hashCode() {
final int prime = 31;
int result = 1;
@@ -317,4 +341,5 @@ public final class TermsFilter extends F
Collections.sort(toSort);
return toSort;
}
+
}
Modified: lucene/dev/branches/lucene2878/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java (original)
+++ lucene/dev/branches/lucene2878/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java Fri Dec 5 15:24:11 2014
@@ -46,12 +46,14 @@ import org.apache.lucene.search.ScoreDoc
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BitDocIdSet;
-import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.RamUsageTester;
import org.apache.lucene.util.TestUtil;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
public class TermsFilterTest extends LuceneTestCase {
public void testCachability() throws Exception {
@@ -336,4 +338,18 @@ public class TermsFilterTest extends Luc
new Term("field1", "c"));
assertEquals("field1:a field1:b field1:c", termsFilter.toString());
}
+
+ public void testRamBytesUsed() {
+ List<Term> terms = new ArrayList<>();
+ final int numTerms = 1000 + random().nextInt(1000);
+ for (int i = 0; i < numTerms; ++i) {
+ terms.add(new Term("f", RandomStrings.randomUnicodeOfLength(random(), 10)));
+ }
+ TermsFilter filter = new TermsFilter(terms);
+ final long actualRamBytesUsed = RamUsageTester.sizeOf(filter);
+ final long expectedRamBytesUsed = filter.ramBytesUsed();
+ // error margin within 1%
+ assertEquals(actualRamBytesUsed, expectedRamBytesUsed, actualRamBytesUsed / 100);
+ }
+
}
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java Fri Dec 5 15:24:11 2014
@@ -36,16 +36,16 @@ public abstract class CompressingCodec e
/**
* Create a random instance.
*/
- public static CompressingCodec randomInstance(Random random, int chunkSize, boolean withSegmentSuffix) {
+ public static CompressingCodec randomInstance(Random random, int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix) {
switch (random.nextInt(4)) {
case 0:
- return new FastCompressingCodec(chunkSize, withSegmentSuffix);
+ return new FastCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix);
case 1:
- return new FastDecompressionCompressingCodec(chunkSize, withSegmentSuffix);
+ return new FastDecompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix);
case 2:
- return new HighCompressionCompressingCodec(chunkSize, withSegmentSuffix);
+ return new HighCompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix);
case 3:
- return new DummyCompressingCodec(chunkSize, withSegmentSuffix);
+ return new DummyCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix);
default:
throw new AssertionError();
}
@@ -56,14 +56,14 @@ public abstract class CompressingCodec e
* suffix
*/
public static CompressingCodec randomInstance(Random random) {
- return randomInstance(random, RandomInts.randomIntBetween(random, 1, 500), false);
+ return randomInstance(random, RandomInts.randomIntBetween(random, 1, 1 << 15), RandomInts.randomIntBetween(random, 64, 1024), false);
}
/**
* Creates a random {@link CompressingCodec} that is using a segment suffix
*/
public static CompressingCodec randomInstance(Random random, boolean withSegmentSuffix) {
- return randomInstance(random, RandomInts.randomIntBetween(random, 1, 500), withSegmentSuffix);
+ return randomInstance(random, RandomInts.randomIntBetween(random, 1, 1 << 15), RandomInts.randomIntBetween(random, 64, 1024), withSegmentSuffix);
}
private final CompressingStoredFieldsFormat storedFieldsFormat;
@@ -72,17 +72,17 @@ public abstract class CompressingCodec e
/**
* Creates a compressing codec with a given segment suffix
*/
- public CompressingCodec(String name, String segmentSuffix, CompressionMode compressionMode, int chunkSize) {
+ public CompressingCodec(String name, String segmentSuffix, CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk) {
super(name, TestUtil.getDefaultCodec());
- this.storedFieldsFormat = new CompressingStoredFieldsFormat(name, segmentSuffix, compressionMode, chunkSize);
+ this.storedFieldsFormat = new CompressingStoredFieldsFormat(name, segmentSuffix, compressionMode, chunkSize, maxDocsPerChunk);
this.termVectorsFormat = new CompressingTermVectorsFormat(name, segmentSuffix, compressionMode, chunkSize);
}
/**
* Creates a compressing codec with an empty segment suffix
*/
- public CompressingCodec(String name, CompressionMode compressionMode, int chunkSize) {
- this(name, "", compressionMode, chunkSize);
+ public CompressingCodec(String name, CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk) {
+ this(name, "", compressionMode, chunkSize, maxDocsPerChunk);
}
@Override
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java Fri Dec 5 15:24:11 2014
@@ -21,14 +21,14 @@ package org.apache.lucene.codecs.compres
public class FastCompressingCodec extends CompressingCodec {
/** Constructor that allows to configure the chunk size. */
- public FastCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+ public FastCompressingCodec(int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix) {
super("FastCompressingStoredFields",
withSegmentSuffix ? "FastCompressingStoredFields" : "",
- CompressionMode.FAST, chunkSize);
+ CompressionMode.FAST, chunkSize, maxDocsPerChunk);
}
/** Default constructor. */
public FastCompressingCodec() {
- this(1 << 14, false);
+ this(1 << 14, 128, false);
}
}
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java Fri Dec 5 15:24:11 2014
@@ -21,14 +21,14 @@ package org.apache.lucene.codecs.compres
public class FastDecompressionCompressingCodec extends CompressingCodec {
/** Constructor that allows to configure the chunk size. */
- public FastDecompressionCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+ public FastDecompressionCompressingCodec(int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix) {
super("FastDecompressionCompressingStoredFields",
withSegmentSuffix ? "FastDecompressionCompressingStoredFields" : "",
- CompressionMode.FAST_DECOMPRESSION, chunkSize);
+ CompressionMode.FAST_DECOMPRESSION, chunkSize, maxDocsPerChunk);
}
/** Default constructor. */
public FastDecompressionCompressingCodec() {
- this(1 << 14, false);
+ this(1 << 14, 256, false);
}
}
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java Fri Dec 5 15:24:11 2014
@@ -21,14 +21,16 @@ package org.apache.lucene.codecs.compres
public class HighCompressionCompressingCodec extends CompressingCodec {
/** Constructor that allows to configure the chunk size. */
- public HighCompressionCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+ public HighCompressionCompressingCodec(int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix) {
super("HighCompressionCompressingStoredFields",
withSegmentSuffix ? "HighCompressionCompressingStoredFields" : "",
- CompressionMode.HIGH_COMPRESSION, chunkSize);
+ CompressionMode.HIGH_COMPRESSION, chunkSize, maxDocsPerChunk);
}
/** Default constructor. */
public HighCompressionCompressingCodec() {
- this(1 << 14, false);
+ // no need to have a higher block length than 32KB since deflate splits
+ // into blocks of 32KB anyway, and this is a lower bound (try to avoid > 32KB)
+ this(24576, 512, false);
}
}
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java Fri Dec 5 15:24:11 2014
@@ -83,15 +83,15 @@ public class DummyCompressingCodec exten
};
/** Constructor that allows to configure the chunk size. */
- public DummyCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+ public DummyCompressingCodec(int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix) {
super("DummyCompressingStoredFields",
withSegmentSuffix ? "DummyCompressingStoredFields" : "",
- DUMMY, chunkSize);
+ DUMMY, chunkSize, maxDocsPerChunk);
}
/** Default constructor. */
public DummyCompressingCodec() {
- this(1 << 14, false);
+ this(1 << 14, 128, false);
}
}
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java Fri Dec 5 15:24:11 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import org.apache.lucene.codecs.Codec;
@@ -626,7 +627,7 @@ public abstract class BaseCompoundFormat
/** Returns a new fake segment */
protected static SegmentInfo newSegmentInfo(Directory dir, String name) {
- return new SegmentInfo(dir, Version.LATEST, name, 10000, false, Codec.getDefault(), null, StringHelper.randomId());
+ return new SegmentInfo(dir, Version.LATEST, name, 10000, false, Codec.getDefault(), null, StringHelper.randomId(), new HashMap<>());
}
/** Creates a file of the specified size with random data. */
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java Fri Dec 5 15:24:11 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
@@ -175,7 +176,7 @@ public abstract class BaseFieldInfoForma
/** Returns a new fake segment */
protected static SegmentInfo newSegmentInfo(Directory dir, String name) {
- return new SegmentInfo(dir, Version.LATEST, name, 10000, false, Codec.getDefault(), null, StringHelper.randomId());
+ return new SegmentInfo(dir, Version.LATEST, name, 10000, false, Codec.getDefault(), null, StringHelper.randomId(), new HashMap<>());
}
@Override
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java Fri Dec 5 15:24:11 2014
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -686,7 +687,7 @@ public abstract class BasePostingsFormat
// randomly index at lower IndexOption
private FieldsProducer buildIndex(Directory dir, IndexOptions maxAllowed, boolean allowPayloads, boolean alwaysTestMax) throws IOException {
Codec codec = getCodec();
- SegmentInfo segmentInfo = new SegmentInfo(dir, Version.LATEST, "_0", maxDoc, false, codec, null, StringHelper.randomId());
+ SegmentInfo segmentInfo = new SegmentInfo(dir, Version.LATEST, "_0", maxDoc, false, codec, null, StringHelper.randomId(), new HashMap<>());
int maxIndexOption = Arrays.asList(IndexOptions.values()).indexOf(maxAllowed);
if (VERBOSE) {
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java Fri Dec 5 15:24:11 2014
@@ -49,7 +49,7 @@ public abstract class BaseSegmentInfoFor
Codec codec = getCodec();
byte id[] = StringHelper.randomId();
SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec,
- Collections.<String,String>emptyMap(), id);
+ Collections.<String,String>emptyMap(), id, new HashMap<>());
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
@@ -63,7 +63,7 @@ public abstract class BaseSegmentInfoFor
Codec codec = getCodec();
byte id[] = StringHelper.randomId();
SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec,
- Collections.<String,String>emptyMap(), id);
+ Collections.<String,String>emptyMap(), id, new HashMap<>());
Set<String> originalFiles = Collections.singleton("_123.a");
info.setFiles(originalFiles);
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
@@ -86,7 +86,7 @@ public abstract class BaseSegmentInfoFor
diagnostics.put("key1", "value1");
diagnostics.put("key2", "value2");
SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec,
- diagnostics, id);
+ diagnostics, id, new HashMap<>());
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
@@ -94,13 +94,30 @@ public abstract class BaseSegmentInfoFor
dir.close();
}
+ /** Test attributes map */
+ public void testAttributes() throws Exception {
+ Directory dir = newDirectory();
+ Codec codec = getCodec();
+ byte id[] = StringHelper.randomId();
+ Map<String,String> attributes = new HashMap<>();
+ attributes.put("key1", "value1");
+ attributes.put("key2", "value2");
+ SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec,
+ Collections.emptyMap(), id, attributes);
+ info.setFiles(Collections.<String>emptySet());
+ codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
+ SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
+ assertEquals(attributes, info2.getAttributes());
+ dir.close();
+ }
+
/** Test unique ID */
public void testUniqueID() throws Exception {
Codec codec = getCodec();
Directory dir = newDirectory();
byte id[] = StringHelper.randomId();
SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec,
- Collections.<String,String>emptyMap(), id);
+ Collections.<String,String>emptyMap(), id, new HashMap<>());
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
@@ -115,7 +132,7 @@ public abstract class BaseSegmentInfoFor
Directory dir = newDirectory();
byte id[] = StringHelper.randomId();
SegmentInfo info = new SegmentInfo(dir, v, "_123", 1, false, codec,
- Collections.<String,String>emptyMap(), id);
+ Collections.<String,String>emptyMap(), id, new HashMap<>());
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
@@ -153,7 +170,14 @@ public abstract class BaseSegmentInfoFor
byte id[] = new byte[StringHelper.ID_LENGTH];
random().nextBytes(id);
- SegmentInfo info = new SegmentInfo(dir, version, name, docCount, isCompoundFile, codec, diagnostics, id);
+ Map<String,String> attributes = new HashMap<>();
+ int numAttributes = random().nextInt(10);
+ for (int j = 0; j < numAttributes; j++) {
+ attributes.put(TestUtil.randomUnicodeString(random()),
+ TestUtil.randomUnicodeString(random()));
+ }
+
+ SegmentInfo info = new SegmentInfo(dir, version, name, docCount, isCompoundFile, codec, diagnostics, id, attributes);
info.setFiles(files);
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
SegmentInfo info2 = codec.segmentInfoFormat().read(dir, name, id, IOContext.DEFAULT);
@@ -174,6 +198,7 @@ public abstract class BaseSegmentInfoFor
assertIDEquals(expected.getId(), actual.getId());
assertEquals(expected.getUseCompoundFile(), actual.getUseCompoundFile());
assertEquals(expected.getVersion(), actual.getVersion());
+ assertEquals(expected.getAttributes(), actual.getAttributes());
}
/** Returns the versions this SI should test */
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/mockfile/VerboseFS.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/mockfile/VerboseFS.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/mockfile/VerboseFS.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/mockfile/VerboseFS.java Fri Dec 5 15:24:11 2014
@@ -167,6 +167,8 @@ public class VerboseFS extends FilterFil
} finally {
if (containsDestructive(options)) {
sop("newFileChannel" + options + ": " + path(path), exception);
+ } else {
+ IOUtils.reThrow(exception);
}
}
throw new AssertionError();
@@ -182,6 +184,8 @@ public class VerboseFS extends FilterFil
} finally {
if (containsDestructive(options)) {
sop("newAsynchronousFileChannel" + options + ": " + path(path), exception);
+ } else {
+ IOUtils.reThrow(exception);
}
}
throw new AssertionError();
@@ -197,6 +201,8 @@ public class VerboseFS extends FilterFil
} finally {
if (containsDestructive(options)) {
sop("newByteChannel" + options + ": " + path(path), exception);
+ } else {
+ IOUtils.reThrow(exception);
}
}
throw new AssertionError();
@@ -271,6 +277,8 @@ public class VerboseFS extends FilterFil
} finally {
if (containsDestructive(options)) {
sop("newByteChannel[SECURE]" + options + ": " + path(path), exception);
+ } else {
+ IOUtils.reThrow(exception);
}
}
throw new AssertionError();
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java Fri Dec 5 15:24:11 2014
@@ -1031,5 +1031,14 @@ public abstract class BaseDirectoryTestC
}
dir.close();
}
+
+ // LUCENE-6084
+ public void testIndexOutputToString() throws Throwable {
+ Directory dir = getDirectory(createTempDir());
+ IndexOutput out = dir.createOutput("camelCase.txt", newIOContext(random()));
+ assertTrue(out.toString(), out.toString().contains("camelCase.txt"));
+ out.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java Fri Dec 5 15:24:11 2014
@@ -38,6 +38,7 @@ public class MockIndexOutputWrapper exte
/** Construct an empty output buffer. */
public MockIndexOutputWrapper(MockDirectoryWrapper dir, IndexOutput delegate, String name) {
+ super("MockIndexOutputWrapper(" + delegate + ")");
this.dir = dir;
this.name = name;
this.delegate = delegate;
Modified: lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java (original)
+++ lucene/dev/branches/lucene2878/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java Fri Dec 5 15:24:11 2014
@@ -59,6 +59,7 @@ public class ThrottledIndexOutput extend
public ThrottledIndexOutput(int bytesPerSecond, long flushDelayMillis,
long closeDelayMillis, long seekDelayMillis, long minBytesWritten,
IndexOutput delegate) {
+ super("ThrottledIndexOutput(" + delegate + ")");
assert bytesPerSecond > 0;
this.delegate = delegate;
this.bytesPerSecond = bytesPerSecond;
Modified: lucene/dev/branches/lucene2878/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/CHANGES.txt?rev=1643326&r1=1643325&r2=1643326&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene2878/solr/CHANGES.txt Fri Dec 5 15:24:11 2014
@@ -219,6 +219,9 @@ New Features
* SOLR-6607: Managing requesthandlers throuh API (Noble Paul)
+* SOLR-4799: faster join using join="zipper" aka merge join for nested DIH EntityProcessors
+ (Mikhail Khludnev via Noble Paul)
+
Bug Fixes
----------------------
@@ -276,14 +279,6 @@ Bug Fixes
* SOLR-6631: DistributedQueue spinning on calling zookeeper getChildren()
(Jessica Cheng Mallet, Mark Miller, Timothy Potter)
-* SOLR-2927: Solr does not unregister all mbeans upon exception in constructor
- causing memory leaks. (tom liu, Sharath Babu, Cyrille Roy, shalin)
-
-* SOLR-6685: ConcurrentModificationException in Overseer Status API. (shalin)
-
-* SOLR-6706: /update/json/docs throws RuntimeException if a nested structure
- contains a non-leaf float field (Noble Paul, shalin)
-
* SOLR-6579:SnapPuller Replication blocks clean shutdown of tomcat
(Philip Black-Knight via Noble Paul)
@@ -294,13 +289,6 @@ Bug Fixes
it caused long startup times on lage indexes even when it wasn't used.
(yonik)
-* SOLR-6662: better validation when parsing command-line options that expect a value
- (Timothy Potter)
-
-* SOLR-6705: Better strategy for dealing with JVM specific options in the start
- scripts; remove -XX:+AggressiveOpts and only set -XX:-UseSuperWord for Java 1.7u40
- to u51. (Uwe Schindler, janhoy, hossman, Timothy Potter)
-
* SOLR-6738: Admin UI - Escape Data on Plugins-View (steffkes)
* SOLR-3774: Solr adds RequestHandler SolrInfoMBeans twice to the JMX server.
@@ -409,10 +397,6 @@ Other Changes
* SOLR-6651: Fix wrong timeout logged in waitForReplicasToComeUp. (shalin)
-* SOLR-6661: Adjust all example configurations to allow overriding error-prone
- relative paths for solrconfig.xml <lib> references with solr.install.dir
- system property; bin/solr scripts will set it appropriately. (ehatcher)
-
* SOLR-6698: Solr is not consistent wrt ZkCredentialsProvider / ZkCredentialProvider.
References to zkCredentialProvider in System properties or configurations should be
changed to zkCredentialsProvider. (Gregory Chanan)
@@ -487,9 +471,67 @@ Bug Fixes
* SOLR-6510: The collapse QParser would throw a NPE when used on a DocValues field on
an empty segment/index. (Christine Poerschke, David Smiley)
+* SOLR-6780: Fixed a bug in how default/appends/invariants params were affecting the set
+ of all "keys" found in the request parameters, resulting in some key=value param pairs
+ being duplicated. This was noticably affecting some areas of the code where iteration
+ was done over the set of all params:
+ * literal.* in ExtractingRequestHandler
+ * facet.* in FacetComponent
+ * spellcheck.[dictionary name].* and spellcheck.collateParam.* in SpellCheckComponent
+ * olap.* in AnalyticsComponent
+ (Alexandre Rafalovitch & hossman)
+
+* SOLR-2927: Solr does not unregister all mbeans upon exception in constructor
+ causing memory leaks. (tom liu, Sharath Babu, Cyrille Roy, shalin)
+
+* SOLR-6685: ConcurrentModificationException in Overseer Status API. (shalin)
+
+* SOLR-6706: /update/json/docs throws RuntimeException if a nested structure
+ contains a non-leaf float field (Noble Paul, shalin)
+
+* SOLR-6610: Slow startup of new clusters because ZkController.publishAndWaitForDownStates
+ always times out. (Jessica Cheng Mallet, shalin, Noble Paul)
+
+* SOLR-6662: better validation when parsing command-line options that expect a value
+ (Timothy Potter)
+
+* SOLR-6732: Fix handling of leader-initiated recovery state was String in older versions
+ and is now a JSON map, caused backwards compatibility issues when doing rolling upgrades of
+ a live cluster while indexing (Timothy Potter)
+
+* SOLR-6705: Better strategy for dealing with JVM specific options in the start
+ scripts; remove -XX:+AggressiveOpts and only set -XX:-UseSuperWord for Java 1.7u40
+ to u51. (Uwe Schindler, janhoy, hossman, Timothy Potter)
+
+* SOLR-6726: better strategy for selecting the JMX RMI port based on SOLR_PORT in bin/solr
+ script (Timothy Potter)
+
+* SOLR-6795: distrib.singlePass returns score even though not asked for.
+ (Per Steffensen via shalin)
+
+* SOLR-6796: distrib.singlePass does not return correct set of fields for multi-fl-parameter
+ requests. (Per Steffensen via shalin)
+
+* SOLR-6776: Transaction log was not flushed at the end of update requests with softCommit
+ specified, which could lead to data loss if the server were killed immediately after the
+ update finished. (Jeffery Yuan via yonik)
+
+Other Changes
+----------------------
+
+* SOLR-6661: Adjust all example configurations to allow overriding error-prone
+ relative paths for solrconfig.xml <lib> references with solr.install.dir
+ system property; bin/solr scripts will set it appropriately. (ehatcher)
+
+* SOLR-6694: Auto-detect JAVA_HOME using the Windows registry if it is not set
+ (janhoy, Timothy Potter)
+
+* SOLR-6653: bin/solr script should return error code >0 when something fails
+ (janhoy, Timothy Potter)
+
================== 4.10.2 ==================
-Bug Fixes
+Bug FixesAnalyticsComponent
----------------------
* SOLR-6509: Solr start scripts interactive mode doesn't honor -z argument (Timothy Potter)