You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/08/13 15:53:27 UTC
svn commit: r1372423 [35/45] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/lucene/
dev-tools/maven/lucene/analysis/common/
dev-tools/maven/lucene/analysis/icu/ d...
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java Mon Aug 13 13:52:46 2012
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@@ -189,6 +190,41 @@ public class TestFuzzyQuery extends Luce
directory.close();
}
+ public void test2() throws Exception {
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
+ addDoc("LANGE", writer);
+ addDoc("LUETH", writer);
+ addDoc("PIRSING", writer);
+ addDoc("RIEGEL", writer);
+ addDoc("TRZECZIAK", writer);
+ addDoc("WALKER", writer);
+ addDoc("WBR", writer);
+ addDoc("WE", writer);
+ addDoc("WEB", writer);
+ addDoc("WEBE", writer);
+ addDoc("WEBER", writer);
+ addDoc("WEBERE", writer);
+ addDoc("WEBREE", writer);
+ addDoc("WEBEREI", writer);
+ addDoc("WBRE", writer);
+ addDoc("WITTKOPF", writer);
+ addDoc("WOJNAROWSKI", writer);
+ addDoc("WRICKE", writer);
+
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ writer.close();
+
+ FuzzyQuery query = new FuzzyQuery(new Term("field", "WEBER"), 2, 1);
+ //query.setRewriteMethod(FuzzyQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(8, hits.length);
+
+ reader.close();
+ directory.close();
+ }
+
/**
* MultiTermQuery provides (via attribute) information about which values
* must be competitive to enter the priority queue.
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestMultiThreadTermVectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestMultiThreadTermVectors.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestMultiThreadTermVectors.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestMultiThreadTermVectors.java Mon Aug 13 13:52:46 2012
@@ -23,7 +23,6 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.*;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Terms;
@@ -179,9 +178,8 @@ class MultiThreadTermVectorsReader imple
}
private void verifyVectors(Fields vectors, int num) throws IOException {
- FieldsEnum fieldsEnum = vectors.iterator();
- while(fieldsEnum.next() != null) {
- Terms terms = fieldsEnum.terms();
+ for (String field : vectors) {
+ Terms terms = vectors.terms(field);
assert terms != null;
verifyVector(terms.iterator(null), num);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestNRTManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestNRTManager.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestNRTManager.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestNRTManager.java Mon Aug 13 13:52:46 2012
@@ -41,7 +41,7 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.ThreadInterruptedException;
-@SuppressCodecs({ "SimpleText", "Memory" })
+@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase {
private final ThreadLocal<Long> lastGens = new ThreadLocal<Long>();
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java Mon Aug 13 13:52:46 2012
@@ -103,8 +103,7 @@ public class TestPositionIncrement exten
DocsAndPositionsEnum pos = MultiFields.getTermPositionsEnum(searcher.getIndexReader(),
MultiFields.getLiveDocs(searcher.getIndexReader()),
"field",
- new BytesRef("1"),
- false);
+ new BytesRef("1"));
pos.nextDoc();
// first token should be at position 0
assertEquals(0, pos.nextPosition());
@@ -112,8 +111,7 @@ public class TestPositionIncrement exten
pos = MultiFields.getTermPositionsEnum(searcher.getIndexReader(),
MultiFields.getLiveDocs(searcher.getIndexReader()),
"field",
- new BytesRef("2"),
- false);
+ new BytesRef("2"));
pos.nextDoc();
// second token should be at position 2
assertEquals(2, pos.nextPosition());
@@ -208,7 +206,7 @@ public class TestPositionIncrement exten
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockPayloadAnalyzer());
Document doc = new Document();
doc.add(new TextField("content", new StringReader(
- "a a b c d e a f g h i j a b k k"), Field.Store.NO));
+ "a a b c d e a f g h i j a b k k")));
writer.addDocument(doc);
final IndexReader readerFromWriter = writer.getReader();
@@ -216,8 +214,7 @@ public class TestPositionIncrement exten
DocsAndPositionsEnum tp = r.termPositionsEnum(r.getLiveDocs(),
"content",
- new BytesRef("a"),
- false);
+ new BytesRef("a"));
int count = 0;
assertTrue(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java Mon Aug 13 13:52:46 2012
@@ -29,7 +29,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.LuceneTestCase;
-@SuppressCodecs({ "SimpleText", "Memory" })
+@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
public class TestSearchWithThreads extends LuceneTestCase {
int NUM_DOCS;
final int NUM_SEARCH_THREADS = 5;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java Mon Aug 13 13:52:46 2012
@@ -43,7 +43,7 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util._TestUtil;
-@SuppressCodecs({ "SimpleText", "Memory" })
+@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
boolean warmCalled;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java Mon Aug 13 13:52:46 2012
@@ -41,7 +41,7 @@ import org.apache.lucene.util._TestUtil;
// - test pulling docs in 2nd round trip...
// - filter too
-@SuppressCodecs({ "SimpleText", "Memory" })
+@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
public class TestShardSearching extends ShardSearchingTestBase {
private static class PreviousSearchState {
@@ -77,8 +77,7 @@ public class TestShardSearching extends
System.out.println("TEST: numNodes=" + numNodes + " runTimeSec=" + runTimeSec + " maxSearcherAgeSeconds=" + maxSearcherAgeSeconds);
}
- start(_TestUtil.getTempDir("TestShardSearching").toString(),
- numNodes,
+ start(numNodes,
runTimeSec,
maxSearcherAgeSeconds
);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSort.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSort.java Mon Aug 13 13:52:46 2012
@@ -59,6 +59,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.DocIdBitSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util._TestUtil;
import org.junit.BeforeClass;
@@ -196,7 +197,7 @@ public class TestSort extends LuceneTest
if (data[i][11] != null) doc.add (new StringField ("parser", data[i][11], Field.Store.NO));
for(IndexableField f : doc.getFields()) {
- if (!f.fieldType().omitNorms()) {
+ if (f.fieldType().indexed() && !f.fieldType().omitNorms()) {
((Field) f).setBoost(2.0f);
}
}
@@ -239,7 +240,7 @@ public class TestSort extends LuceneTest
doc.add(new SortedBytesDocValuesField("string2", new BytesRef(num2)));
doc.add (new Field ("tracer2", num2, onlyStored));
for(IndexableField f2 : doc.getFields()) {
- if (!f2.fieldType().omitNorms()) {
+ if (f2.fieldType().indexed() && !f2.fieldType().omitNorms()) {
((Field) f2).setBoost(2.0f);
}
}
@@ -255,7 +256,7 @@ public class TestSort extends LuceneTest
doc.add (new Field ("tracer2_fixed", num2Fixed, onlyStored));
for(IndexableField f2 : doc.getFields()) {
- if (!f2.fieldType().omitNorms()) {
+ if (f2.fieldType().indexed() && !f2.fieldType().omitNorms()) {
((Field) f2).setBoost(2.0f);
}
}
@@ -810,7 +811,7 @@ public class TestSort extends LuceneTest
assertMatches (full, queryG, sort, "ZYXW");
// Do the same for a ParallelMultiSearcher
- ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8));
+ ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8), new NamedThreadFactory("testEmptyFieldSort"));
IndexSearcher parallelSearcher=new IndexSearcher (full.getIndexReader(), exec);
sort.setSort (new SortField ("int", SortField.Type.INT),
@@ -852,7 +853,7 @@ public class TestSort extends LuceneTest
// test a variety of sorts using a parallel multisearcher
public void testParallelMultiSort() throws Exception {
- ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8));
+ ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8), new NamedThreadFactory("testParallelMultiSort"));
IndexSearcher searcher = new IndexSearcher(
new MultiReader(searchX.getIndexReader(),
searchY.getIndexReader()), exec);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java Mon Aug 13 13:52:46 2012
@@ -145,7 +145,6 @@ public class TestSubScorerFreqs extends
}
}
- @SuppressWarnings("unchecked")
@Test
public void testBooleanQuery() throws Exception {
TermQuery aQuery = new TermQuery(new Term("f", "a"));
@@ -161,11 +160,15 @@ public class TestSubScorerFreqs extends
query.add(inner, Occur.MUST);
query.add(aQuery, Occur.MUST);
query.add(dQuery, Occur.MUST);
- Set<String>[] occurList = new Set[] {
+
+ // Only needed in Java6; Java7+ has a @SafeVarargs annotated Arrays#asList()!
+ // see http://docs.oracle.com/javase/7/docs/api/java/lang/SafeVarargs.html
+ @SuppressWarnings("unchecked") final Iterable<Set<String>> occurList = Arrays.asList(
Collections.singleton("MUST"),
new HashSet<String>(Arrays.asList("MUST", "SHOULD"))
- };
- for (Set<String> occur : occurList) {
+ );
+
+ for (final Set<String> occur : occurList) {
CountingCollector c = new CountingCollector(TopScoreDocCollector.create(
10, true), occur);
s.search(query, null, c);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java Mon Aug 13 13:52:46 2012
@@ -23,7 +23,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.*;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.search.positions.IntervalIterator;
import org.apache.lucene.search.positions.Interval;
import org.apache.lucene.store.Directory;
@@ -63,11 +63,11 @@ public class TestTermQuery extends Lucen
TermsEnum te = MultiFields.getTerms(reader,
fieldName).iterator(null);
te.seekExact(new BytesRef("1"), false);
- DocsAndPositionsEnum docsAndPositions = te.docsAndPositions(null, null, false);
+ DocsAndPositionsEnum docsAndPositions = te.docsAndPositions(null, null, DocsAndPositionsEnum.FLAG_PAYLOADS);
assertEquals(39, reader.docFreq(new Term(fieldName, "1")));
docsAndPositions.nextDoc();
docsAndPositions.nextPosition();
- boolean payloadsIndexed = docsAndPositions.hasPayload();
+ boolean payloadsIndexed = false; // TODO we should enable payloads here
IndexSearcher searcher = new IndexSearcher(reader);
writer.close();
@@ -78,7 +78,7 @@ public class TestTermQuery extends Lucen
List<AtomicReaderContext> leaves = topReaderContext.leaves();
Weight weight = one.createWeight(searcher);
for (AtomicReaderContext atomicReaderContext : leaves) {
- Scorer scorer = weight.scorer(atomicReaderContext, true, true, FeatureFlags.POSITIONS, null);
+ Scorer scorer = weight.scorer(atomicReaderContext, true, true, PostingFeatures.POSITIONS, null);
assertNotNull(scorer);
int toDoc = 1 + random().nextInt(atomicReaderContext.reader().docFreq(new Term(fieldName, "1")) - 1 );
final int advance = scorer.advance(toDoc);
@@ -126,18 +126,9 @@ public class TestTermQuery extends Lucen
public final void checkPayload(int pos, Interval interval,
boolean payloadsIndexed) throws IOException {
if (payloadsIndexed) {
- boolean wasPayloadAvailable = interval.payloadAvailable();
- BytesRef bytes = new BytesRef();
- assertTrue(interval.nextPayload(bytes));
- assertFalse(interval.payloadAvailable());
- if (!wasPayloadAvailable) {
- // if payload has 0 length interval or rather docs&pos enum will treat is as not existing
- assertEquals(0, bytes.length);
- } else {
- assertTrue(bytes.length > 0);
- }
+ assertNotNull(interval.nextPayload());
} else {
- assertFalse(interval.payloadAvailable());
+ assertNull(interval.nextPayload());
}
}
@@ -182,7 +173,7 @@ public class TestTermQuery extends Lucen
List<AtomicReaderContext> leaves = topReaderContext.leaves();
Weight weight = one.createWeight(searcher);
for (AtomicReaderContext atomicReaderContext : leaves) {
- Scorer scorer = weight.scorer(atomicReaderContext, true, true, FeatureFlags.POSITIONS, null);
+ Scorer scorer = weight.scorer(atomicReaderContext, true, true, PostingFeatures.POSITIONS, null);
assertNotNull(scorer);
int initDoc = 0;
int maxDoc = atomicReaderContext.reader().maxDoc();
@@ -268,7 +259,7 @@ public class TestTermQuery extends Lucen
Weight weight = one.createWeight(searcher);
Interval interval = null;
for (AtomicReaderContext atomicReaderContext : leaves) {
- Scorer scorer = weight.scorer(atomicReaderContext, true, true, FeatureFlags.POSITIONS, null);
+ Scorer scorer = weight.scorer(atomicReaderContext, true, true, PostingFeatures.POSITIONS, null);
assertNotNull(scorer);
int initDoc = 0;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java Mon Aug 13 13:52:46 2012
@@ -29,7 +29,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -79,7 +79,7 @@ public class TestTermScorer extends Luce
Weight weight = indexSearcher.createNormalizedWeight(termQuery);
assertTrue(indexSearcher.getTopReaderContext() instanceof AtomicReaderContext);
AtomicReaderContext context = (AtomicReaderContext)indexSearcher.getTopReaderContext();
- Scorer ts = weight.scorer(context, true, true, FeatureFlags.DOCS, context.reader().getLiveDocs());
+ Scorer ts = weight.scorer(context, true, true, PostingFeatures.DOCS_AND_FREQS, context.reader().getLiveDocs());
// we have 2 documents with the term all in them, one document for all the
// other values
final List<TestHit> docs = new ArrayList<TestHit>();
@@ -141,7 +141,7 @@ public class TestTermScorer extends Luce
Weight weight = indexSearcher.createNormalizedWeight(termQuery);
assertTrue(indexSearcher.getTopReaderContext() instanceof AtomicReaderContext);
AtomicReaderContext context = (AtomicReaderContext) indexSearcher.getTopReaderContext();
- Scorer ts = weight.scorer(context, true, true, FeatureFlags.DOCS, context.reader().getLiveDocs());
+ Scorer ts = weight.scorer(context, true, true, PostingFeatures.DOCS_AND_FREQS, context.reader().getLiveDocs());
assertTrue("next did not return a doc",
ts.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertTrue("score is not correct", ts.score() == 1.6931472f);
@@ -160,7 +160,7 @@ public class TestTermScorer extends Luce
Weight weight = indexSearcher.createNormalizedWeight(termQuery);
assertTrue(indexSearcher.getTopReaderContext() instanceof AtomicReaderContext);
AtomicReaderContext context = (AtomicReaderContext) indexSearcher.getTopReaderContext();
- Scorer ts = weight.scorer(context, true, true, FeatureFlags.DOCS, context.reader().getLiveDocs());
+ Scorer ts = weight.scorer(context, true, true, PostingFeatures.DOCS_AND_FREQS, context.reader().getLiveDocs());
assertTrue("Didn't skip", ts.advance(3) != DocIdSetIterator.NO_MORE_DOCS);
// The next doc should be doc 5
assertTrue("doc should be number 5", ts.docID() == 5);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java Mon Aug 13 13:52:46 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -124,29 +125,29 @@ public class TestTermVectors extends Luc
assertEquals(4, v.size());
String[] expectedFields = new String[]{"a", "b", "c", "x"};
int[] expectedPositions = new int[]{1, 2, 0};
- FieldsEnum fieldsEnum = v.iterator();
+ Iterator<String> fieldsEnum = v.iterator();
for(int i=0;i<expectedFields.length;i++) {
assertEquals(expectedFields[i], fieldsEnum.next());
assertEquals(3, v.terms(expectedFields[i]).size());
DocsAndPositionsEnum dpEnum = null;
- Terms terms = fieldsEnum.terms();
+ Terms terms = v.terms(expectedFields[i]);
assertNotNull(terms);
TermsEnum termsEnum = terms.iterator(null);
assertEquals("content", termsEnum.next().utf8ToString());
- dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(1, dpEnum.freq());
assertEquals(expectedPositions[0], dpEnum.nextPosition());
assertEquals("here", termsEnum.next().utf8ToString());
- dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(1, dpEnum.freq());
assertEquals(expectedPositions[1], dpEnum.nextPosition());
assertEquals("some", termsEnum.next().utf8ToString());
- dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(1, dpEnum.freq());
assertEquals(expectedPositions[2], dpEnum.nextPosition());
@@ -176,7 +177,7 @@ public class TestTermVectors extends Luc
if (shouldBePosVector || shouldBeOffVector) {
while(true) {
- dpEnum = termsEnum.docsAndPositions(null, dpEnum, shouldBeOffVector);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
assertNotNull(dpEnum);
assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
@@ -251,17 +252,17 @@ public class TestTermVectors extends Luc
writer.close();
IndexSearcher knownSearcher = newSearcher(reader);
knownSearcher.setSimilarity(new DefaultSimilarity());
- FieldsEnum fields = MultiFields.getFields(knownSearcher.reader).iterator();
+ Fields fields = MultiFields.getFields(knownSearcher.reader);
DocsEnum docs = null;
- while(fields.next() != null) {
- Terms terms = fields.terms();
+ for (String fieldName : fields) {
+ Terms terms = fields.terms(fieldName);
assertNotNull(terms); // NOTE: kinda sketchy assumptions, but ideally we would fix fieldsenum api...
TermsEnum termsEnum = terms.iterator(null);
while (termsEnum.next() != null) {
String text = termsEnum.term().utf8ToString();
- docs = _TestUtil.docs(random(), termsEnum, MultiFields.getLiveDocs(knownSearcher.reader), docs, true);
+ docs = _TestUtil.docs(random(), termsEnum, MultiFields.getLiveDocs(knownSearcher.reader), docs, DocsEnum.FLAG_FREQS);
while (docs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
int docId = docs.docID();
@@ -426,7 +427,7 @@ public class TestTermVectors extends Luc
assertNotNull(termsEnum.next());
assertEquals("one", termsEnum.term().utf8ToString());
assertEquals(5, termsEnum.totalTermFreq());
- DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false);
+ DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
assertNotNull(dpEnum);
assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(5, dpEnum.freq());
@@ -434,7 +435,7 @@ public class TestTermVectors extends Luc
assertEquals(i, dpEnum.nextPosition());
}
- dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
assertNotNull(dpEnum);
assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(5, dpEnum.freq());
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBasicIntervals.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBasicIntervals.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBasicIntervals.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBasicIntervals.java Mon Aug 13 13:52:46 2012
@@ -26,7 +26,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.search.positions.IntervalIterator.IntervalFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -185,7 +185,7 @@ public class TestBasicIntervals extends
Weight createWeight = rewrite.createWeight(new IndexSearcher(r));
Scorer scorer = createWeight.scorer(r.getTopReaderContext(), random()
- .nextBoolean(), true, FeatureFlags.POSITIONS, r.getLiveDocs());
+ .nextBoolean(), true, PostingFeatures.POSITIONS, r.getLiveDocs());
IntervalIterator positions = scorer.positions(false);
positions.scorer.advance(11);
positions.scorerAdvanced(11);
@@ -229,7 +229,7 @@ public class TestBasicIntervals extends
Weight createWeight = rewrite.createWeight(new IndexSearcher(r));
Scorer scorer = createWeight.scorer(r.getTopReaderContext(), random()
- .nextBoolean(), true, FeatureFlags.POSITIONS, r.getLiveDocs());
+ .nextBoolean(), true, PostingFeatures.POSITIONS, r.getLiveDocs());
IntervalIterator positions = scorer.positions(false);
advanceIter(positions, 4);
@@ -289,7 +289,7 @@ public class TestBasicIntervals extends
AtomicReader r = this.reader.getTopReaderContext().leaves().get(0).reader();
Weight createWeight = rewrite.createWeight(new IndexSearcher(r));
Scorer scorer = createWeight.scorer(r.getTopReaderContext(), random()
- .nextBoolean(), true, FeatureFlags.POSITIONS, r.getLiveDocs());
+ .nextBoolean(), true, PostingFeatures.POSITIONS, r.getLiveDocs());
IntervalIterator iterator = scorer.positions(false);
assertEquals(4, advanceIter(iterator, 4));
@@ -350,7 +350,7 @@ public class TestBasicIntervals extends
AtomicReader r = this.reader.getTopReaderContext().leaves().get(0).reader();
Weight createWeight = rewrite.createWeight(new IndexSearcher(r));
- Scorer scorer = createWeight.scorer(r.getTopReaderContext(), true, true, FeatureFlags.POSITIONS, r.getLiveDocs());
+ Scorer scorer = createWeight.scorer(r.getTopReaderContext(), true, true, PostingFeatures.POSITIONS, r.getLiveDocs());
return scorer.positions(false);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBlockIntervalIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBlockIntervalIterator.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBlockIntervalIterator.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBlockIntervalIterator.java Mon Aug 13 13:52:46 2012
@@ -21,7 +21,7 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.search.positions.IntervalIterator.IntervalFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -114,7 +114,7 @@ public class TestBlockIntervalIterator e
List<AtomicReaderContext> leaves = topReaderContext.leaves();
assertEquals(1, leaves.size());
for (AtomicReaderContext atomicReaderContext : leaves) {
- Scorer scorer = weight.scorer(atomicReaderContext, true, true, FeatureFlags.POSITIONS, atomicReaderContext.reader().getLiveDocs());
+ Scorer scorer = weight.scorer(atomicReaderContext, true, true, PostingFeatures.POSITIONS, atomicReaderContext.reader().getLiveDocs());
{
int nextDoc = scorer.nextDoc();
assertEquals(0, nextDoc);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBrouwerianQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBrouwerianQuery.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBrouwerianQuery.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestBrouwerianQuery.java Mon Aug 13 13:52:46 2012
@@ -29,7 +29,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
@@ -135,7 +135,7 @@ public class TestBrouwerianQuery extends
List<AtomicReaderContext> leaves = topReaderContext.leaves();
assertEquals(1, leaves.size());
for (AtomicReaderContext atomicReaderContext : leaves) {
- Scorer scorer = weight.scorer(atomicReaderContext, true, true, FeatureFlags.POSITIONS, atomicReaderContext.reader().getLiveDocs());
+ Scorer scorer = weight.scorer(atomicReaderContext, true, true, PostingFeatures.POSITIONS, atomicReaderContext.reader().getLiveDocs());
IntervalIterator positions = scorer.positions(false);
int nextDoc = scorer.nextDoc();
assertEquals(1, positions.scorerAdvanced(nextDoc));
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestConjunctionIntervalIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestConjunctionIntervalIterator.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestConjunctionIntervalIterator.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestConjunctionIntervalIterator.java Mon Aug 13 13:52:46 2012
@@ -21,7 +21,7 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -102,7 +102,7 @@ public class TestConjunctionIntervalIter
List<AtomicReaderContext> leaves = topReaderContext.leaves();
assertEquals(1, leaves.size());
for (AtomicReaderContext atomicReaderContext : leaves) {
- Scorer scorer = weight.scorer(atomicReaderContext, true, true, FeatureFlags.POSITIONS, atomicReaderContext.reader().getLiveDocs());
+ Scorer scorer = weight.scorer(atomicReaderContext, true, true, PostingFeatures.POSITIONS, atomicReaderContext.reader().getLiveDocs());
{
int nextDoc = scorer.nextDoc();
assertEquals(0, nextDoc);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestDisjunctionIntervalIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestDisjunctionIntervalIterator.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestDisjunctionIntervalIterator.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestDisjunctionIntervalIterator.java Mon Aug 13 13:52:46 2012
@@ -6,7 +6,7 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -105,7 +105,7 @@ public class TestDisjunctionIntervalIter
List<AtomicReaderContext> leaves = topReaderContext.leaves();
assertEquals(1, leaves.size());
Scorer scorer = weight.scorer(leaves.get(0),
- true, true, FeatureFlags.POSITIONS, leaves.get(0).reader().getLiveDocs());
+ true, true, PostingFeatures.POSITIONS, leaves.get(0).reader().getLiveDocs());
IntervalIterator positions = scorer.positions(false);
for (int i = 0; i < 2; i++) {
@@ -160,7 +160,7 @@ public class TestDisjunctionIntervalIter
assertEquals(1, leaves.size());
for (AtomicReaderContext atomicReaderContext : leaves) {
Scorer scorer = weight.scorer(atomicReaderContext,
- true, true, FeatureFlags.POSITIONS, atomicReaderContext.reader().getLiveDocs());
+ true, true, PostingFeatures.POSITIONS, atomicReaderContext.reader().getLiveDocs());
{
int nextDoc = scorer.nextDoc();
assertEquals(0, nextDoc);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestOrderedConjunctionIntervalIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestOrderedConjunctionIntervalIterator.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestOrderedConjunctionIntervalIterator.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestOrderedConjunctionIntervalIterator.java Mon Aug 13 13:52:46 2012
@@ -6,7 +6,7 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.search.positions.IntervalIterator.IntervalFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -107,7 +107,7 @@ public class TestOrderedConjunctionInter
List<AtomicReaderContext> leaves = topReaderContext.leaves();
assertEquals(1, leaves.size());
for (AtomicReaderContext atomicReaderContext : leaves) {
- Scorer scorer = weight.scorer(atomicReaderContext, true, true, FeatureFlags.POSITIONS, atomicReaderContext.reader()
+ Scorer scorer = weight.scorer(atomicReaderContext, true, true, PostingFeatures.POSITIONS, atomicReaderContext.reader()
.getLiveDocs());
{
int nextDoc = scorer.nextDoc();
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionsAndOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionsAndOffsets.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionsAndOffsets.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionsAndOffsets.java Mon Aug 13 13:52:46 2012
@@ -30,7 +30,7 @@ import org.apache.lucene.document.FieldT
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -100,7 +100,7 @@ public class TestPositionsAndOffsets ext
List<AtomicReaderContext> leaves = topReaderContext.leaves();
assertEquals(1, leaves.size());
Scorer scorer = weight.scorer(leaves.get(0),
- true, true, FeatureFlags.POSITIONS, leaves.get(0).reader().getLiveDocs());
+ true, true, PostingFeatures.POSITIONS, leaves.get(0).reader().getLiveDocs());
int nextDoc = scorer.nextDoc();
assertEquals(0, nextDoc);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java Mon Aug 13 13:52:46 2012
@@ -160,7 +160,7 @@ public class MultiSpansWrapper extends S
}
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
if (current == null) {
return false;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java Mon Aug 13 13:52:46 2012
@@ -96,7 +96,7 @@ public class TestBasics extends LuceneTe
}
}
- static final Analyzer simplePayloadAnalyzer = new Analyzer() {
+ static Analyzer simplePayloadAnalyzer = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
@@ -130,6 +130,7 @@ public class TestBasics extends LuceneTe
searcher = null;
reader = null;
directory = null;
+ simplePayloadAnalyzer = null;
}
@Test
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java Mon Aug 13 13:52:46 2012
@@ -30,7 +30,7 @@ import org.apache.lucene.search.Explanat
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -168,7 +168,7 @@ public class TestNearSpansOrdered extend
Weight w = searcher.createNormalizedWeight(q);
IndexReaderContext topReaderContext = searcher.getTopReaderContext();
AtomicReaderContext leave = topReaderContext.leaves().get(0);
- Scorer s = w.scorer(leave, true, false, FeatureFlags.POSITIONS, leave.reader().getLiveDocs());
+ Scorer s = w.scorer(leave, true, false, PostingFeatures.POSITIONS, leave.reader().getLiveDocs());
assertEquals(1, s.advance(1));
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Mon Aug 13 13:52:46 2012
@@ -256,7 +256,7 @@ public class TestPayloadSpans extends Lu
newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer()));
Document doc = new Document();
- doc.add(new TextField("content", new StringReader("a b c d e f g h i j a k"), Field.Store.NO));
+ doc.add(new TextField("content", new StringReader("a b c d e f g h i j a k")));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
@@ -293,7 +293,7 @@ public class TestPayloadSpans extends Lu
newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer()));
Document doc = new Document();
- doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k"), Field.Store.NO));
+ doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k")));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
IndexSearcher is = newSearcher(reader);
@@ -328,7 +328,7 @@ public class TestPayloadSpans extends Lu
newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer()));
Document doc = new Document();
- doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a"), Field.Store.NO));
+ doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a")));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
IndexSearcher is = newSearcher(reader);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java Mon Aug 13 13:52:46 2012
@@ -38,7 +38,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
@@ -429,7 +429,7 @@ public class TestSpans extends LuceneTes
slop,
ordered);
- spanScorer = searcher.createNormalizedWeight(snq).scorer(ctx, true, false, FeatureFlags.POSITIONS, ctx.reader().getLiveDocs());
+ spanScorer = searcher.createNormalizedWeight(snq).scorer(ctx, true, false, PostingFeatures.POSITIONS, ctx.reader().getLiveDocs());
} finally {
searcher.setSimilarity(oldSim);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java Mon Aug 13 13:52:46 2012
@@ -18,9 +18,11 @@ package org.apache.lucene.store;
*/
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.lucene.store.MockDirectoryWrapper.Throttling;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -41,6 +43,89 @@ public class TestDirectory extends Lucen
}
}
}
+
+ // test is occasionally very slow, i dont know why
+ // try this seed: 7D7E036AD12927F5:93333EF9E6DE44DE
+ @Nightly
+ public void testThreadSafety() throws Exception {
+ final BaseDirectoryWrapper dir = newDirectory();
+ dir.setCheckIndexOnClose(false); // we arent making an index
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)dir).setThrottling(Throttling.NEVER); // makes this test really slow
+ }
+
+ if (VERBOSE) {
+ System.out.println(dir);
+ }
+
+ class TheThread extends Thread {
+ private String name;
+
+ public TheThread(String name) {
+ this.name = name;
+ }
+
+ public void run() {
+ for (int i = 0; i < 3000; i++) {
+ String fileName = this.name + i;
+ try {
+ //System.out.println("create:" + fileName);
+ IndexOutput output = dir.createOutput(fileName, newIOContext(random()));
+ output.close();
+ assertTrue(dir.fileExists(fileName));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ };
+
+ class TheThread2 extends Thread {
+ private String name;
+
+ public TheThread2(String name) {
+ this.name = name;
+ }
+
+ public void run() {
+ for (int i = 0; i < 10000; i++) {
+ try {
+ String[] files = dir.listAll();
+ for (String file : files) {
+ //System.out.println("file:" + file);
+ try {
+ IndexInput input = dir.openInput(file, newIOContext(random()));
+ input.close();
+ } catch (FileNotFoundException e) {
+ // ignore
+ } catch (IOException e) {
+ if (e.getMessage().contains("still open for writing")) {
+ // ignore
+ } else {
+ throw new RuntimeException(e);
+ }
+ }
+ if (random().nextBoolean()) {
+ break;
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ };
+
+ TheThread theThread = new TheThread("t1");
+ TheThread2 theThread2 = new TheThread2("t2");
+ theThread.start();
+ theThread2.start();
+
+ theThread.join();
+ theThread2.join();
+
+ dir.close();
+ }
// Test that different instances of FSDirectory can coexist on the same
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java Mon Aug 13 13:52:46 2012
@@ -128,9 +128,10 @@ public class TestWeakIdentityMap extends
assertNotNull(it.next());
c++;
}
- assertTrue(size >= c);
- assertTrue(c >= map.size());
- size = map.size();
+ final int newSize = map.size();
+ assertTrue("previousSize("+size+")>=iteratorSize("+c+")", size >= c);
+ assertTrue("iteratorSize("+c+")>=newSize("+newSize+")", c >= newSize);
+ size = newSize;
} catch (InterruptedException ie) {}
map.clear();
@@ -159,7 +160,7 @@ public class TestWeakIdentityMap extends
public void testConcurrentHashMap() throws Exception {
// don't make threadCount and keyCount random, otherwise easily OOMs or fails otherwise:
final int threadCount = 8, keyCount = 1024;
- final ExecutorService exec = Executors.newFixedThreadPool(threadCount);
+ final ExecutorService exec = Executors.newFixedThreadPool(threadCount, new NamedThreadFactory("testConcurrentHashMap"));
final WeakIdentityMap<Object,Integer> map =
WeakIdentityMap.newConcurrentHashMap();
// we keep strong references to the keys,
@@ -228,9 +229,10 @@ public class TestWeakIdentityMap extends
assertNotNull(it.next());
c++;
}
- assertTrue(size >= c);
- assertTrue(c >= map.size());
- size = map.size();
+ final int newSize = map.size();
+ assertTrue("previousSize("+size+")>=iteratorSize("+c+")", size >= c);
+ assertTrue("iteratorSize("+c+")>=newSize("+newSize+")", c >= newSize);
+ size = newSize;
} catch (InterruptedException ie) {}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java Mon Aug 13 13:52:46 2012
@@ -67,7 +67,7 @@ import org.apache.lucene.util.fst.FST.By
import org.apache.lucene.util.fst.PairOutputs.Pair;
import org.apache.lucene.util.packed.PackedInts;
-@SuppressCodecs({ "SimpleText", "Memory" })
+@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
@Slow
public class TestFSTs extends LuceneTestCase {
@@ -76,7 +76,7 @@ public class TestFSTs extends LuceneTest
@Override
public void setUp() throws Exception {
super.setUp();
- dir = newDirectory();
+ dir = newMockDirectory();
dir.setPreventDoubleWrite(false);
}
@@ -1096,18 +1096,11 @@ public class TestFSTs extends LuceneTest
// file, up until a time limit
public void testRealTerms() throws Exception {
- // TODO: is this necessary? we use the annotation...
- final String defaultFormat = _TestUtil.getPostingsFormat("abracadabra");
- if (defaultFormat.equals("SimpleText") || defaultFormat.equals("Memory")) {
- // no
- Codec.setDefault(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()));
- }
-
final LineFileDocs docs = new LineFileDocs(random(), true);
final int RUN_TIME_MSEC = atLeast(500);
final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(-1).setRAMBufferSizeMB(64);
final File tempDir = _TestUtil.getTempDir("fstlines");
- final MockDirectoryWrapper dir = newFSDirectory(tempDir);
+ final Directory dir = newFSDirectory(tempDir);
final IndexWriter writer = new IndexWriter(dir, conf);
final long stopTime = System.currentTimeMillis() + RUN_TIME_MSEC;
Document doc;
Modified: lucene/dev/branches/LUCENE-2878/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java Mon Aug 13 13:52:46 2012
@@ -190,7 +190,7 @@ public class IndexFiles {
// so that the text of the file is tokenized and indexed, but not stored.
// Note that FileReader expects the file to be in UTF-8 encoding.
// If that's not the case searching for special characters will fail.
- doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8")), Field.Store.NO));
+ doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8"))));
if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
// New index, so we just add the document (no old document can be there):
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/package.html?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/package.html (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/package.html Mon Aug 13 13:52:46 2012
@@ -19,7 +19,6 @@
<title>Faceted Indexing and Search</title>
</head>
<body>
- <!-- NOTE: if you update the userguide link, also update the one in src/java/overview.html -->
Provides faceted indexing and search capabilities. The <a href="doc-files/userguide.html">userguide</a> is recommended for a start.
</body>
</html>
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java Mon Aug 13 13:52:46 2012
@@ -72,12 +72,14 @@ public class FacetsCollector extends Col
protected ScoredDocIdCollector initScoredDocCollector(
FacetSearchParams facetSearchParams, IndexReader indexReader,
TaxonomyReader taxonomyReader) {
+ boolean scoresNeeded = false;
for (FacetRequest frq : facetSearchParams.getFacetRequests()) {
if (frq.requireDocumentScore()) {
- return ScoredDocIdCollector.create(1000, true);
+ scoresNeeded = true;
+ break;
}
}
- return ScoredDocIdCollector.create(indexReader.maxDoc(), false);
+ return ScoredDocIdCollector.create(indexReader.maxDoc(), scoresNeeded);
}
/**
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java Mon Aug 13 13:52:46 2012
@@ -53,7 +53,7 @@ public class PayloadIterator {
this.buffer = buffer;
// TODO (Facet): avoid Multi*?
Bits liveDocs = MultiFields.getLiveDocs(indexReader);
- this.tp = MultiFields.getTermPositionsEnum(indexReader, liveDocs, term.field(), term.bytes(), false);
+ this.tp = MultiFields.getTermPositionsEnum(indexReader, liveDocs, term.field(), term.bytes(), DocsAndPositionsEnum.FLAG_PAYLOADS);
}
/**
@@ -99,16 +99,13 @@ public class PayloadIterator {
// Prepare for payload extraction
tp.nextPosition();
- // TODO: fix bug in SepCodec and then remove this check (the null check should be enough)
- if (!tp.hasPayload()) {
- return false;
- }
-
BytesRef br = tp.getPayload();
-
- if (br == null || br.length == 0) {
+
+ if (br == null) {
return false;
}
+
+ assert br.length > 0;
this.payloadLength = br.length;
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java Mon Aug 13 13:52:46 2012
@@ -9,7 +9,7 @@ import org.apache.lucene.search.DocIdSet
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -52,7 +52,7 @@ public abstract class ScoredDocIdCollect
@Override
public void collect(int doc) {
- docIds.fastSet(docBase + doc);
+ docIds.set(docBase + doc);
++numDocIds;
}
@@ -103,7 +103,9 @@ public abstract class ScoredDocIdCollect
@SuppressWarnings("synthetic-access")
public ScoringDocIdCollector(int maxDoc) {
super(maxDoc);
- scores = new float[maxDoc];
+ // only matching documents have an entry in the scores array. Therefore start with
+ // a small array and grow when needed.
+ scores = new float[64];
}
@Override
@@ -111,7 +113,7 @@ public abstract class ScoredDocIdCollect
@Override
public void collect(int doc) throws IOException {
- docIds.fastSet(docBase + doc);
+ docIds.set(docBase + doc);
float score = this.scorer.score();
if (numDocIds >= scores.length) {
@@ -167,7 +169,7 @@ public abstract class ScoredDocIdCollect
protected int numDocIds;
protected int docBase;
- protected final OpenBitSet docIds;
+ protected final FixedBitSet docIds;
/**
* Creates a new {@link ScoredDocIdCollector} with the given parameters.
@@ -187,7 +189,7 @@ public abstract class ScoredDocIdCollect
private ScoredDocIdCollector(int maxDoc) {
numDocIds = 0;
- docIds = new OpenBitSet(maxDoc);
+ docIds = new FixedBitSet(maxDoc);
}
/** Returns the default score used when scoring is disabled. */
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/results/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/results/package.html?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/results/package.html (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/results/package.html Mon Aug 13 13:52:46 2012
@@ -20,15 +20,8 @@
</head>
<body>
<h1>Results of Faceted Search</h1>
- <!--
<p>
The results of facets accumulation are obtained as a list of {@link org.apache.lucene.facet.search.results.FacetResult} elements.
- See two API calls for obtaining these results:
- <ul>
- <li></li>
- </ul>
-
</p>
- -->
</body>
</html>
\ No newline at end of file
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java Mon Aug 13 13:52:46 2012
@@ -108,7 +108,7 @@ class TakmiSampleFixer implements Sample
Bits liveDocs = MultiFields.getLiveDocs(indexReader);
int updatedCount = countIntersection(MultiFields.getTermDocsEnum(indexReader, liveDocs,
drillDownTerm.field(), drillDownTerm.bytes(),
- false),
+ 0),
docIds.iterator());
fresNode.setValue(updatedCount);
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java Mon Aug 13 13:52:46 2012
@@ -196,7 +196,7 @@ public class DirectoryTaxonomyReader imp
indexReaderLock.readLock().lock();
// TODO (Facet): avoid Multi*?
Bits liveDocs = MultiFields.getLiveDocs(indexReader);
- DocsEnum docs = MultiFields.getTermDocsEnum(indexReader, liveDocs, Consts.FULL, new BytesRef(path), false);
+ DocsEnum docs = MultiFields.getTermDocsEnum(indexReader, liveDocs, Consts.FULL, new BytesRef(path), 0);
if (docs != null && docs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
ret = docs.docID();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java Mon Aug 13 13:52:46 2012
@@ -375,7 +375,7 @@ public class DirectoryTaxonomyWriter imp
* returning the category's ordinal, or a negative number in case the
* category does not yet exist in the taxonomy.
*/
- protected int findCategory(CategoryPath categoryPath) throws IOException {
+ protected synchronized int findCategory(CategoryPath categoryPath) throws IOException {
// If we can find the category in the cache, or we know the cache is
// complete, we can return the response directly from it
int res = cache.get(categoryPath);
@@ -411,7 +411,7 @@ public class DirectoryTaxonomyWriter imp
final BytesRef catTerm = new BytesRef(categoryPath.toString(delimiter));
int base = 0;
for (AtomicReader r : reader.getSequentialSubReaders()) {
- DocsEnum docs = r.termDocsEnum(null, Consts.FULL, catTerm, false);
+ DocsEnum docs = r.termDocsEnum(null, Consts.FULL, catTerm, 0);
if (docs != null) {
doc = docs.nextDoc() + base;
break;
@@ -454,7 +454,7 @@ public class DirectoryTaxonomyWriter imp
final BytesRef catTerm = new BytesRef(categoryPath.toString(delimiter, prefixLen));
int base = 0;
for (AtomicReader r : reader.getSequentialSubReaders()) {
- DocsEnum docs = r.termDocsEnum(null, Consts.FULL, catTerm, false);
+ DocsEnum docs = r.termDocsEnum(null, Consts.FULL, catTerm, 0);
if (docs != null) {
doc = docs.nextDoc() + base;
break;
@@ -474,12 +474,11 @@ public class DirectoryTaxonomyWriter imp
@Override
public int addCategory(CategoryPath categoryPath) throws IOException {
ensureOpen();
- // If the category is already in the cache and/or the taxonomy, we
- // should return its existing ordinal
- int res = findCategory(categoryPath);
+ // check the cache outside the synchronized block. this results in better
+ // concurrency when categories are there.
+ int res = cache.get(categoryPath);
if (res < 0) {
- // the category is neither in the cache nor in the index - following code
- // cannot be executed in parallel.
+ // the category is not in the cache - following code cannot be executed in parallel.
synchronized (this) {
res = findCategory(categoryPath);
if (res < 0) {
@@ -494,7 +493,6 @@ public class DirectoryTaxonomyWriter imp
}
}
return res;
-
}
/**
@@ -769,7 +767,7 @@ public class DirectoryTaxonomyWriter imp
// 'validation' checks.
cp.clear();
cp.add(t.utf8ToString(), delimiter);
- docsEnum = termsEnum.docs(null, docsEnum, false);
+ docsEnum = termsEnum.docs(null, docsEnum, 0);
boolean res = cache.put(cp, docsEnum.nextDoc() + base);
assert !res : "entries should not have been evicted from the cache";
} else {
@@ -861,7 +859,7 @@ public class DirectoryTaxonomyWriter imp
// the findCategory() call above failed to find it.
ordinal = addCategory(cp);
}
- docs = te.docs(null, docs, false);
+ docs = te.docs(null, docs, 0);
ordinalMap.addMapping(docs.nextDoc() + base, ordinal);
}
base += ar.maxDoc(); // no deletions, so we're ok
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java Mon Aug 13 13:52:46 2012
@@ -106,7 +106,7 @@ class ParentArray {
Bits liveDocs = MultiFields.getLiveDocs(indexReader);
DocsAndPositionsEnum positions = MultiFields.getTermPositionsEnum(indexReader, liveDocs,
Consts.FIELD_PAYLOADS, new BytesRef(Consts.PAYLOAD_PARENT),
- false);
+ DocsAndPositionsEnum.FLAG_PAYLOADS);
if ((positions == null || positions.advance(first) == DocIdSetIterator.NO_MORE_DOCS) && first < num) {
throw new CorruptIndexException("Missing parent data for category " + first);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java Mon Aug 13 13:52:46 2012
@@ -22,22 +22,25 @@ import org.apache.lucene.facet.taxonomy.
/**
* TaxonomyWriterCache is a relatively simple interface for a cache of
- * category->ordinal mappings, used in TaxonomyWriter implementations
- * (such as {@link DirectoryTaxonomyWriter}).
- * <P>
- * It basically has put() methods for adding a mapping, and get() for looking
- * a mapping up the cache. The cache does <B>not</B> guarantee to hold
- * everything that has been put into it, and might in fact selectively
- * delete some of the mappings (e.g., the ones least recently used).
- * This means that if get() returns a negative response, it does not
- * necessarily mean that the category doesn't exist - just that it is not
- * in the cache. The caller can only infer that the category doesn't exist
- * if it knows the cache to be complete (because all the categories were
- * loaded into the cache, and since then no put() returned true).
- * <P> However,
- * if it does so, it should clear out large parts of the cache at once, because
- * the user will typically need to work hard to recover from every cache
+ * category->ordinal mappings, used in TaxonomyWriter implementations (such as
+ * {@link DirectoryTaxonomyWriter}).
+ * <p>
+ * It basically has put() methods for adding a mapping, and get() for looking a
+ * mapping up the cache. The cache does <B>not</B> guarantee to hold everything
+ * that has been put into it, and might in fact selectively delete some of the
+ * mappings (e.g., the ones least recently used). This means that if get()
+ * returns a negative response, it does not necessarily mean that the category
+ * doesn't exist - just that it is not in the cache. The caller can only infer
+ * that the category doesn't exist if it knows the cache to be complete (because
+ * all the categories were loaded into the cache, and since then no put()
+ * returned true).
+ * <p>
+ * However, if it does so, it should clear out large parts of the cache at once,
+ * because the user will typically need to work hard to recover from every cache
* cleanup (see {@link #put(CategoryPath, int)}'s return value).
+ * <p>
+ * <b>NOTE:</b> the cache may be accessed concurrently by multiple threads,
+ * therefore cache implementations should take this into consideration.
*
* @lucene.experimental
*/
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/overview.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/overview.html?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/overview.html (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/overview.html Mon Aug 13 13:52:46 2012
@@ -21,7 +21,6 @@
</title>
</head>
<body>
- <!-- NOTE: if you update the userguide link, also update the one in o.a.l.facet/package.html -->
Provides faceted indexing and search capabilities (checkout the <a href="org/apache/lucene/facet/doc-files/userguide.html">userguide</a>).
</body>
</html>
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java Mon Aug 13 13:52:46 2012
@@ -284,7 +284,7 @@ public abstract class FacetTestBase exte
TermsEnum te = terms.iterator(null);
DocsEnum de = null;
while (te.next() != null) {
- de = _TestUtil.docs(random(), te, liveDocs, de, false);
+ de = _TestUtil.docs(random(), te, liveDocs, de, 0);
int cnt = 0;
while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
cnt++;
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java Mon Aug 13 13:52:46 2012
@@ -88,7 +88,7 @@ public class TestMultipleCategoryLists e
// Obtain facets results and hand-test them
assertCorrectResults(facetsCollector);
- DocsEnum td = _TestUtil.docs(random(), ir, "$facets", new BytesRef("$fulltree$"), MultiFields.getLiveDocs(ir), null, false);
+ DocsEnum td = _TestUtil.docs(random(), ir, "$facets", new BytesRef("$fulltree$"), MultiFields.getLiveDocs(ir), null, 0);
assertTrue(td.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
tr.close();
@@ -182,7 +182,7 @@ public class TestMultipleCategoryLists e
}
private void assertPostingListExists(String field, String text, IndexReader ir) throws IOException {
- DocsEnum de = _TestUtil.docs(random(), ir, field, new BytesRef(text), null, null, false);
+ DocsEnum de = _TestUtil.docs(random(), ir, field, new BytesRef(text), null, null, 0);
assertTrue(de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java Mon Aug 13 13:52:46 2012
@@ -60,18 +60,14 @@ public final class QueryTermExtractor
public static final WeightedTerm[] getIdfWeightedTerms(Query query, IndexReader reader, String fieldName)
{
WeightedTerm[] terms=getTerms(query,false, fieldName);
- int totalNumDocs=reader.numDocs();
+ int totalNumDocs=reader.maxDoc();
for (int i = 0; i < terms.length; i++)
{
try
{
int docFreq=reader.docFreq(new Term(fieldName,terms[i].term));
- // docFreq counts deletes
- if(totalNumDocs < docFreq) {
- docFreq = totalNumDocs;
- }
//IDF algorithm taken from DefaultSimilarity class
- float idf=(float)(Math.log((float)totalNumDocs/(double)(docFreq+1)) + 1.0);
+ float idf=(float)(Math.log(totalNumDocs/(double)(docFreq+1)) + 1.0);
terms[i].weight*=idf;
}
catch (IOException e)
Modified: lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Mon Aug 13 13:52:46 2012
@@ -125,20 +125,7 @@ public class TokenSources {
}
private static boolean hasPositions(Terms vector) throws IOException {
- final TermsEnum termsEnum = vector.iterator(null);
- if (termsEnum.next() != null) {
- DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false);
- if (dpEnum != null) {
- int doc = dpEnum.nextDoc();
- assert doc >= 0 && doc != DocIdSetIterator.NO_MORE_DOCS;
- int pos = dpEnum.nextPosition();
- if (pos >= 0) {
- return true;
- }
- }
- }
-
- return false;
+ return vector.hasPositions();
}
/**
@@ -222,18 +209,21 @@ public class TokenSources {
DocsAndPositionsEnum dpEnum = null;
while ((text = termsEnum.next()) != null) {
- dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
if (dpEnum == null) {
throw new IllegalArgumentException(
"Required TermVector Offset information was not found");
}
-
final String term = text.utf8ToString();
dpEnum.nextDoc();
final int freq = dpEnum.freq();
for(int posUpto=0;posUpto<freq;posUpto++) {
final int pos = dpEnum.nextPosition();
+ if (dpEnum.startOffset() < 0) {
+ throw new IllegalArgumentException(
+ "Required TermVector Offset information was not found");
+ }
final Token token = new Token(term,
dpEnum.startOffset(),
dpEnum.endOffset());
Modified: lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java Mon Aug 13 13:52:46 2012
@@ -56,18 +56,13 @@ public final class TokenStreamFromTermPo
termAttribute = addAttribute(CharTermAttribute.class);
positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
offsetAttribute = addAttribute(OffsetAttribute.class);
+ final boolean hasOffsets = vector.hasOffsets();
final TermsEnum termsEnum = vector.iterator(null);
BytesRef text;
DocsAndPositionsEnum dpEnum = null;
while((text = termsEnum.next()) != null) {
- dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
- final boolean hasOffsets;
- if (dpEnum == null) {
- hasOffsets = false;
- dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
- } else {
- hasOffsets = true;
- }
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ assert dpEnum != null; // presumably checked by TokenSources.hasPositions earlier
dpEnum.nextDoc();
final int freq = dpEnum.freq();
for (int j = 0; j < freq; j++) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Mon Aug 13 13:52:46 2012
@@ -431,7 +431,7 @@ public class WeightedSpanTermExtractor {
Map<String,WeightedSpanTerm> terms = new PositionCheckingMap<String>();
extract(query, terms);
- int totalNumDocs = reader.numDocs();
+ int totalNumDocs = reader.maxDoc();
Set<String> weightedTerms = terms.keySet();
Iterator<String> it = weightedTerms.iterator();
@@ -439,12 +439,8 @@ public class WeightedSpanTermExtractor {
while (it.hasNext()) {
WeightedSpanTerm weightedSpanTerm = terms.get(it.next());
int docFreq = reader.docFreq(new Term(fieldName, weightedSpanTerm.term));
- // docFreq counts deletes
- if(totalNumDocs < docFreq) {
- docFreq = totalNumDocs;
- }
// IDF algorithm taken from DefaultSimilarity class
- float idf = (float) (Math.log((float) totalNumDocs / (double) (docFreq + 1)) + 1.0);
+ float idf = (float) (Math.log(totalNumDocs / (double) (docFreq + 1)) + 1.0);
weightedSpanTerm.weight *= idf;
}
} finally {
Modified: lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/HighlightingIntervalCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/HighlightingIntervalCollector.java?rev=1372423&r1=1372422&r2=1372423&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/HighlightingIntervalCollector.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/HighlightingIntervalCollector.java Mon Aug 13 13:52:46 2012
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight.FeatureFlags;
+import org.apache.lucene.search.Weight.PostingFeatures;
import org.apache.lucene.search.positions.IntervalIterator;
import org.apache.lucene.search.positions.IntervalIterator.IntervalCollector;
import org.apache.lucene.search.positions.Interval;
@@ -90,8 +90,8 @@ public class HighlightingIntervalCollect
}
@Override
- public FeatureFlags scorerFlags() {
- return FeatureFlags.OFFSETS;
+ public PostingFeatures postingFeatures() {
+ return PostingFeatures.OFFSETS;
}
@Override