You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/11/03 14:35:20 UTC
svn commit: r1197110 [6/10] - in /lucene/dev/branches/solrcloud: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/contrib/ dev-tools/idea/modules/benchmark/
dev-tools/idea/solr/contrib/langid/ dev-tools/...
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Thu Nov 3 13:35:07 2011
@@ -29,12 +29,13 @@ import org.apache.lucene.index.SerialMer
import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util._TestUtil;
public class TestCachingWrapperFilter extends LuceneTestCase {
-
+
public void testCachingWorks() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, dir);
@@ -46,15 +47,15 @@ public class TestCachingWrapperFilter ex
CachingWrapperFilter cacher = new CachingWrapperFilter(filter);
// first time, nested filter is called
- cacher.getDocIdSet(context);
+ cacher.getDocIdSet(context, context.reader.getLiveDocs());
assertTrue("first time", filter.wasCalled());
// make sure no exception if cache is holding the wrong docIdSet
- cacher.getDocIdSet(context);
+ cacher.getDocIdSet(context, context.reader.getLiveDocs());
// second time, nested filter should not be called
filter.clear();
- cacher.getDocIdSet(context);
+ cacher.getDocIdSet(context, context.reader.getLiveDocs());
assertFalse("second time", filter.wasCalled());
reader.close();
@@ -71,14 +72,14 @@ public class TestCachingWrapperFilter ex
final Filter filter = new Filter() {
@Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) {
+ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) {
return null;
}
};
CachingWrapperFilter cacher = new CachingWrapperFilter(filter);
// the caching filter should return the empty set constant
- assertSame(DocIdSet.EMPTY_DOCIDSET, cacher.getDocIdSet(context));
+ assertSame(DocIdSet.EMPTY_DOCIDSET, cacher.getDocIdSet(context, context.reader.getLiveDocs()));
reader.close();
dir.close();
@@ -94,7 +95,7 @@ public class TestCachingWrapperFilter ex
final Filter filter = new Filter() {
@Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) {
+ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) {
return new DocIdSet() {
@Override
public DocIdSetIterator iterator() {
@@ -106,7 +107,7 @@ public class TestCachingWrapperFilter ex
CachingWrapperFilter cacher = new CachingWrapperFilter(filter);
// the caching filter should return the empty set constant
- assertSame(DocIdSet.EMPTY_DOCIDSET, cacher.getDocIdSet(context));
+ assertSame(DocIdSet.EMPTY_DOCIDSET, cacher.getDocIdSet(context, context.reader.getLiveDocs()));
reader.close();
dir.close();
@@ -116,8 +117,8 @@ public class TestCachingWrapperFilter ex
assertTrue(reader.getTopReaderContext().isAtomic);
AtomicReaderContext context = (AtomicReaderContext) reader.getTopReaderContext();
final CachingWrapperFilter cacher = new CachingWrapperFilter(filter);
- final DocIdSet originalSet = filter.getDocIdSet(context);
- final DocIdSet cachedSet = cacher.getDocIdSet(context);
+ final DocIdSet originalSet = filter.getDocIdSet(context, context.reader.getLiveDocs());
+ final DocIdSet cachedSet = cacher.getDocIdSet(context, context.reader.getLiveDocs());
assertTrue(cachedSet.isCacheable());
assertEquals(shouldCacheable, originalSet.isCacheable());
//System.out.println("Original: "+originalSet.getClass().getName()+" -- cached: "+cachedSet.getClass().getName());
@@ -145,7 +146,7 @@ public class TestCachingWrapperFilter ex
// a fixedbitset filter is always cacheable
assertDocIdSetCacheable(reader, new Filter() {
@Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) {
+ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) {
return new FixedBitSet(context.reader.maxDoc());
}
}, true);
@@ -187,91 +188,41 @@ public class TestCachingWrapperFilter ex
final Filter startFilter = new QueryWrapperFilter(new TermQuery(new Term("id", "1")));
- // ignore deletions
- CachingWrapperFilter filter = new CachingWrapperFilter(startFilter, CachingWrapperFilter.DeletesMode.IGNORE);
-
- docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
- assertEquals("[query + filter] Should find a hit...", 1, docs.totalHits);
- ConstantScoreQuery constantScore = new ConstantScoreQuery(filter);
- docs = searcher.search(constantScore, 1);
- assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
-
- // now delete the doc, refresh the reader, and see that it's not there
- _TestUtil.keepFullyDeletedSegments(writer.w);
- writer.deleteDocuments(new Term("id", "1"));
-
- reader = refreshReader(reader);
- searcher.close();
- searcher = newSearcher(reader, false);
+ CachingWrapperFilter filter = new CachingWrapperFilter(startFilter);
docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
- assertEquals("[query + filter] Should *not* find a hit...", 0, docs.totalHits);
-
- docs = searcher.search(constantScore, 1);
- assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
-
-
- // force cache to regenerate:
- filter = new CachingWrapperFilter(startFilter, CachingWrapperFilter.DeletesMode.RECACHE);
-
- writer.addDocument(doc);
-
- reader = refreshReader(reader);
- searcher.close();
- searcher = newSearcher(reader, false);
-
- docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
-
assertEquals("[query + filter] Should find a hit...", 1, docs.totalHits);
-
- constantScore = new ConstantScoreQuery(filter);
+ int missCount = filter.missCount;
+ assertTrue(missCount > 0);
+ Query constantScore = new ConstantScoreQuery(filter);
docs = searcher.search(constantScore, 1);
assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
+ assertEquals(missCount, filter.missCount);
// NOTE: important to hold ref here so GC doesn't clear
// the cache entry! Else the assert below may sometimes
// fail:
IndexReader oldReader = reader;
- // make sure we get a cache hit when we reopen reader
- // that had no change to deletions
- reader = refreshReader(reader);
- assertTrue(reader != oldReader);
- searcher.close();
- searcher = newSearcher(reader, false);
- int missCount = filter.missCount;
- docs = searcher.search(constantScore, 1);
- assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
- assertEquals(missCount, filter.missCount);
-
- // now delete the doc, refresh the reader, and see that it's not there
- writer.deleteDocuments(new Term("id", "1"));
-
- reader = refreshReader(reader);
- searcher.close();
- searcher = newSearcher(reader, false);
-
- missCount = filter.missCount;
- docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
- assertEquals(missCount+1, filter.missCount);
- assertEquals("[query + filter] Should *not* find a hit...", 0, docs.totalHits);
- docs = searcher.search(constantScore, 1);
- assertEquals("[just filter] Should *not* find a hit...", 0, docs.totalHits);
-
-
- // apply deletions dynamically
- filter = new CachingWrapperFilter(startFilter, CachingWrapperFilter.DeletesMode.DYNAMIC);
-
writer.addDocument(doc);
reader = refreshReader(reader);
searcher.close();
searcher = newSearcher(reader, false);
docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
- assertEquals("[query + filter] Should find a hit...", 1, docs.totalHits);
+ assertEquals("[query + filter] Should find 2 hits...", 2, docs.totalHits);
+ assertTrue(filter.missCount > missCount);
+ missCount = filter.missCount;
+
constantScore = new ConstantScoreQuery(filter);
docs = searcher.search(constantScore, 1);
- assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
+ assertEquals("[just filter] Should find a hit...", 2, docs.totalHits);
+ assertEquals(missCount, filter.missCount);
+
+ // NOTE: important to hold ref here so GC doesn't clear
+ // the cache entry! Else the assert below may sometimes
+ // fail:
+ IndexReader oldReader2 = reader;
// now delete the doc, refresh the reader, and see that it's not there
writer.deleteDocuments(new Term("id", "1"));
@@ -282,12 +233,10 @@ public class TestCachingWrapperFilter ex
docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
assertEquals("[query + filter] Should *not* find a hit...", 0, docs.totalHits);
+ assertEquals(missCount, filter.missCount);
- missCount = filter.missCount;
docs = searcher.search(constantScore, 1);
assertEquals("[just filter] Should *not* find a hit...", 0, docs.totalHits);
-
- // doesn't count as a miss
assertEquals(missCount, filter.missCount);
// NOTE: silliness to make sure JRE does not optimize
@@ -295,6 +244,7 @@ public class TestCachingWrapperFilter ex
// CachingWrapperFilter's WeakHashMap from dropping the
// entry:
assertTrue(oldReader != null);
+ assertTrue(oldReader2 != null);
searcher.close();
reader.close();
@@ -312,4 +262,5 @@ public class TestCachingWrapperFilter ex
return oldReader;
}
}
+
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java Thu Nov 3 13:35:07 2011
@@ -130,5 +130,32 @@ public class TestConstantScoreQuery exte
if (directory != null) directory.close();
}
}
+
+ public void testConstantScoreQueryAndFilter() throws Exception {
+ Directory d = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random, d);
+ Document doc = new Document();
+ doc.add(newField("field", "a", StringField.TYPE_UNSTORED));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(newField("field", "b", StringField.TYPE_UNSTORED));
+ w.addDocument(doc);
+ IndexReader r = w.getReader();
+ w.close();
+
+ Filter filterB = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "b"))));
+ Query query = new ConstantScoreQuery(filterB);
+
+ IndexSearcher s = new IndexSearcher(r);
+ assertEquals(1, s.search(query, filterB, 1).totalHits); // Query for field:b, Filter field:b
+
+ Filter filterA = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "a"))));
+ query = new ConstantScoreQuery(filterA);
+
+ assertEquals(0, s.search(query, filterB, 1).totalHits); // Query field:b, Filter field:a
+
+ r.close();
+ d.close();
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDocIdSet.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDocIdSet.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDocIdSet.java Thu Nov 3 13:35:07 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
public class TestDocIdSet extends LuceneTestCase {
@@ -114,7 +115,7 @@ public class TestDocIdSet extends Lucene
// Now search w/ a Filter which returns a null DocIdSet
Filter f = new Filter() {
@Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException {
+ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
return null;
}
};
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFieldCacheRewriteMethod.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFieldCacheRewriteMethod.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFieldCacheRewriteMethod.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFieldCacheRewriteMethod.java Thu Nov 3 13:35:07 2011
@@ -30,10 +30,10 @@ public class TestFieldCacheRewriteMethod
/** Test fieldcache rewrite against filter rewrite */
@Override
protected void assertSame(String regexp) throws IOException {
- RegexpQuery fieldCache = new RegexpQuery(new Term("field", regexp), RegExp.NONE);
+ RegexpQuery fieldCache = new RegexpQuery(new Term(fieldName, regexp), RegExp.NONE);
fieldCache.setRewriteMethod(new FieldCacheRewriteMethod());
- RegexpQuery filter = new RegexpQuery(new Term("field", regexp), RegExp.NONE);
+ RegexpQuery filter = new RegexpQuery(new Term(fieldName, regexp), RegExp.NONE);
filter.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
TopDocs fieldCacheDocs = searcher1.search(fieldCache, 25);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFilteredQuery.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFilteredQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFilteredQuery.java Thu Nov 3 13:35:07 2011
@@ -28,6 +28,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.DocIdBitSet;
import org.apache.lucene.util.LuceneTestCase;
@@ -82,6 +83,7 @@ public class TestFilteredQuery extends L
writer.close ();
searcher = newSearcher(reader);
+
query = new TermQuery (new Term ("field", "three"));
filter = newStaticFilterB();
}
@@ -90,10 +92,11 @@ public class TestFilteredQuery extends L
private static Filter newStaticFilterB() {
return new Filter() {
@Override
- public DocIdSet getDocIdSet (AtomicReaderContext context) {
+ public DocIdSet getDocIdSet (AtomicReaderContext context, Bits acceptDocs) {
+ if (acceptDocs == null) acceptDocs = new Bits.MatchAllBits(5);
BitSet bitset = new BitSet(5);
- bitset.set (1);
- bitset.set (3);
+ if (acceptDocs.get(1)) bitset.set(1);
+ if (acceptDocs.get(3)) bitset.set(3);
return new DocIdBitSet(bitset);
}
};
@@ -106,10 +109,16 @@ public class TestFilteredQuery extends L
directory.close();
super.tearDown();
}
+
+ public void testFilteredQuery() throws Exception {
+ // force the filter to be executed as bits
+ tFilteredQuery(true);
+ // force the filter to be executed as iterator
+ tFilteredQuery(false);
+ }
- public void testFilteredQuery()
- throws Exception {
- Query filteredquery = new FilteredQuery (query, filter);
+ private void tFilteredQuery(final boolean useRandomAccess) throws Exception {
+ Query filteredquery = new FilteredQueryRA(query, filter, useRandomAccess);
ScoreDoc[] hits = searcher.search (filteredquery, null, 1000).scoreDocs;
assertEquals (1, hits.length);
assertEquals (1, hits[0].doc);
@@ -119,18 +128,18 @@ public class TestFilteredQuery extends L
assertEquals (1, hits.length);
assertEquals (1, hits[0].doc);
- filteredquery = new FilteredQuery (new TermQuery (new Term ("field", "one")), filter);
+ filteredquery = new FilteredQueryRA(new TermQuery (new Term ("field", "one")), filter, useRandomAccess);
hits = searcher.search (filteredquery, null, 1000).scoreDocs;
assertEquals (2, hits.length);
QueryUtils.check(random, filteredquery,searcher);
- filteredquery = new FilteredQuery (new TermQuery (new Term ("field", "x")), filter);
+ filteredquery = new FilteredQueryRA(new TermQuery (new Term ("field", "x")), filter, useRandomAccess);
hits = searcher.search (filteredquery, null, 1000).scoreDocs;
assertEquals (1, hits.length);
assertEquals (3, hits[0].doc);
QueryUtils.check(random, filteredquery,searcher);
- filteredquery = new FilteredQuery (new TermQuery (new Term ("field", "y")), filter);
+ filteredquery = new FilteredQueryRA(new TermQuery (new Term ("field", "y")), filter, useRandomAccess);
hits = searcher.search (filteredquery, null, 1000).scoreDocs;
assertEquals (0, hits.length);
QueryUtils.check(random, filteredquery,searcher);
@@ -147,7 +156,7 @@ public class TestFilteredQuery extends L
BooleanQuery bq2 = new BooleanQuery();
tq = new TermQuery (new Term ("field", "one"));
- filteredquery = new FilteredQuery(tq, f);
+ filteredquery = new FilteredQueryRA(tq, f, useRandomAccess);
filteredquery.setBoost(boost);
bq2.add(filteredquery, Occur.MUST);
bq2.add(new TermQuery (new Term ("field", "five")), Occur.MUST);
@@ -161,7 +170,8 @@ public class TestFilteredQuery extends L
private static Filter newStaticFilterA() {
return new Filter() {
@Override
- public DocIdSet getDocIdSet (AtomicReaderContext context) {
+ public DocIdSet getDocIdSet (AtomicReaderContext context, Bits acceptDocs) {
+ assertNull("acceptDocs should be null, as we have an index without deletions", acceptDocs);
BitSet bitset = new BitSet(5);
bitset.set(0, 5);
return new DocIdBitSet(bitset);
@@ -187,40 +197,112 @@ public class TestFilteredQuery extends L
* This tests FilteredQuery's rewrite correctness
*/
public void testRangeQuery() throws Exception {
+ // force the filter to be executed as bits
+ tRangeQuery(true);
+ tRangeQuery(false);
+ }
+
+ private void tRangeQuery(final boolean useRandomAccess) throws Exception {
TermRangeQuery rq = TermRangeQuery.newStringRange(
"sorter", "b", "d", true, true);
- Query filteredquery = new FilteredQuery(rq, filter);
+ Query filteredquery = new FilteredQueryRA(rq, filter, useRandomAccess);
ScoreDoc[] hits = searcher.search(filteredquery, null, 1000).scoreDocs;
assertEquals(2, hits.length);
QueryUtils.check(random, filteredquery,searcher);
}
- public void testBoolean() throws Exception {
+ public void testBooleanMUST() throws Exception {
+ // force the filter to be executed as bits
+ tBooleanMUST(true);
+ // force the filter to be executed as iterator
+ tBooleanMUST(false);
+ }
+
+ private void tBooleanMUST(final boolean useRandomAccess) throws Exception {
BooleanQuery bq = new BooleanQuery();
- Query query = new FilteredQuery(new MatchAllDocsQuery(),
- new SingleDocTestFilter(0));
+ Query query = new FilteredQueryRA(new MatchAllDocsQuery(), new SingleDocTestFilter(0), useRandomAccess);
bq.add(query, BooleanClause.Occur.MUST);
- query = new FilteredQuery(new MatchAllDocsQuery(),
- new SingleDocTestFilter(1));
+ query = new FilteredQueryRA(new MatchAllDocsQuery(), new SingleDocTestFilter(1), useRandomAccess);
bq.add(query, BooleanClause.Occur.MUST);
ScoreDoc[] hits = searcher.search(bq, null, 1000).scoreDocs;
assertEquals(0, hits.length);
QueryUtils.check(random, query,searcher);
}
+ public void testBooleanSHOULD() throws Exception {
+ // force the filter to be executed as bits
+ tBooleanSHOULD(true);
+ // force the filter to be executed as iterator
+ tBooleanSHOULD(false);
+ }
+
+ private void tBooleanSHOULD(final boolean useRandomAccess) throws Exception {
+ BooleanQuery bq = new BooleanQuery();
+ Query query = new FilteredQueryRA(new MatchAllDocsQuery(), new SingleDocTestFilter(0), useRandomAccess);
+ bq.add(query, BooleanClause.Occur.SHOULD);
+ query = new FilteredQueryRA(new MatchAllDocsQuery(), new SingleDocTestFilter(1), useRandomAccess);
+ bq.add(query, BooleanClause.Occur.SHOULD);
+ ScoreDoc[] hits = searcher.search(bq, null, 1000).scoreDocs;
+ assertEquals(2, hits.length);
+ QueryUtils.check(random, query,searcher);
+ }
+
// Make sure BooleanQuery, which does out-of-order
// scoring, inside FilteredQuery, works
public void testBoolean2() throws Exception {
+ // force the filter to be executed as bits
+ tBoolean2(true);
+ // force the filter to be executed as iterator
+ tBoolean2(false);
+ }
+
+ private void tBoolean2(final boolean useRandomAccess) throws Exception {
BooleanQuery bq = new BooleanQuery();
- Query query = new FilteredQuery(bq,
- new SingleDocTestFilter(0));
+ Query query = new FilteredQueryRA(bq, new SingleDocTestFilter(0), useRandomAccess);
bq.add(new TermQuery(new Term("field", "one")), BooleanClause.Occur.SHOULD);
bq.add(new TermQuery(new Term("field", "two")), BooleanClause.Occur.SHOULD);
ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;
assertEquals(1, hits.length);
QueryUtils.check(random, query, searcher);
}
+
+ public void testChainedFilters() throws Exception {
+ // force the filter to be executed as bits
+ tChainedFilters(true);
+ // force the filter to be executed as iterator
+ tChainedFilters(false);
+ }
+
+ private void tChainedFilters(final boolean useRandomAccess) throws Exception {
+ Query query = new TestFilteredQuery.FilteredQueryRA(new TestFilteredQuery.FilteredQueryRA(
+ new MatchAllDocsQuery(), new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "three")))), useRandomAccess),
+ new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "four")))), useRandomAccess);
+ ScoreDoc[] hits = searcher.search(query, 10).scoreDocs;
+ assertEquals(2, hits.length);
+ QueryUtils.check(random, query, searcher);
+
+ // one more:
+ query = new TestFilteredQuery.FilteredQueryRA(query,
+ new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "five")))), useRandomAccess);
+ hits = searcher.search(query, 10).scoreDocs;
+ assertEquals(1, hits.length);
+ QueryUtils.check(random, query, searcher);
+ }
+
+ public static final class FilteredQueryRA extends FilteredQuery {
+ private final boolean useRandomAccess;
+
+ public FilteredQueryRA(Query q, Filter f, boolean useRandomAccess) {
+ super(q,f);
+ this.useRandomAccess = useRandomAccess;
+ }
+
+ @Override
+ protected boolean useRandomAccess(Bits bits, int firstFilterDoc) {
+ return useRandomAccess;
+ }
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java Thu Nov 3 13:35:07 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
@@ -95,7 +96,8 @@ public class TestFilteredSearch extends
}
@Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) {
+ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) {
+ assertNull("acceptDocs should be null, as we have an index without deletions", acceptDocs);
assert context.isAtomic;
final FixedBitSet set = new FixedBitSet(context.reader.maxDoc());
int docBase = context.docBase;
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java Thu Nov 3 13:35:07 2011
@@ -182,13 +182,13 @@ public class TestNumericRangeQuery32 ext
public void testInverseRange() throws Exception {
AtomicReaderContext context = (AtomicReaderContext) new SlowMultiReaderWrapper(searcher.getIndexReader()).getTopReaderContext();
NumericRangeFilter<Integer> f = NumericRangeFilter.newIntRange("field8", 8, 1000, -1000, true, true);
- assertSame("A inverse range should return the EMPTY_DOCIDSET instance", DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context));
+ assertSame("A inverse range should return the EMPTY_DOCIDSET instance", DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context, context.reader.getLiveDocs()));
f = NumericRangeFilter.newIntRange("field8", 8, Integer.MAX_VALUE, null, false, false);
assertSame("A exclusive range starting with Integer.MAX_VALUE should return the EMPTY_DOCIDSET instance",
- DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context));
+ DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context, context.reader.getLiveDocs()));
f = NumericRangeFilter.newIntRange("field8", 8, null, Integer.MIN_VALUE, false, false);
assertSame("A exclusive range ending with Integer.MIN_VALUE should return the EMPTY_DOCIDSET instance",
- DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context));
+ DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context, context.reader.getLiveDocs()));
}
@Test
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java Thu Nov 3 13:35:07 2011
@@ -188,13 +188,13 @@ public class TestNumericRangeQuery64 ext
AtomicReaderContext context = (AtomicReaderContext) new SlowMultiReaderWrapper(searcher.getIndexReader()).getTopReaderContext();
NumericRangeFilter<Long> f = NumericRangeFilter.newLongRange("field8", 8, 1000L, -1000L, true, true);
assertSame("A inverse range should return the EMPTY_DOCIDSET instance", DocIdSet.EMPTY_DOCIDSET,
- f.getDocIdSet(context));
+ f.getDocIdSet(context, context.reader.getLiveDocs()));
f = NumericRangeFilter.newLongRange("field8", 8, Long.MAX_VALUE, null, false, false);
assertSame("A exclusive range starting with Long.MAX_VALUE should return the EMPTY_DOCIDSET instance",
- DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context));
+ DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context, context.reader.getLiveDocs()));
f = NumericRangeFilter.newLongRange("field8", 8, null, Long.MIN_VALUE, false, false);
assertSame("A exclusive range ending with Long.MIN_VALUE should return the EMPTY_DOCIDSET instance",
- DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context));
+ DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context, context.reader.getLiveDocs()));
}
@Test
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java Thu Nov 3 13:35:07 2011
@@ -55,16 +55,18 @@ public class TestRegexpRandom2 extends L
protected IndexSearcher searcher2;
private IndexReader reader;
private Directory dir;
-
+ protected String fieldName;
+
@Override
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
+ fieldName = random.nextBoolean() ? "field" : ""; // sometimes use an empty string as field name
RandomIndexWriter writer = new RandomIndexWriter(random, dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.KEYWORD, false))
.setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000)));
Document doc = new Document();
- Field field = newField("field", "", StringField.TYPE_UNSTORED);
+ Field field = newField(fieldName, "", StringField.TYPE_UNSTORED);
doc.add(field);
List<String> terms = new ArrayList<String>();
int num = atLeast(200);
@@ -141,7 +143,7 @@ public class TestRegexpRandom2 extends L
public void testRegexps() throws Exception {
// we generate aweful regexps: good for testing.
// but for preflex codec, the test can be very slow, so use less iterations.
- int num = CodecProvider.getDefault().getFieldCodec("field").equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : atLeast(1000);
+ int num = CodecProvider.getDefault().getFieldCodec(fieldName).equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : atLeast(1000);
for (int i = 0; i < num; i++) {
String reg = AutomatonTestUtil.randomRegexp(random);
if (VERBOSE) {
@@ -155,8 +157,8 @@ public class TestRegexpRandom2 extends L
* simple regexpquery implementation.
*/
protected void assertSame(String regexp) throws IOException {
- RegexpQuery smart = new RegexpQuery(new Term("field", regexp), RegExp.NONE);
- DumbRegexpQuery dumb = new DumbRegexpQuery(new Term("field", regexp), RegExp.NONE);
+ RegexpQuery smart = new RegexpQuery(new Term(fieldName, regexp), RegExp.NONE);
+ DumbRegexpQuery dumb = new DumbRegexpQuery(new Term(fieldName, regexp), RegExp.NONE);
TopDocs smartDocs = searcher1.search(smart, 25);
TopDocs dumbDocs = searcher2.search(dumb, 25);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestScorerPerf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestScorerPerf.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestScorerPerf.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestScorerPerf.java Thu Nov 3 13:35:07 2011
@@ -1,5 +1,6 @@
package org.apache.lucene.search;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.DocIdBitSet;
import org.apache.lucene.util.LuceneTestCase;
@@ -141,7 +142,8 @@ public class TestScorerPerf extends Luce
final BitSet rnd = sets[random.nextInt(sets.length)];
Query q = new ConstantScoreQuery(new Filter() {
@Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) {
+ public DocIdSet getDocIdSet (AtomicReaderContext context, Bits acceptDocs) {
+ assertNull("acceptDocs should be null, as we have an index without deletions", acceptDocs);
return new DocIdBitSet(rnd);
}
});
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSort.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSort.java Thu Nov 3 13:35:07 2011
@@ -53,6 +53,7 @@ import org.apache.lucene.search.cache.Lo
import org.apache.lucene.search.cache.ShortValuesCreator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.DocIdBitSet;
import org.apache.lucene.util.LuceneTestCase;
@@ -730,7 +731,8 @@ public class TestSort extends LuceneTest
// a filter that only allows through the first hit
Filter filt = new Filter() {
@Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException {
+ public DocIdSet getDocIdSet (AtomicReaderContext context, Bits acceptDocs) {
+ assertNull("acceptDocs should be null, as we have no deletions", acceptDocs);
BitSet bs = new BitSet(context.reader.maxDoc());
bs.set(0, context.reader.maxDoc());
bs.set(docs1.scoreDocs[0].doc);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSpanQueryFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSpanQueryFilter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSpanQueryFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSpanQueryFilter.java Thu Nov 3 13:35:07 2011
@@ -49,7 +49,7 @@ public class TestSpanQueryFilter extends
int subIndex = ReaderUtil.subIndex(number, leaves); // find the reader with this document in it
SpanTermQuery query = new SpanTermQuery(new Term("field", English.intToEnglish(number).trim()));
SpanQueryFilter filter = new SpanQueryFilter(query);
- SpanFilterResult result = filter.bitSpans(leaves[subIndex]);
+ SpanFilterResult result = filter.bitSpans(leaves[subIndex], leaves[subIndex].reader.getLiveDocs());
DocIdSet docIdSet = result.getDocIdSet();
assertTrue("docIdSet is null and it shouldn't be", docIdSet != null);
assertContainsDocId("docIdSet doesn't contain docId 10", docIdSet, number - leaves[subIndex].docBase);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestTopDocsMerge.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestTopDocsMerge.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestTopDocsMerge.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestTopDocsMerge.java Thu Nov 3 13:35:07 2011
@@ -44,11 +44,11 @@ public class TestTopDocsMerge extends Lu
}
public void search(Weight weight, Collector collector) throws IOException {
- search(ctx, weight, null, collector);
+ search(ctx, weight, collector);
}
public TopDocs search(Weight weight, int topN) throws IOException {
- return search(ctx, weight, null, null, topN);
+ return search(ctx, weight, null, topN);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java Thu Nov 3 13:35:07 2011
@@ -203,12 +203,18 @@ public class SpoofIndexSearcher extends
// ------------------------ Not implemented methods ------------------------
+
@Override
public TermsEnum iterator() throws IOException {
return null;
}
@Override
+ public long getUniqueTermCount() throws IOException {
+ return -1;
+ }
+
+ @Override
public Comparator<BytesRef> getComparator() throws IOException {
return null;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/store/TestCopyBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/store/TestCopyBytes.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/store/TestCopyBytes.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/store/TestCopyBytes.java Thu Nov 3 13:35:07 2011
@@ -17,6 +17,8 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -104,4 +106,68 @@ public class TestCopyBytes extends Lucen
dir.close();
}
}
+
+ // LUCENE-3541
+ public void testCopyBytesWithThreads() throws Exception {
+ int datalen = _TestUtil.nextInt(random, 101, 10000);
+ byte data[] = new byte[datalen];
+ random.nextBytes(data);
+
+ Directory d = newDirectory();
+ IndexOutput output = d.createOutput("data", IOContext.DEFAULT);
+ output.writeBytes(data, 0, datalen);
+ output.close();
+
+ IndexInput input = d.openInput("data", IOContext.DEFAULT);
+ IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT);
+ // copy our 100-byte header
+ input.copyBytes(outputHeader, 100);
+ outputHeader.close();
+
+ // now make N copies of the remaining bytes
+ CopyThread copies[] = new CopyThread[10];
+ for (int i = 0; i < copies.length; i++) {
+ copies[i] = new CopyThread((IndexInput) input.clone(), d.createOutput("copy" + i, IOContext.DEFAULT));
+ }
+
+ for (int i = 0; i < copies.length; i++) {
+ copies[i].start();
+ }
+
+ for (int i = 0; i < copies.length; i++) {
+ copies[i].join();
+ }
+
+ for (int i = 0; i < copies.length; i++) {
+ IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT);
+ byte[] dataCopy = new byte[datalen];
+ System.arraycopy(data, 0, dataCopy, 0, 100); // copy the header for easy testing
+ copiedData.readBytes(dataCopy, 100, datalen-100);
+ assertArrayEquals(data, dataCopy);
+ copiedData.close();
+ }
+ input.close();
+ d.close();
+
+ }
+
+ static class CopyThread extends Thread {
+ final IndexInput src;
+ final IndexOutput dst;
+
+ CopyThread(IndexInput src, IndexOutput dst) {
+ this.src = src;
+ this.dst = dst;
+ }
+
+ @Override
+ public void run() {
+ try {
+ src.copyBytes(dst, src.length()-100);
+ dst.close();
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestCharsRef.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestCharsRef.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestCharsRef.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestCharsRef.java Thu Nov 3 13:35:07 2011
@@ -38,4 +38,33 @@ public class TestCharsRef extends Lucene
assertEquals(utf8[i].utf8ToString(), utf16[i].toString());
}
}
+
+ public void testAppend() {
+ CharsRef ref = new CharsRef();
+ StringBuilder builder = new StringBuilder();
+ int numStrings = atLeast(10);
+ for (int i = 0; i < numStrings; i++) {
+ char[] charArray = _TestUtil.randomRealisticUnicodeString(random, 1, 100).toCharArray();
+ int offset = random.nextInt(charArray.length);
+ int length = charArray.length - offset;
+ builder.append(charArray, offset, length);
+ ref.append(charArray, offset, length);
+ }
+
+ assertEquals(builder.toString(), ref.toString());
+ }
+
+ public void testCopy() {
+ int numIters = atLeast(10);
+ for (int i = 0; i < numIters; i++) {
+ CharsRef ref = new CharsRef();
+ char[] charArray = _TestUtil.randomRealisticUnicodeString(random, 1, 100).toCharArray();
+ int offset = random.nextInt(charArray.length);
+ int length = charArray.length - offset;
+ String str = new String(charArray, offset, length);
+ ref.copy(charArray, offset, length);
+ assertEquals(str, ref.toString());
+ }
+
+ }
}
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java Thu Nov 3 13:35:07 2011
@@ -37,7 +37,7 @@ public final class CJKAnalyzer extends S
/**
* File containing default CJK stopwords.
* <p/>
- * Currently it concains some common English words that are not usually
+ * Currently it contains some common English words that are not usually
* useful for searching and some double-byte interpunctions.
*/
public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java Thu Nov 3 13:35:07 2011
@@ -24,20 +24,17 @@ import java.util.LinkedList;
import java.util.Locale;
import java.util.Set;
-import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
- * Base class for decomposition token filters. <a name="version"/>
+ * Base class for decomposition token filters.
* <p>
* You must specify the required {@link Version} compatibility when creating
* CompoundWordTokenFilterBase:
@@ -46,6 +43,13 @@ import org.apache.lucene.util.Version;
* supplementary characters in strings and char arrays provided as compound word
* dictionaries.
* </ul>
+ * <p>If you pass in a {@link org.apache.lucene.analysis.util.CharArraySet} as dictionary,
+ * it should be case-insensitive unless it contains only lowercased entries and you
+ * have {@link org.apache.lucene.analysis.core.LowerCaseFilter} before this filter in your analysis chain.
+ * For optional performance (as this filter does lots of lookups to the dictionary,
+ * you should use the latter analysis chain/CharArraySet). Be aware: If you supply arbitrary
+ * {@link Set Sets} to the ctors, they will be automatically
+ * transformed to case-insensitive!
*/
public abstract class CompoundWordTokenFilterBase extends TokenFilter {
/**
@@ -64,37 +68,22 @@ public abstract class CompoundWordTokenF
public static final int DEFAULT_MAX_SUBWORD_SIZE = 15;
protected final CharArraySet dictionary;
- protected final LinkedList<Token> tokens;
+ protected final LinkedList<CompoundToken> tokens;
protected final int minWordSize;
protected final int minSubwordSize;
protected final int maxSubwordSize;
protected final boolean onlyLongestMatch;
- private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
- private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
+ protected final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ protected final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
- private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
- private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
-
- private final Token wrapper = new Token();
-
- protected CompoundWordTokenFilterBase(Version matchVersion, TokenStream input, String[] dictionary, int minWordSize, int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
- this(matchVersion, input,makeDictionary(dictionary),minWordSize,minSubwordSize,maxSubwordSize, onlyLongestMatch);
- }
- protected CompoundWordTokenFilterBase(Version matchVersion, TokenStream input, String[] dictionary, boolean onlyLongestMatch) {
- this(matchVersion, input,makeDictionary(dictionary),DEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE, onlyLongestMatch);
- }
+ private AttributeSource.State current;
protected CompoundWordTokenFilterBase(Version matchVersion, TokenStream input, Set<?> dictionary, boolean onlyLongestMatch) {
this(matchVersion, input,dictionary,DEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE, onlyLongestMatch);
}
- protected CompoundWordTokenFilterBase(Version matchVersion, TokenStream input, String[] dictionary) {
- this(matchVersion, input,makeDictionary(dictionary),DEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE, false);
- }
-
protected CompoundWordTokenFilterBase(Version matchVersion, TokenStream input, Set<?> dictionary) {
this(matchVersion, input,dictionary,DEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE, false);
}
@@ -102,7 +91,7 @@ public abstract class CompoundWordTokenF
protected CompoundWordTokenFilterBase(Version matchVersion, TokenStream input, Set<?> dictionary, int minWordSize, int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
super(input);
- this.tokens=new LinkedList<Token>();
+ this.tokens=new LinkedList<CompoundToken>();
this.minWordSize=minWordSize;
this.minSubwordSize=minSubwordSize;
this.maxSubwordSize=maxSubwordSize;
@@ -111,113 +100,68 @@ public abstract class CompoundWordTokenF
if (dictionary==null || dictionary instanceof CharArraySet) {
this.dictionary = (CharArraySet) dictionary;
} else {
- this.dictionary = new CharArraySet(matchVersion, dictionary.size(), false);
- addAllLowerCase(this.dictionary, dictionary);
+ this.dictionary = new CharArraySet(matchVersion, dictionary, true);
}
}
-
- /**
- * Create a set of words from an array
- * The resulting Set does case insensitive matching
- * TODO We should look for a faster dictionary lookup approach.
- * @param dictionary
- * @return {@link Set} of lowercased terms
- */
- public static Set<?> makeDictionary(final String[] dictionary) {
- return makeDictionary(Version.LUCENE_30, dictionary);
- }
-
- public static Set<?> makeDictionary(final Version matchVersion, final String[] dictionary) {
- if (dictionary == null) {
- return null;
- }
- // is the below really case insensitive?
- CharArraySet dict = new CharArraySet(matchVersion, dictionary.length, false);
- addAllLowerCase(dict, Arrays.asList(dictionary));
- return dict;
- }
-
- private void setToken(final Token token) throws IOException {
- clearAttributes();
- termAtt.copyBuffer(token.buffer(), 0, token.length());
- flagsAtt.setFlags(token.getFlags());
- typeAtt.setType(token.type());
- offsetAtt.setOffset(token.startOffset(), token.endOffset());
- posIncAtt.setPositionIncrement(token.getPositionIncrement());
- payloadAtt.setPayload(token.getPayload());
- }
@Override
public final boolean incrementToken() throws IOException {
- if (tokens.size() > 0) {
- setToken(tokens.removeFirst());
+ if (!tokens.isEmpty()) {
+ assert current != null;
+ CompoundToken token = tokens.removeFirst();
+ restoreState(current); // keep all other attributes untouched
+ termAtt.setEmpty().append(token.txt);
+ offsetAtt.setOffset(token.startOffset, token.endOffset);
+ posIncAtt.setPositionIncrement(0);
return true;
}
- if (!input.incrementToken())
- return false;
-
- wrapper.copyBuffer(termAtt.buffer(), 0, termAtt.length());
- wrapper.setStartOffset(offsetAtt.startOffset());
- wrapper.setEndOffset(offsetAtt.endOffset());
- wrapper.setFlags(flagsAtt.getFlags());
- wrapper.setType(typeAtt.type());
- wrapper.setPositionIncrement(posIncAtt.getPositionIncrement());
- wrapper.setPayload(payloadAtt.getPayload());
-
- decompose(wrapper);
-
- if (tokens.size() > 0) {
- setToken(tokens.removeFirst());
+ current = null; // not really needed, but for safety
+ if (input.incrementToken()) {
+ // Only words longer than minWordSize get processed
+ if (termAtt.length() >= this.minWordSize) {
+ decompose();
+ // only capture the state if we really need it for producing new tokens
+ if (!tokens.isEmpty()) {
+ current = captureState();
+ }
+ }
+ // return original token:
return true;
} else {
return false;
}
}
-
- protected static void addAllLowerCase(CharArraySet target, Collection<?> col) {
- for (Object obj : col) {
- String string = (String) obj;
- target.add(string.toLowerCase(Locale.ENGLISH));
- }
- }
-
- protected static char[] makeLowerCaseCopy(final char[] buffer) {
- char[] result=new char[buffer.length];
- System.arraycopy(buffer, 0, result, 0, buffer.length);
-
- for (int i=0;i<buffer.length;++i) {
- result[i]=Character.toLowerCase(buffer[i]);
- }
-
- return result;
- }
-
- protected final Token createToken(final int offset, final int length,
- final Token prototype) {
- int newStart = prototype.startOffset() + offset;
- Token t = prototype.clone(prototype.buffer(), offset, length, newStart, newStart+length);
- t.setPositionIncrement(0);
- return t;
- }
- protected void decompose(final Token token) {
- // In any case we give the original token back
- tokens.add((Token) token.clone());
-
- // Only words longer than minWordSize get processed
- if (token.length() < this.minWordSize) {
- return;
- }
-
- decomposeInternal(token);
- }
-
- protected abstract void decomposeInternal(final Token token);
+ /** Decomposes the current {@link #termAtt} and places {@link CompoundToken} instances in the {@link #tokens} list.
+ * The original token may not be placed in the list, as it is automatically passed through this filter.
+ */
+ protected abstract void decompose();
@Override
public void reset() throws IOException {
super.reset();
tokens.clear();
+ current = null;
}
+
+ /**
+ * Helper class to hold decompounded token information
+ */
+ protected class CompoundToken {
+ public final CharSequence txt;
+ public final int startOffset, endOffset;
+
+ /** Construct the compound token based on a slice of the current {@link CompoundWordTokenFilterBase#termAtt}. */
+ public CompoundToken(int offset, int length) {
+ final int newStart = CompoundWordTokenFilterBase.this.offsetAtt.startOffset() + offset;
+ this.txt = CompoundWordTokenFilterBase.this.termAtt.subSequence(offset, offset + length);
+ // TODO: This ignores the original endOffset, if a CharFilter/Tokenizer/Filter removed
+ // chars from the term, offsets may not match correctly (other filters producing tokens
+ // may also have this problem):
+ this.startOffset = newStart;
+ this.endOffset = newStart + length;
+ }
+
+ }
}
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java Thu Nov 3 13:35:07 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.analysis.compo
import java.util.Set;
-import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.util.Version;
@@ -31,52 +30,23 @@ import org.apache.lucene.util.Version;
* "Donaudampfschiff" becomes Donau, dampf, schiff so that you can find
* "Donaudampfschiff" even when you only enter "schiff".
* It uses a brute-force algorithm to achieve this.
- * </p>
+ * <p>
+ * You must specify the required {@link Version} compatibility when creating
+ * CompoundWordTokenFilterBase:
+ * <ul>
+ * <li>As of 3.1, CompoundWordTokenFilterBase correctly handles Unicode 4.0
+ * supplementary characters in strings and char arrays provided as compound word
+ * dictionaries.
+ * </ul>
+ * <p>If you pass in a {@link org.apache.lucene.analysis.util.CharArraySet} as dictionary,
+ * it should be case-insensitive unless it contains only lowercased entries and you
+ * have {@link org.apache.lucene.analysis.core.LowerCaseFilter} before this filter in your analysis chain.
+ * For optional performance (as this filter does lots of lookups to the dictionary,
+ * you should use the latter analysis chain/CharArraySet). Be aware: If you supply arbitrary
+ * {@link Set Sets} to the ctors, they will be automatically
+ * transformed to case-insensitive!
*/
public class DictionaryCompoundWordTokenFilter extends CompoundWordTokenFilterBase {
- /**
- * Creates a new {@link DictionaryCompoundWordTokenFilter}
- *
- * @param matchVersion
- * Lucene version to enable correct Unicode 4.0 behavior in the
- * dictionaries if Version > 3.0. See <a
- * href="CompoundWordTokenFilterBase#version"
- * >CompoundWordTokenFilterBase</a> for details.
- * @param input
- * the {@link TokenStream} to process
- * @param dictionary
- * the word dictionary to match against
- * @param minWordSize
- * only words longer than this get processed
- * @param minSubwordSize
- * only subwords longer than this get to the output stream
- * @param maxSubwordSize
- * only subwords shorter than this get to the output stream
- * @param onlyLongestMatch
- * Add only the longest matching subword to the stream
- */
- public DictionaryCompoundWordTokenFilter(Version matchVersion, TokenStream input, String[] dictionary,
- int minWordSize, int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
- super(matchVersion, input, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);
- }
-
- /**
- * Creates a new {@link DictionaryCompoundWordTokenFilter}
- *
- * @param matchVersion
- * Lucene version to enable correct Unicode 4.0 behavior in the
- * dictionaries if Version > 3.0. See <a
- * href="CompoundWordTokenFilterBase#version"
- * >CompoundWordTokenFilterBase</a> for details.
- *
- * @param input
- * the {@link TokenStream} to process
- * @param dictionary
- * the word dictionary to match against
- */
- public DictionaryCompoundWordTokenFilter(Version matchVersion, TokenStream input, String[] dictionary) {
- super(matchVersion, input, dictionary);
- }
/**
* Creates a new {@link DictionaryCompoundWordTokenFilter}
@@ -89,12 +59,9 @@ public class DictionaryCompoundWordToken
* @param input
* the {@link TokenStream} to process
* @param dictionary
- * the word dictionary to match against. If this is a
- * {@link org.apache.lucene.analysis.util.CharArraySet CharArraySet} it
- * must have set ignoreCase=false and only contain lower case
- * strings.
+ * the word dictionary to match against.
*/
- public DictionaryCompoundWordTokenFilter(Version matchVersion, TokenStream input, Set dictionary) {
+ public DictionaryCompoundWordTokenFilter(Version matchVersion, TokenStream input, Set<?> dictionary) {
super(matchVersion, input, dictionary);
}
@@ -109,10 +76,7 @@ public class DictionaryCompoundWordToken
* @param input
* the {@link TokenStream} to process
* @param dictionary
- * the word dictionary to match against. If this is a
- * {@link org.apache.lucene.analysis.util.CharArraySet CharArraySet} it
- * must have set ignoreCase=false and only contain lower case
- * strings.
+ * the word dictionary to match against.
* @param minWordSize
* only words longer than this get processed
* @param minSubwordSize
@@ -122,37 +86,31 @@ public class DictionaryCompoundWordToken
* @param onlyLongestMatch
* Add only the longest matching subword to the stream
*/
- public DictionaryCompoundWordTokenFilter(Version matchVersion, TokenStream input, Set dictionary,
+ public DictionaryCompoundWordTokenFilter(Version matchVersion, TokenStream input, Set<?> dictionary,
int minWordSize, int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
super(matchVersion, input, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);
}
@Override
- protected void decomposeInternal(final Token token) {
- // Only words longer than minWordSize get processed
- if (token.length() < this.minWordSize) {
- return;
- }
-
- char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.buffer());
-
- for (int i=0;i<=token.length()-this.minSubwordSize;++i) {
- Token longestMatchToken=null;
+ protected void decompose() {
+ final int len = termAtt.length();
+ for (int i=0;i<=len-this.minSubwordSize;++i) {
+ CompoundToken longestMatchToken=null;
for (int j=this.minSubwordSize;j<=this.maxSubwordSize;++j) {
- if(i+j>token.length()) {
+ if(i+j>len) {
break;
}
- if(dictionary.contains(lowerCaseTermBuffer, i, j)) {
+ if(dictionary.contains(termAtt.buffer(), i, j)) {
if (this.onlyLongestMatch) {
if (longestMatchToken!=null) {
- if (longestMatchToken.length()<j) {
- longestMatchToken=createToken(i,j,token);
+ if (longestMatchToken.txt.length()<j) {
+ longestMatchToken=new CompoundToken(i,j);
}
} else {
- longestMatchToken=createToken(i,j,token);
+ longestMatchToken=new CompoundToken(i,j);
}
} else {
- tokens.add(createToken(i,j,token));
+ tokens.add(new CompoundToken(i,j));
}
}
}
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java Thu Nov 3 13:35:07 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.analysis.compo
import java.io.File;
import java.util.Set;
-import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.compound.hyphenation.Hyphenation;
@@ -34,64 +33,25 @@ import org.xml.sax.InputSource;
* "Donaudampfschiff" becomes Donau, dampf, schiff so that you can find
* "Donaudampfschiff" even when you only enter "schiff". It uses a hyphenation
* grammar and a word dictionary to achieve this.
- * </p>
+ * <p>
+ * You must specify the required {@link Version} compatibility when creating
+ * CompoundWordTokenFilterBase:
+ * <ul>
+ * <li>As of 3.1, CompoundWordTokenFilterBase correctly handles Unicode 4.0
+ * supplementary characters in strings and char arrays provided as compound word
+ * dictionaries.
+ * </ul>
+ * <p>If you pass in a {@link org.apache.lucene.analysis.util.CharArraySet} as dictionary,
+ * it should be case-insensitive unless it contains only lowercased entries and you
+ * have {@link org.apache.lucene.analysis.core.LowerCaseFilter} before this filter in your analysis chain.
+ * For optional performance (as this filter does lots of lookups to the dictionary,
+ * you should use the latter analysis chain/CharArraySet). Be aware: If you supply arbitrary
+ * {@link Set Sets} to the ctors, they will be automatically
+ * transformed to case-insensitive!
*/
public class HyphenationCompoundWordTokenFilter extends
CompoundWordTokenFilterBase {
private HyphenationTree hyphenator;
-
- /**
- * Creates a new {@link HyphenationCompoundWordTokenFilter} instance.
- *
- * @param matchVersion
- * Lucene version to enable correct Unicode 4.0 behavior in the
- * dictionaries if Version > 3.0. See <a
- * href="CompoundWordTokenFilterBase#version"
- * >CompoundWordTokenFilterBase</a> for details.
- * @param input
- * the {@link TokenStream} to process
- * @param hyphenator
- * the hyphenation pattern tree to use for hyphenation
- * @param dictionary
- * the word dictionary to match against
- * @param minWordSize
- * only words longer than this get processed
- * @param minSubwordSize
- * only subwords longer than this get to the output stream
- * @param maxSubwordSize
- * only subwords shorter than this get to the output stream
- * @param onlyLongestMatch
- * Add only the longest matching subword to the stream
- */
- public HyphenationCompoundWordTokenFilter(Version matchVersion, TokenStream input,
- HyphenationTree hyphenator, String[] dictionary, int minWordSize,
- int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
- super(matchVersion, input, dictionary, minWordSize, minSubwordSize, maxSubwordSize,
- onlyLongestMatch);
-
- this.hyphenator = hyphenator;
- }
-
- /**
- * Creates a new {@link HyphenationCompoundWordTokenFilter} instance.
- *
- * @param matchVersion
- * Lucene version to enable correct Unicode 4.0 behavior in the
- * dictionaries if Version > 3.0. See <a
- * href="CompoundWordTokenFilterBase#version"
- * >CompoundWordTokenFilterBase</a> for details.
- * @param input
- * the {@link TokenStream} to process
- * @param hyphenator
- * the hyphenation pattern tree to use for hyphenation
- * @param dictionary
- * the word dictionary to match against
- */
- public HyphenationCompoundWordTokenFilter(Version matchVersion, TokenStream input,
- HyphenationTree hyphenator, String[] dictionary) {
- this(matchVersion, input, hyphenator, makeDictionary(dictionary), DEFAULT_MIN_WORD_SIZE,
- DEFAULT_MIN_SUBWORD_SIZE, DEFAULT_MAX_SUBWORD_SIZE, false);
- }
/**
* Creates a new {@link HyphenationCompoundWordTokenFilter} instance.
@@ -106,10 +66,7 @@ public class HyphenationCompoundWordToke
* @param hyphenator
* the hyphenation pattern tree to use for hyphenation
* @param dictionary
- * the word dictionary to match against. If this is a
- * {@link org.apache.lucene.analysis.util.CharArraySet CharArraySet} it
- * must have set ignoreCase=false and only contain lower case
- * strings.
+ * the word dictionary to match against.
*/
public HyphenationCompoundWordTokenFilter(Version matchVersion, TokenStream input,
HyphenationTree hyphenator, Set<?> dictionary) {
@@ -130,10 +87,7 @@ public class HyphenationCompoundWordToke
* @param hyphenator
* the hyphenation pattern tree to use for hyphenation
* @param dictionary
- * the word dictionary to match against. If this is a
- * {@link org.apache.lucene.analysis.util.CharArraySet CharArraySet} it
- * must have set ignoreCase=false and only contain lower case
- * strings.
+ * the word dictionary to match against.
* @param minWordSize
* only words longer than this get processed
* @param minSubwordSize
@@ -218,22 +172,20 @@ public class HyphenationCompoundWordToke
}
@Override
- protected void decomposeInternal(final Token token) {
+ protected void decompose() {
// get the hyphenation points
- Hyphenation hyphens = hyphenator.hyphenate(token.buffer(), 0, token
- .length(), 1, 1);
+ Hyphenation hyphens = hyphenator.hyphenate(termAtt.buffer(), 0, termAtt.length(), 1, 1);
// No hyphen points found -> exit
if (hyphens == null) {
return;
}
final int[] hyp = hyphens.getHyphenationPoints();
- char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.buffer());
for (int i = 0; i < hyp.length; ++i) {
int remaining = hyp.length - i;
int start = hyp[i];
- Token longestMatchToken = null;
+ CompoundToken longestMatchToken = null;
for (int j = 1; j < remaining; j++) {
int partLength = hyp[i + j] - start;
@@ -250,34 +202,33 @@ public class HyphenationCompoundWordToke
}
// check the dictionary
- if (dictionary == null || dictionary.contains(lowerCaseTermBuffer, start, partLength)) {
+ if (dictionary == null || dictionary.contains(termAtt.buffer(), start, partLength)) {
if (this.onlyLongestMatch) {
if (longestMatchToken != null) {
- if (longestMatchToken.length() < partLength) {
- longestMatchToken = createToken(start, partLength, token);
+ if (longestMatchToken.txt.length() < partLength) {
+ longestMatchToken = new CompoundToken(start, partLength);
}
} else {
- longestMatchToken = createToken(start, partLength, token);
+ longestMatchToken = new CompoundToken(start, partLength);
}
} else {
- tokens.add(createToken(start, partLength, token));
+ tokens.add(new CompoundToken(start, partLength));
}
- } else if (dictionary.contains(lowerCaseTermBuffer, start,
- partLength - 1)) {
+ } else if (dictionary.contains(termAtt.buffer(), start, partLength - 1)) {
// check the dictionary again with a word that is one character
// shorter
// to avoid problems with genitive 's characters and other binding
// characters
if (this.onlyLongestMatch) {
if (longestMatchToken != null) {
- if (longestMatchToken.length() < partLength - 1) {
- longestMatchToken = createToken(start, partLength - 1, token);
+ if (longestMatchToken.txt.length() < partLength - 1) {
+ longestMatchToken = new CompoundToken(start, partLength - 1);
}
} else {
- longestMatchToken = createToken(start, partLength - 1, token);
+ longestMatchToken = new CompoundToken(start, partLength - 1);
}
} else {
- tokens.add(createToken(start, partLength - 1, token));
+ tokens.add(new CompoundToken(start, partLength - 1));
}
}
}
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java Thu Nov 3 13:35:07 2011
@@ -225,7 +225,7 @@ public class HunspellDictionary {
}
/**
- * Parses the encoding specificed in the affix file readable through the provided InputStream
+ * Parses the encoding specified in the affix file readable through the provided InputStream
*
* @param affix InputStream for reading the affix file
* @return Encoding specified in the affix file
@@ -277,10 +277,10 @@ public class HunspellDictionary {
}
/**
- * Determines the appropriate {@link FlagParsingStrategy} based on the FLAG definiton line taken from the affix file
+ * Determines the appropriate {@link FlagParsingStrategy} based on the FLAG definition line taken from the affix file
*
* @param flagLine Line containing the flag information
- * @return FlagParsingStrategy that handles parsing flags in the way specified in the FLAG definiton
+ * @return FlagParsingStrategy that handles parsing flags in the way specified in the FLAG definition
*/
private FlagParsingStrategy getFlagParsingStrategy(String flagLine) {
String flagType = flagLine.substring(5);
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellWord.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellWord.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellWord.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellWord.java Thu Nov 3 13:35:07 2011
@@ -52,7 +52,7 @@ public class HunspellWord {
/**
* Returns the flags associated with the word
*
- * @return Flags asssociated with the word
+ * @return Flags associated with the word
*/
public char[] getFlags() {
return flags;
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java Thu Nov 3 13:35:07 2011
@@ -65,13 +65,13 @@ import org.apache.lucene.util.fst.FST;
* <p><b>NOTE</b>: when a match occurs, the output tokens
* associated with the matching rule are "stacked" on top of
* the input stream (if the rule had
- * <code>keepOrig=true</code>) and also on top of aother
+ * <code>keepOrig=true</code>) and also on top of another
* matched rule's output tokens. This is not a correct
- * solution, as really the output should be an abitrary
+ * solution, as really the output should be an arbitrary
* graph/lattice. For example, with the above match, you
* would expect an exact <code>PhraseQuery</code> <code>"y b
* c"</code> to match the parsed tokens, but it will fail to
- * do so. This limitations is necessary because Lucene's
+ * do so. This limitation is necessary because Lucene's
* TokenStream (and index) cannot yet represent an arbitrary
* graph.</p>
*
@@ -90,7 +90,7 @@ import org.apache.lucene.util.fst.FST;
// http://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_string_matching_algorithm
// It improves over the current approach here
// because it does not fully re-start matching at every
-// token. For exampl,e if one pattern is "a b c x"
+// token. For example if one pattern is "a b c x"
// and another is "b c d" and the input is "a b c d", on
// trying to parse "a b c x" but failing when you got to x,
// rather than starting over again your really should
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java Thu Nov 3 13:35:07 2011
@@ -44,9 +44,9 @@ import org.apache.lucene.util.fst.FST;
public class SynonymMap {
/** for multiword support, you must separate words with this separator */
public static final char WORD_SEPARATOR = 0;
- /** map<input word, list<ord>> */
+ /** map<input word, list<ord>> */
public final FST<BytesRef> fst;
- /** map<ord, outputword> */
+ /** map<ord, outputword> */
public final BytesRefHash words;
/** maxHorizontalContext: maximum context we need on the tokenstream */
public final int maxHorizontalContext;
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java Thu Nov 3 13:35:07 2011
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.lang.Character.UnicodeBlock;
import java.text.BreakIterator;
import java.util.Locale;
-import javax.swing.text.Segment;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -28,6 +27,7 @@ import org.apache.lucene.analysis.core.L
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.util.CharArrayIterator;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
@@ -56,7 +56,7 @@ public final class ThaiWordFilter extend
DBBI_AVAILABLE = proto.isBoundary(4);
}
private final BreakIterator breaker = (BreakIterator) proto.clone();
- private final Segment charIterator = new Segment();
+ private final CharArrayIterator charIterator = CharArrayIterator.newWordInstance();
private final boolean handlePosIncr;
@@ -113,9 +113,7 @@ public final class ThaiWordFilter extend
}
// reinit CharacterIterator
- charIterator.array = clonedTermAtt.buffer();
- charIterator.offset = 0;
- charIterator.count = clonedTermAtt.length();
+ charIterator.setText(clonedTermAtt.buffer(), 0, clonedTermAtt.length());
breaker.setText(charIterator);
int end = breaker.next();
if (end != BreakIterator.DONE) {
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java Thu Nov 3 13:35:07 2011
@@ -34,7 +34,7 @@ public class CollatedTermAttributeImpl e
* @param collator Collation key generator
*/
public CollatedTermAttributeImpl(Collator collator) {
- // clone in case JRE doesnt properly sync,
+ // clone in case JRE doesn't properly sync,
// or to reduce contention in case they do
this.collator = (Collator) collator.clone();
}