You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/07/18 15:35:33 UTC
svn commit: r1147873 [9/13] - in /lucene/dev/branches/positions: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/lucene/contrib/demo/ dev-tools/idea/lucene/contrib/memory/
dev-tools/idea/lucene/contrib/querypar...
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestFieldCache.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestFieldCache.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestFieldCache.java Mon Jul 18 13:34:29 2011
@@ -33,13 +33,14 @@ import java.io.PrintStream;
public class TestFieldCache extends LuceneTestCase {
protected IndexReader reader;
- private static final int NUM_DOCS = atLeast(1000);
+ private int NUM_DOCS;
private String[] unicodeStrings;
private Directory directory;
@Override
public void setUp() throws Exception {
super.setUp();
+ NUM_DOCS = atLeast(1000);
directory = newDirectory();
RandomIndexWriter writer= new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
long theLong = Long.MAX_VALUE;
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java Mon Jul 18 13:34:29 2011
@@ -30,7 +30,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
@@ -97,7 +97,7 @@ public class TestFilteredSearch extends
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context) {
assert context.isAtomic;
- final OpenBitSet set = new OpenBitSet();
+ final FixedBitSet set = new FixedBitSet(context.reader.maxDoc());
int docBase = context.docBase;
final int limit = docBase+context.reader.maxDoc();
for (;index < docs.length; index++) {
@@ -108,7 +108,7 @@ public class TestFilteredSearch extends
set.set(docId-docBase);
}
}
- return set.isEmpty()?null:set;
+ return set.cardinality() == 0 ? null:set;
}
public void reset(){
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java Mon Jul 18 13:34:29 2011
@@ -23,7 +23,6 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiFields;
-import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
@@ -465,7 +464,7 @@ public class TestMultiPhraseQuery extend
* using query parser, MPQ will be created, and will not be strict about having all query terms
* in each position - one of each position is sufficient (OR logic)
*/
- public void testZeroPosIncrSloppyParsedAnd() throws IOException, ParseException {
+ public void testZeroPosIncrSloppyParsedAnd() throws IOException {
MultiPhraseQuery q = new MultiPhraseQuery();
q.add(new Term[]{ new Term("field", "a"), new Term("field", "1") }, -1);
q.add(new Term[]{ new Term("field", "b"), new Term("field", "1") }, 0);
@@ -509,7 +508,7 @@ public class TestMultiPhraseQuery extend
/**
* PQ AND Mode - Manually creating a phrase query
*/
- public void testZeroPosIncrSloppyPqAnd() throws IOException, ParseException {
+ public void testZeroPosIncrSloppyPqAnd() throws IOException {
final PhraseQuery pq = new PhraseQuery();
for (TokenAndPos tap : INCR_0_QUERY_TOKENS_AND) {
pq.add(new Term("field",tap.token), tap.pos);
@@ -524,7 +523,7 @@ public class TestMultiPhraseQuery extend
/**
* MPQ AND Mode - Manually creating a multiple phrase query
*/
- public void testZeroPosIncrSloppyMpqAnd() throws IOException, ParseException {
+ public void testZeroPosIncrSloppyMpqAnd() throws IOException {
final MultiPhraseQuery mpq = new MultiPhraseQuery();
for (TokenAndPos tap : INCR_0_QUERY_TOKENS_AND) {
mpq.add(new Term[]{new Term("field",tap.token)}, tap.pos); //AND logic
@@ -539,7 +538,7 @@ public class TestMultiPhraseQuery extend
/**
* MPQ Combined AND OR Mode - Manually creating a multiple phrase query
*/
- public void testZeroPosIncrSloppyMpqAndOrMatch() throws IOException, ParseException {
+ public void testZeroPosIncrSloppyMpqAndOrMatch() throws IOException {
final MultiPhraseQuery mpq = new MultiPhraseQuery();
for (TokenAndPos tap[] : INCR_0_QUERY_TOKENS_AND_OR_MATCH) {
Term[] terms = tapTerms(tap);
@@ -556,7 +555,7 @@ public class TestMultiPhraseQuery extend
/**
* MPQ Combined AND OR Mode - Manually creating a multiple phrase query - with no match
*/
- public void testZeroPosIncrSloppyMpqAndOrNoMatch() throws IOException, ParseException {
+ public void testZeroPosIncrSloppyMpqAndOrNoMatch() throws IOException {
final MultiPhraseQuery mpq = new MultiPhraseQuery();
for (TokenAndPos tap[] : INCR_0_QUERY_TOKENS_AND_OR_NO_MATCHN) {
Term[] terms = tapTerms(tap);
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java Mon Jul 18 13:34:29 2011
@@ -44,7 +44,7 @@ public class TestNumericRangeQuery32 ext
// shift the starting of the values to the left, to also have negative values:
private static final int startOffset = - 1 << 15;
// number of docs to generate for testing
- private static final int noDocs = atLeast(4096);
+ private static int noDocs;
private static Directory directory = null;
private static IndexReader reader = null;
@@ -52,6 +52,7 @@ public class TestNumericRangeQuery32 ext
@BeforeClass
public static void beforeClass() throws Exception {
+ noDocs = atLeast(4096);
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java Mon Jul 18 13:34:29 2011
@@ -41,7 +41,7 @@ public class TestNumericRangeQuery64 ext
// shift the starting of the values to the left, to also have negative values:
private static final long startOffset = - 1L << 31;
// number of docs to generate for testing
- private static final int noDocs = atLeast(4096);
+ private static int noDocs;
private static Directory directory = null;
private static IndexReader reader = null;
@@ -49,6 +49,7 @@ public class TestNumericRangeQuery64 ext
@BeforeClass
public static void beforeClass() throws Exception {
+ noDocs = atLeast(4096);
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java Mon Jul 18 13:34:29 2011
@@ -27,6 +27,7 @@ import org.apache.lucene.index.MultiNorm
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
public class TestSimilarityProvider extends LuceneTestCase {
@@ -125,6 +126,11 @@ public class TestSimilarityProvider exte
public float idf(int docFreq, int numDocs) {
return 1f;
}
+
+ @Override
+ public float scorePayload(int doc, int start, int end, BytesRef payload) {
+ return 1f;
+ }
}
private class Sim2 extends TFIDFSimilarity {
@@ -147,5 +153,10 @@ public class TestSimilarityProvider exte
public float idf(int docFreq, int numDocs) {
return 10f;
}
+
+ @Override
+ public float scorePayload(int doc, int start, int end, BytesRef payload) {
+ return 1f;
+ }
}
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSort.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSort.java Mon Jul 18 13:34:29 2011
@@ -38,7 +38,6 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.Term;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.values.ValueType;
-import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.FieldValueHitQueue.Entry;
import org.apache.lucene.search.cache.ByteValuesCreator;
@@ -54,6 +53,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.DocIdBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
+import org.junit.BeforeClass;
/**
* Unit tests for sorting code.
@@ -66,7 +66,7 @@ import org.apache.lucene.util._TestUtil;
public class TestSort extends LuceneTestCase {
// true if our codec supports docvalues: true unless codec is preflex (3.x)
boolean supportsDocValues = CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex") == false;
- private static final int NUM_STRINGS = atLeast(6000);
+ private static int NUM_STRINGS;
private IndexSearcher full;
private IndexSearcher searchX;
private IndexSearcher searchY;
@@ -79,6 +79,10 @@ public class TestSort extends LuceneTest
private Query queryM;
private Sort sort;
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ NUM_STRINGS = atLeast(6000);
+ }
// document data:
// the tracer field is used to determine which document was hit
// the contents field is used to search and sort by relevance
@@ -365,7 +369,7 @@ public class TestSort extends LuceneTest
/**
* Test String sorting: small queue to many matches, multi field sort, reverse sort
*/
- public void testStringSort() throws IOException, ParseException {
+ public void testStringSort() throws IOException {
ScoreDoc[] result = null;
IndexSearcher searcher = getFullStrings();
sort.setSort(
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSpanQueryFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSpanQueryFilter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSpanQueryFilter.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestSpanQueryFilter.java Mon Jul 18 13:34:29 2011
@@ -47,13 +47,12 @@ public class TestSpanQueryFilter extends
writer.close();
AtomicReaderContext[] leaves = ReaderUtil.leaves(reader.getTopReaderContext());
int subIndex = ReaderUtil.subIndex(number, leaves); // find the reader with this document in it
-
SpanTermQuery query = new SpanTermQuery(new Term("field", English.intToEnglish(number).trim()));
SpanQueryFilter filter = new SpanQueryFilter(query);
SpanFilterResult result = filter.bitSpans(leaves[subIndex]);
DocIdSet docIdSet = result.getDocIdSet();
assertTrue("docIdSet is null and it shouldn't be", docIdSet != null);
- assertContainsDocId("docIdSet doesn't contain docId 10", docIdSet, number);
+ assertContainsDocId("docIdSet doesn't contain docId 10", docIdSet, number - leaves[subIndex].docBase);
List<SpanFilterResult.PositionInfo> spans = result.getPositions();
assertTrue("spans is null and it shouldn't be", spans != null);
int size = getDocIdSetSize(docIdSet);
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestTermQuery.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestTermQuery.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestTermQuery.java Mon Jul 18 13:34:29 2011
@@ -68,6 +68,7 @@ public class TestTermQuery extends Lucen
IndexReader reader = writer.getReader();
DocsAndPositionsEnum docsAndPositions = MultiFields.getTerms(reader,
fieldName).docsAndPositions(null, new BytesRef("1"), null);
+ assertEquals(39, reader.docFreq(new Term(fieldName, "1")));
docsAndPositions.nextDoc();
docsAndPositions.nextPosition();
boolean payloadsIndexed = docsAndPositions.hasPayload();
@@ -84,7 +85,8 @@ public class TestTermQuery extends Lucen
Scorer scorer = weight.scorer(atomicReaderContext, ScorerContext.def()
.needsPositions(true).needsPayloads(payloadsIndexed));
assertNotNull(scorer);
- final int advance = scorer.advance(1 + random.nextInt(27));
+ int toDoc = 1 + random.nextInt(atomicReaderContext.reader.docFreq(new Term(fieldName, "1")) - 1 );
+ final int advance = scorer.advance(toDoc);
PositionIntervalIterator positions = scorer.positions();
do {
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/cache/TestEntryCreators.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/cache/TestEntryCreators.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/cache/TestEntryCreators.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/cache/TestEntryCreators.java Mon Jul 18 13:34:29 2011
@@ -31,14 +31,20 @@ import org.apache.lucene.search.FieldCac
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
+import org.junit.BeforeClass;
import static org.hamcrest.CoreMatchers.*;
public class TestEntryCreators extends LuceneTestCase {
protected IndexReader reader;
- private static final int NUM_DOCS = atLeast(500);
+ private static int NUM_DOCS;
private Directory directory;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ NUM_DOCS = atLeast(500);
+ }
static class NumberTypeTester {
String funcName;
@@ -220,7 +226,7 @@ public class TestEntryCreators extends L
}
assertEquals( "Cached numTerms does not match : "+tester, distinctTerms.size(), cachedVals.numTerms );
assertEquals( "Cached numDocs does not match : "+tester, numDocs, cachedVals.numDocs );
- assertEquals( "Ordinal should match numDocs : "+tester, numDocs, ((OpenBitSet)cachedVals.valid).cardinality() );
+ assertEquals( "Ordinal should match numDocs : "+tester, numDocs, ((FixedBitSet)cachedVals.valid).cardinality() );
}
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Mon Jul 18 13:34:29 2011
@@ -42,6 +42,7 @@ import org.apache.lucene.search.spans.Sp
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.English;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TermContext;
@@ -315,9 +316,9 @@ public class TestPayloadNearQuery extend
return new DefaultSimilarity() {
@Override
- public float scorePayload(int docId, int start, int end, byte[] payload, int offset, int length) {
+ public float scorePayload(int docId, int start, int end, BytesRef payload) {
//we know it is size 4 here, so ignore the offset/length
- return payload[offset];
+ return payload.bytes[payload.offset];
}
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Mon Jul 18 13:34:29 2011
@@ -16,6 +16,7 @@ package org.apache.lucene.search.payload
* limitations under the License.
*/
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.English;
import org.apache.lucene.search.DefaultSimilarityProvider;
@@ -309,9 +310,9 @@ public class TestPayloadTermQuery extend
// TODO: Remove warning after API has been finalized
@Override
- public float scorePayload(int docId, int start, int end, byte[] payload, int offset, int length) {
+ public float scorePayload(int docId, int start, int end, BytesRef payload) {
//we know it is size 4 here, so ignore the offset/length
- return payload[offset];
+ return payload.bytes[payload.offset];
}
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java Mon Jul 18 13:34:29 2011
@@ -135,8 +135,8 @@ final class JustCompileSearchSpans {
static final class JustCompileSpanScorer extends SpanScorer {
protected JustCompileSpanScorer(Spans spans, Weight weight,
- Similarity similarity, Similarity.SloppyDocScorer docScorer) throws IOException {
- super(spans, weight, similarity, docScorer);
+ Similarity.SloppyDocScorer docScorer) throws IOException {
+ super(spans, weight, docScorer);
}
@Override
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestBitVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestBitVector.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestBitVector.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestBitVector.java Mon Jul 18 13:34:29 2011
@@ -19,7 +19,7 @@ package org.apache.lucene.util;
import java.io.IOException;
-import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
@@ -153,7 +153,7 @@ public class TestBitVector extends Lucen
assertTrue(doCompare(bv,compare));
}
}
-
+
/**
* Test r/w when size/count cause switching between bit-set and d-gaps file formats.
*/
@@ -165,6 +165,26 @@ public class TestBitVector extends Lucen
doTestDgaps(10000,40,43);
doTestDgaps(100000,415,418);
doTestDgaps(1000000,3123,3126);
+ // now exercise skipping of fully populated byte in the bitset (they are omitted if bitset is sparse)
+ MockDirectoryWrapper d = new MockDirectoryWrapper(random, new RAMDirectory());
+ d.setPreventDoubleWrite(false);
+ BitVector bv = new BitVector(10000);
+ bv.set(0);
+ for (int i = 8; i < 16; i++) {
+ bv.set(i);
+ } // make sure we have once byte full of set bits
+ for (int i = 32; i < 40; i++) {
+ bv.set(i);
+ } // get a second byte full of set bits
+ // add some more bits here
+ for (int i = 40; i < 10000; i++) {
+ if (random.nextInt(1000) == 0) {
+ bv.set(i);
+ }
+ }
+ bv.write(d, "TESTBV", newIOContext(random));
+ BitVector compare = new BitVector(d, "TESTBV", newIOContext(random));
+ assertTrue(doCompare(bv,compare));
}
private void doTestDgaps(int size, int count1, int count2) throws IOException {
@@ -183,7 +203,7 @@ public class TestBitVector extends Lucen
assertTrue(doCompare(bv,bv2));
bv = bv2;
bv.clear(i);
- assertEquals(i+1,size-bv.count());
+ assertEquals(i+1, size-bv.count());
bv.write(d, "TESTBV", newIOContext(random));
}
// now start decreasing number of set bits
@@ -196,6 +216,54 @@ public class TestBitVector extends Lucen
bv.write(d, "TESTBV", newIOContext(random));
}
}
+
+ public void testSparseWrite() throws IOException {
+ Directory d = newDirectory();
+ final int numBits = 10240;
+ BitVector bv = new BitVector(numBits);
+ bv.invertAll();
+ int numToClear = random.nextInt(5);
+ for(int i=0;i<numToClear;i++) {
+ bv.clear(random.nextInt(numBits));
+ }
+ bv.write(d, "test", newIOContext(random));
+ final long size = d.fileLength("test");
+ assertTrue("size=" + size, size < 100);
+ d.close();
+ }
+
+ public void testClearedBitNearEnd() throws IOException {
+ Directory d = newDirectory();
+ final int numBits = _TestUtil.nextInt(random, 7, 1000);
+ BitVector bv = new BitVector(numBits);
+ bv.invertAll();
+ bv.clear(numBits-_TestUtil.nextInt(random, 1, 7));
+ bv.write(d, "test", newIOContext(random));
+ assertEquals(numBits-1, bv.count());
+ d.close();
+ }
+
+ public void testMostlySet() throws IOException {
+ Directory d = newDirectory();
+ final int numBits = _TestUtil.nextInt(random, 30, 1000);
+ for(int numClear=0;numClear<20;numClear++) {
+ BitVector bv = new BitVector(numBits);
+ bv.invertAll();
+ int count = 0;
+ while(count < numClear) {
+ final int bit = random.nextInt(numBits);
+ // Don't use getAndClear, so that count is recomputed
+ if (bv.get(bit)) {
+ bv.clear(bit);
+ count++;
+ assertEquals(numBits-count, bv.count());
+ }
+ }
+ }
+
+ d.close();
+ }
+
/**
* Compare two BitVectors.
* This should really be an equals method on the BitVector itself.
@@ -211,6 +279,7 @@ public class TestBitVector extends Lucen
break;
}
}
+ assertEquals(bv.count(), compare.count());
return equal;
}
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java Mon Jul 18 13:34:29 2011
@@ -1,5 +1,7 @@
package org.apache.lucene.util;
+import org.junit.BeforeClass;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -22,8 +24,14 @@ package org.apache.lucene.util;
*/
@Deprecated
public class TestIndexableBinaryStringTools extends LuceneTestCase {
- private static final int NUM_RANDOM_TESTS = atLeast(200);
- private static final int MAX_RANDOM_BINARY_LENGTH = atLeast(300);
+ private static int NUM_RANDOM_TESTS;
+ private static int MAX_RANDOM_BINARY_LENGTH;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ NUM_RANDOM_TESTS = atLeast(200);
+ MAX_RANDOM_BINARY_LENGTH = atLeast(300);
+ }
public void testSingleBinaryRoundTrip() {
byte[] binary = new byte[] { (byte) 0x23, (byte) 0x98, (byte) 0x13,
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java Mon Jul 18 13:34:29 2011
@@ -184,6 +184,7 @@ public class TestNumericUtils extends Lu
private void assertLongRangeSplit(final long lower, final long upper, int precisionStep,
final boolean useBitSet, final Iterable<Long> expectedBounds, final Iterable<Integer> expectedShifts
) throws Exception {
+ // Cannot use FixedBitSet since the range could be long:
final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null;
final Iterator<Long> neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator();
final Iterator<Integer> neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator();
@@ -212,7 +213,7 @@ public class TestNumericUtils extends Lu
if (useBitSet) {
// after flipping all bits in the range, the cardinality should be zero
bits.flip(0,upper-lower+1);
- assertTrue("The sub-range concenated should match the whole range", bits.isEmpty());
+ assertEquals("The sub-range concenated should match the whole range", 0, bits.cardinality());
}
}
@@ -424,7 +425,7 @@ public class TestNumericUtils extends Lu
private void assertIntRangeSplit(final int lower, final int upper, int precisionStep,
final boolean useBitSet, final Iterable<Integer> expectedBounds, final Iterable<Integer> expectedShifts
) throws Exception {
- final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null;
+ final FixedBitSet bits=useBitSet ? new FixedBitSet(upper-lower+1) : null;
final Iterator<Integer> neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator();
final Iterator<Integer> neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator();
@@ -451,8 +452,8 @@ public class TestNumericUtils extends Lu
if (useBitSet) {
// after flipping all bits in the range, the cardinality should be zero
- bits.flip(0,upper-lower+1);
- assertTrue("The sub-range concenated should match the whole range", bits.isEmpty());
+ bits.flip(0, upper-lower+1);
+ assertEquals("The sub-range concenated should match the whole range", 0, bits.cardinality());
}
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java Mon Jul 18 13:34:29 2011
@@ -150,7 +150,7 @@ public class TestFSTs extends LuceneTest
for(IntsRef term : terms2) {
pairs.add(new FSTTester.InputOutput<Object>(term, NO_OUTPUT));
}
- FST<Object> fst = new FSTTester<Object>(random, dir, inputMode, pairs, outputs).doTest(0, 0);
+ FST<Object> fst = new FSTTester<Object>(random, dir, inputMode, pairs, outputs).doTest(0, 0, false);
assertNotNull(fst);
assertEquals(22, fst.getNodeCount());
assertEquals(27, fst.getArcCount());
@@ -163,7 +163,7 @@ public class TestFSTs extends LuceneTest
for(int idx=0;idx<terms2.length;idx++) {
pairs.add(new FSTTester.InputOutput<Long>(terms2[idx], outputs.get(idx)));
}
- final FST<Long> fst = new FSTTester<Long>(random, dir, inputMode, pairs, outputs).doTest(0, 0);
+ final FST<Long> fst = new FSTTester<Long>(random, dir, inputMode, pairs, outputs).doTest(0, 0, false);
assertNotNull(fst);
assertEquals(22, fst.getNodeCount());
assertEquals(27, fst.getArcCount());
@@ -178,7 +178,7 @@ public class TestFSTs extends LuceneTest
final BytesRef output = random.nextInt(30) == 17 ? NO_OUTPUT : new BytesRef(Integer.toString(idx));
pairs.add(new FSTTester.InputOutput<BytesRef>(terms2[idx], output));
}
- final FST<BytesRef> fst = new FSTTester<BytesRef>(random, dir, inputMode, pairs, outputs).doTest(0, 0);
+ final FST<BytesRef> fst = new FSTTester<BytesRef>(random, dir, inputMode, pairs, outputs).doTest(0, 0, false);
assertNotNull(fst);
assertEquals(24, fst.getNodeCount());
assertEquals(30, fst.getArcCount());
@@ -359,14 +359,14 @@ public class TestFSTs extends LuceneTest
public void doTest() throws IOException {
// no pruning
- doTest(0, 0);
+ doTest(0, 0, true);
if (!(outputs instanceof UpToTwoPositiveIntOutputs)) {
// simple pruning
- doTest(_TestUtil.nextInt(random, 1, 1+pairs.size()), 0);
+ doTest(_TestUtil.nextInt(random, 1, 1+pairs.size()), 0, true);
// leafy pruning
- doTest(0, _TestUtil.nextInt(random, 1, 1+pairs.size()));
+ doTest(0, _TestUtil.nextInt(random, 1, 1+pairs.size()), true);
}
}
@@ -446,14 +446,17 @@ public class TestFSTs extends LuceneTest
}
- FST<T> doTest(int prune1, int prune2) throws IOException {
+ FST<T> doTest(int prune1, int prune2, boolean allowRandomSuffixSharing) throws IOException {
if (VERBOSE) {
System.out.println("TEST: prune1=" + prune1 + " prune2=" + prune2);
}
final Builder<T> builder = new Builder<T>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4,
prune1, prune2,
- prune1==0 && prune2==0, outputs);
+ prune1==0 && prune2==0,
+ allowRandomSuffixSharing ? random.nextBoolean() : true,
+ allowRandomSuffixSharing ? _TestUtil.nextInt(random, 1, 10) : Integer.MAX_VALUE,
+ outputs);
for(InputOutput<T> pair : pairs) {
if (pair.output instanceof UpToTwoPositiveIntOutputs.TwoLongs) {
@@ -1017,7 +1020,7 @@ public class TestFSTs extends LuceneTest
IndexReader r = IndexReader.open(writer, true);
writer.close();
final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(random.nextBoolean());
- Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, outputs);
+ Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
boolean storeOrd = random.nextBoolean();
if (VERBOSE) {
@@ -1145,7 +1148,7 @@ public class TestFSTs extends LuceneTest
this.inputMode = inputMode;
this.outputs = outputs;
- builder = new Builder<T>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, 0, prune, prune == 0, outputs);
+ builder = new Builder<T>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, 0, prune, prune == 0, true, Integer.MAX_VALUE, outputs);
}
protected abstract T getOutput(IntsRef input, int ord) throws IOException;
@@ -1245,7 +1248,7 @@ public class TestFSTs extends LuceneTest
}
}
- // java -cp build/classes/test:build/classes/java:lib/junit-4.7.jar org.apache.lucene.util.fst.TestFSTs /x/tmp/allTerms3.txt out
+ // java -cp build/classes/test:build/classes/java:build/classes/test-framework:lib/junit-4.7.jar org.apache.lucene.util.fst.TestFSTs /x/tmp/allTerms3.txt out
public static void main(String[] args) throws IOException {
int prune = 0;
int limit = Integer.MAX_VALUE;
@@ -1351,7 +1354,7 @@ public class TestFSTs extends LuceneTest
public void testSingleString() throws Exception {
final Outputs<Object> outputs = NoOutputs.getSingleton();
- final Builder<Object> b = new Builder<Object>(FST.INPUT_TYPE.BYTE1, 0, 0, true, outputs);
+ final Builder<Object> b = new Builder<Object>(FST.INPUT_TYPE.BYTE1, outputs);
b.add(new BytesRef("foobar"), outputs.getNoOutput());
final BytesRefFSTEnum<Object> fstEnum = new BytesRefFSTEnum<Object>(b.finish());
assertNull(fstEnum.seekFloor(new BytesRef("foo")));
@@ -1368,7 +1371,7 @@ public class TestFSTs extends LuceneTest
final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
// Build an FST mapping BytesRef -> Long
- final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, outputs);
+ final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
final BytesRef a = new BytesRef("a");
final BytesRef b = new BytesRef("b");
@@ -1413,7 +1416,7 @@ public class TestFSTs extends LuceneTest
FST<Object> compile(String[] lines) throws IOException {
final NoOutputs outputs = NoOutputs.getSingleton();
final Object nothing = outputs.getNoOutput();
- final Builder<Object> b = new Builder<Object>(FST.INPUT_TYPE.BYTE1, 0, 0, true, outputs);
+ final Builder<Object> b = new Builder<Object>(FST.INPUT_TYPE.BYTE1, outputs);
int line = 0;
final BytesRef term = new BytesRef();
@@ -1488,7 +1491,7 @@ public class TestFSTs extends LuceneTest
public void testNonFinalStopNodes() throws Exception {
final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
final Long nothing = outputs.getNoOutput();
- final Builder<Long> b = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, outputs);
+ final Builder<Long> b = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
final FST<Long> fst = new FST<Long>(FST.INPUT_TYPE.BYTE1, outputs);
Modified: lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java (original)
+++ lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java Mon Jul 18 13:34:29 2011
@@ -26,7 +26,6 @@ import org.apache.lucene.analysis.util.F
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.util.Version;
/**
Modified: lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java (original)
+++ lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java Mon Jul 18 13:34:29 2011
@@ -1,3 +1,5 @@
+package org.apache.lucene.analysis.synonym;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -15,245 +17,550 @@
* limitations under the License.
*/
-package org.apache.lucene.analysis.synonym;
+import java.io.IOException;
-import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.util.fst.FST;
+
+/**
+ * Matches single or multi word synonyms in a token stream.
+ * This token stream cannot properly handle position
+ * increments != 1, ie, you should place this filter before
+ * filtering out stop words.
+ *
+ * <p>Note that with the current implementation, parsing is
+ * greedy, so whenever multiple parses would apply, the rule
+ * starting the earliest and parsing the most tokens wins.
+ * For example if you have these rules:
+ *
+ * <pre>
+ * a -> x
+ * a b -> y
+ * b c d -> z
+ * </pre>
+ *
+ * Then input <code>a b c d e</code> parses to <code>y b c
+ * d</code>, ie the 2nd rule "wins" because it started
+ * earliest and matched the most input tokens of other rules
+ * starting at that point.</p>
+ *
+ * <p>A future improvement to this filter could allow
+ * non-greedy parsing, such that the 3rd rule would win, and
+ * also separately allow multiple parses, such that all 3
+ * rules would match, perhaps even on a rule by rule
+ * basis.</p>
+ *
+ * <p><b>NOTE</b>: when a match occurs, the output tokens
+ * associated with the matching rule are "stacked" on top of
+ * the input stream (if the rule had
+ * <code>keepOrig=true</code>) and also on top of aother
+ * matched rule's output tokens. This is not a correct
+ * solution, as really the output should be an abitrary
+ * graph/lattice. For example, with the above match, you
+ * would expect an exact <code>PhraseQuery</code> <code>"y b
+ * c"</code> to match the parsed tokens, but it will fail to
+ * do so. This limitations is necessary because Lucene's
+ * TokenStream (and index) cannot yet represent an arbitrary
+ * graph.</p>
+ *
+ * <p><b>NOTE</b>: If multiple incoming tokens arrive on the
+ * same position, only the first token at that position is
+ * used for parsing. Subsequent tokens simply pass through
+ * and are not parsed. A future improvement would be to
+ * allow these tokens to also be matched.</p>
+ */
+
+// TODO: maybe we should resolve token -> wordID then run
+// FST on wordIDs, for better perf?
+
+// TODO: a more efficient approach would be Aho/Corasick's
+// algorithm
+// http://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_string_matching_algorithm
+// It improves over the current approach here
+// because it does not fully re-start matching at every
+// token. For exampl,e if one pattern is "a b c x"
+// and another is "b c d" and the input is "a b c d", on
+// trying to parse "a b c x" but failing when you got to x,
+// rather than starting over again your really should
+// immediately recognize that "b c d" matches at the next
+// input. I suspect this won't matter that much in
+// practice, but it's possible on some set of synonyms it
+// will. We'd have to modify Aho/Corasick to enforce our
+// conflict resolving (eg greedy matching) because that algo
+// finds all matches.
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-
-/** SynonymFilter handles multi-token synonyms with variable position increment offsets.
- * <p>
- * The matched tokens from the input stream may be optionally passed through (includeOrig=true)
- * or discarded. If the original tokens are included, the position increments may be modified
- * to retain absolute positions after merging with the synonym tokenstream.
- * <p>
- * Generated synonyms will start at the same position as the first matched source token.
- */
public final class SynonymFilter extends TokenFilter {
- private final SynonymMap map; // Map<String, SynonymMap>
- private Iterator<AttributeSource> replacement; // iterator over generated tokens
+ public static final String TYPE_SYNONYM = "SYNONYM";
- public SynonymFilter(TokenStream in, SynonymMap map) {
- super(in);
- if (map == null)
- throw new IllegalArgumentException("map is required");
-
- this.map = map;
- // just ensuring these attributes exist...
- addAttribute(CharTermAttribute.class);
- addAttribute(PositionIncrementAttribute.class);
- addAttribute(OffsetAttribute.class);
- addAttribute(TypeAttribute.class);
- }
+ private final SynonymMap synonyms;
+ private final boolean ignoreCase;
+ private final int rollBufferSize;
- /*
- * Need to worry about multiple scenarios:
- * - need to go for the longest match
- * a b => foo #shouldn't match if "a b" is followed by "c d"
- * a b c d => bar
- * - need to backtrack - retry matches for tokens already read
- * a b c d => foo
- * b c => bar
- * If the input stream is "a b c x", one will consume "a b c d"
- * trying to match the first rule... all but "a" should be
- * pushed back so a match may be made on "b c".
- * - don't try and match generated tokens (thus need separate queue)
- * matching is not recursive.
- * - handle optional generation of original tokens in all these cases,
- * merging token streams to preserve token positions.
- * - preserve original positionIncrement of first matched token
- */
- @Override
- public boolean incrementToken() throws IOException {
- while (true) {
- // if there are any generated tokens, return them... don't try any
- // matches against them, as we specifically don't want recursion.
- if (replacement!=null && replacement.hasNext()) {
- copy(this, replacement.next());
- return true;
- }
+ private int captureCount;
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+
+ // How many future input tokens have already been matched
+ // to a synonym; because the matching is "greedy" we don't
+ // try to do any more matching for such tokens:
+ private int inputSkipCount;
+
+ // Hold all buffered (read ahead) stacked input tokens for
+ // a future position. When multiple tokens are at the
+ // same position, we only store (and match against) the
+ // term for the first token at the position, but capture
+ // state for (and enumerate) all other tokens at this
+ // position:
+ private static class PendingInput {
+ final CharsRef term = new CharsRef();
+ AttributeSource.State state;
+ boolean keepOrig;
+ boolean consumed = true;
+ int startOffset;
+ int endOffset;
+
+ public void reset() {
+ state = null;
+ consumed = true;
+ keepOrig = false;
+ }
+ };
- // common case fast-path of first token not matching anything
- AttributeSource firstTok = nextTok();
- if (firstTok == null) return false;
- CharTermAttribute termAtt = firstTok.addAttribute(CharTermAttribute.class);
- SynonymMap result = map.submap!=null ? map.submap.get(termAtt.buffer(), 0, termAtt.length()) : null;
- if (result == null) {
- copy(this, firstTok);
- return true;
+ // Rolling buffer, holding pending input tokens we had to
+ // clone because we needed to look ahead, indexed by
+ // position:
+ private final PendingInput[] futureInputs;
+
+ // Holds pending output synonyms for one future position:
+ private static class PendingOutputs {
+ CharsRef[] outputs;
+ int upto;
+ int count;
+ int posIncr = 1;
+
+ public PendingOutputs() {
+ outputs = new CharsRef[1];
+ }
+
+ public void reset() {
+ upto = count = 0;
+ posIncr = 1;
+ }
+
+ public CharsRef pullNext() {
+ assert upto < count;
+ final CharsRef result = outputs[upto++];
+ posIncr = 0;
+ if (upto == count) {
+ reset();
}
+ return result;
+ }
- // fast-path failed, clone ourselves if needed
- if (firstTok == this)
- firstTok = cloneAttributes();
- // OK, we matched a token, so find the longest match.
-
- matched = new LinkedList<AttributeSource>();
-
- result = match(result);
-
- if (result==null) {
- // no match, simply return the first token read.
- copy(this, firstTok);
- return true;
+ public void add(char[] output, int offset, int len) {
+ if (count == outputs.length) {
+ final CharsRef[] next = new CharsRef[ArrayUtil.oversize(1+count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ System.arraycopy(outputs, 0, next, 0, count);
+ outputs = next;
}
+ if (outputs[count] == null) {
+ outputs[count] = new CharsRef();
+ }
+ outputs[count].copy(output, offset, len);
+ count++;
+ }
+ };
+
+ private final ByteArrayDataInput bytesReader = new ByteArrayDataInput();
+
+ // Rolling buffer, holding stack of pending synonym
+ // outputs, indexed by position:
+ private final PendingOutputs[] futureOutputs;
+
+ // Where (in rolling buffers) to write next input saved state:
+ private int nextWrite;
+
+ // Where (in rolling buffers) to read next input saved state:
+ private int nextRead;
+
+ // True once we've read last token
+ private boolean finished;
+
+ private final FST.Arc<BytesRef> scratchArc;
+
+ private final FST<BytesRef> fst;
+
+ private final BytesRef scratchBytes = new BytesRef();
+ private final CharsRef scratchChars = new CharsRef();
+
+ /**
+ * @param input input tokenstream
+ * @param synonyms synonym map
+ * @param ignoreCase case-folds input for matching with {@link Character#toLowerCase(int)}.
+ * Note, if you set this to true, its your responsibility to lowercase
+ * the input entries when you create the {@link SynonymMap}
+ */
+ public SynonymFilter(TokenStream input, SynonymMap synonyms, boolean ignoreCase) {
+ super(input);
+ this.synonyms = synonyms;
+ this.ignoreCase = ignoreCase;
+ this.fst = synonyms.fst;
+
+ if (fst == null) {
+ throw new IllegalArgumentException("fst must be non-null");
+ }
+
+ // Must be 1+ so that when roll buffer is at full
+ // lookahead we can distinguish this full buffer from
+ // the empty buffer:
+ rollBufferSize = 1+synonyms.maxHorizontalContext;
+
+ futureInputs = new PendingInput[rollBufferSize];
+ futureOutputs = new PendingOutputs[rollBufferSize];
+ for(int pos=0;pos<rollBufferSize;pos++) {
+ futureInputs[pos] = new PendingInput();
+ futureOutputs[pos] = new PendingOutputs();
+ }
- // reuse, or create new one each time?
- ArrayList<AttributeSource> generated = new ArrayList<AttributeSource>(result.synonyms.length + matched.size() + 1);
+ //System.out.println("FSTFilt maxH=" + synonyms.maxHorizontalContext);
- //
- // there was a match... let's generate the new tokens, merging
- // in the matched tokens (position increments need adjusting)
- //
- AttributeSource lastTok = matched.isEmpty() ? firstTok : matched.getLast();
- boolean includeOrig = result.includeOrig();
-
- AttributeSource origTok = includeOrig ? firstTok : null;
- PositionIncrementAttribute firstPosIncAtt = firstTok.addAttribute(PositionIncrementAttribute.class);
- int origPos = firstPosIncAtt.getPositionIncrement(); // position of origTok in the original stream
- int repPos=0; // curr position in replacement token stream
- int pos=0; // current position in merged token stream
-
- for (int i=0; i<result.synonyms.length; i++) {
- Token repTok = result.synonyms[i];
- AttributeSource newTok = firstTok.cloneAttributes();
- CharTermAttribute newTermAtt = newTok.addAttribute(CharTermAttribute.class);
- OffsetAttribute newOffsetAtt = newTok.addAttribute(OffsetAttribute.class);
- PositionIncrementAttribute newPosIncAtt = newTok.addAttribute(PositionIncrementAttribute.class);
-
- OffsetAttribute lastOffsetAtt = lastTok.addAttribute(OffsetAttribute.class);
-
- newOffsetAtt.setOffset(newOffsetAtt.startOffset(), lastOffsetAtt.endOffset());
- newTermAtt.copyBuffer(repTok.buffer(), 0, repTok.length());
- repPos += repTok.getPositionIncrement();
- if (i==0) repPos=origPos; // make position of first token equal to original
-
- // if necessary, insert original tokens and adjust position increment
- while (origTok != null && origPos <= repPos) {
- PositionIncrementAttribute origPosInc = origTok.addAttribute(PositionIncrementAttribute.class);
- origPosInc.setPositionIncrement(origPos-pos);
- generated.add(origTok);
- pos += origPosInc.getPositionIncrement();
- origTok = matched.isEmpty() ? null : matched.removeFirst();
- if (origTok != null) {
- origPosInc = origTok.addAttribute(PositionIncrementAttribute.class);
- origPos += origPosInc.getPositionIncrement();
+ scratchArc = new FST.Arc<BytesRef>();
+ }
+
+ private void capture() {
+ captureCount++;
+ //System.out.println(" capture slot=" + nextWrite);
+ final PendingInput input = futureInputs[nextWrite];
+
+ input.state = captureState();
+ input.consumed = false;
+ input.term.copy(termAtt.buffer(), 0, termAtt.length());
+
+ nextWrite = rollIncr(nextWrite);
+
+ // Buffer head should never catch up to tail:
+ assert nextWrite != nextRead;
+ }
+
+ /*
+ This is the core of this TokenFilter: it locates the
+ synonym matches and buffers up the results into
+ futureInputs/Outputs.
+
+ NOTE: this calls input.incrementToken and does not
+ capture the state if no further tokens were checked. So
+ caller must then forward state to our caller, or capture:
+ */
+
+ private void parse() throws IOException {
+ //System.out.println("\nS: parse");
+
+ assert inputSkipCount == 0;
+
+ int curNextRead = nextRead;
+
+ // Holds the longest match we've seen so far:
+ BytesRef matchOutput = null;
+ int matchInputLength = 0;
+
+ BytesRef pendingOutput = fst.outputs.getNoOutput();
+ fst.getFirstArc(scratchArc);
+
+ assert scratchArc.output == fst.outputs.getNoOutput();
+
+ int tokenCount = 0;
+
+ byToken:
+ while(true) {
+
+ // Pull next token's chars:
+ final char[] buffer;
+ final int bufferLen;
+ //System.out.println(" cycle nextRead=" + curNextRead + " nextWrite=" + nextWrite);
+
+ if (curNextRead == nextWrite) {
+
+ // We used up our lookahead buffer of input tokens
+ // -- pull next real input token:
+
+ if (finished) {
+ break;
+ } else {
+ //System.out.println(" input.incrToken");
+ assert futureInputs[nextWrite].consumed;
+ // Not correct: a syn match whose output is longer
+ // than its input can set future inputs keepOrig
+ // to true:
+ //assert !futureInputs[nextWrite].keepOrig;
+ if (input.incrementToken()) {
+ buffer = termAtt.buffer();
+ bufferLen = termAtt.length();
+ final PendingInput input = futureInputs[nextWrite];
+ input.startOffset = offsetAtt.startOffset();
+ input.endOffset = offsetAtt.endOffset();
+ //System.out.println(" new token=" + new String(buffer, 0, bufferLen));
+ if (nextRead != nextWrite) {
+ capture();
+ } else {
+ input.consumed = false;
+ }
+
+ } else {
+ // No more input tokens
+ //System.out.println(" set end");
+ finished = true;
+ break;
}
}
+ } else {
+ // Still in our lookahead
+ buffer = futureInputs[curNextRead].term.chars;
+ bufferLen = futureInputs[curNextRead].term.length;
+ //System.out.println(" old token=" + new String(buffer, 0, bufferLen));
+ }
+
+ tokenCount++;
+
+ // Run each char in this token through the FST:
+ int bufUpto = 0;
+ while(bufUpto < bufferLen) {
+ final int codePoint = Character.codePointAt(buffer, bufUpto, bufferLen);
+ if (fst.findTargetArc(ignoreCase ? Character.toLowerCase(codePoint) : codePoint, scratchArc, scratchArc) == null) {
+ //System.out.println(" stop");
+ break byToken;
+ }
+
+ // Accum the output
+ pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output);
+ //System.out.println(" char=" + buffer[bufUpto] + " output=" + pendingOutput + " arc.output=" + scratchArc.output);
+ bufUpto += Character.charCount(codePoint);
+ }
- newPosIncAtt.setPositionIncrement(repPos - pos);
- generated.add(newTok);
- pos += newPosIncAtt.getPositionIncrement();
+ // OK, entire token matched; now see if this is a final
+ // state:
+ if (scratchArc.isFinal()) {
+ matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput);
+ matchInputLength = tokenCount;
+ //System.out.println(" found matchLength=" + matchInputLength + " output=" + matchOutput);
}
- // finish up any leftover original tokens
- while (origTok!=null) {
- PositionIncrementAttribute origPosInc = origTok.addAttribute(PositionIncrementAttribute.class);
- origPosInc.setPositionIncrement(origPos-pos);
- generated.add(origTok);
- pos += origPosInc.getPositionIncrement();
- origTok = matched.isEmpty() ? null : matched.removeFirst();
- if (origTok != null) {
- origPosInc = origTok.addAttribute(PositionIncrementAttribute.class);
- origPos += origPosInc.getPositionIncrement();
+ // See if the FST wants to continue matching (ie, needs to
+ // see the next input token):
+ if (fst.findTargetArc(SynonymMap.WORD_SEPARATOR, scratchArc, scratchArc) == null) {
+ // No further rules can match here; we're done
+ // searching for matching rules starting at the
+ // current input position.
+ break;
+ } else {
+ // More matching is possible -- accum the output (if
+ // any) of the WORD_SEP arc:
+ pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output);
+ if (nextRead == nextWrite) {
+ capture();
}
}
- // what if we replaced a longer sequence with a shorter one?
- // a/0 b/5 => foo/0
- // should I re-create the gap on the next buffered token?
-
- replacement = generated.iterator();
- // Now return to the top of the loop to read and return the first
- // generated token.. The reason this is done is that we may have generated
- // nothing at all, and may need to continue with more matching logic.
+ curNextRead = rollIncr(curNextRead);
}
- }
+ if (nextRead == nextWrite && !finished) {
+ //System.out.println(" skip write slot=" + nextWrite);
+ nextWrite = rollIncr(nextWrite);
+ }
- //
- // Defer creation of the buffer until the first time it is used to
- // optimize short fields with no matches.
- //
- private LinkedList<AttributeSource> buffer;
- private LinkedList<AttributeSource> matched;
-
- private boolean exhausted;
-
- private AttributeSource nextTok() throws IOException {
- if (buffer!=null && !buffer.isEmpty()) {
- return buffer.removeFirst();
+ if (matchOutput != null) {
+ //System.out.println(" add matchLength=" + matchInputLength + " output=" + matchOutput);
+ inputSkipCount = matchInputLength;
+ addOutput(matchOutput);
+ } else if (nextRead != nextWrite) {
+ // Even though we had no match here, we set to 1
+ // because we need to skip current input token before
+ // trying to match again:
+ inputSkipCount = 1;
} else {
- if (!exhausted && input.incrementToken()) {
- return this;
- } else {
- exhausted = true;
- return null;
+ assert finished;
+ }
+
+ //System.out.println(" parse done inputSkipCount=" + inputSkipCount + " nextRead=" + nextRead + " nextWrite=" + nextWrite);
+ }
+
+ // Interleaves all output tokens onto the futureOutputs:
+ private void addOutput(BytesRef bytes) {
+ bytesReader.reset(bytes.bytes, bytes.offset, bytes.length);
+
+ final int code = bytesReader.readVInt();
+ final boolean keepOrig = (code & 0x1) == 0;
+ final int count = code >>> 1;
+ //System.out.println(" addOutput count=" + count + " keepOrig=" + keepOrig);
+ for(int outputIDX=0;outputIDX<count;outputIDX++) {
+ synonyms.words.get(bytesReader.readVInt(),
+ scratchBytes);
+ //System.out.println(" outIDX=" + outputIDX + " bytes=" + scratchBytes.length);
+ UnicodeUtil.UTF8toUTF16(scratchBytes, scratchChars);
+ int lastStart = scratchChars.offset;
+ final int chEnd = lastStart + scratchChars.length;
+ int outputUpto = nextRead;
+ for(int chIDX=lastStart;chIDX<=chEnd;chIDX++) {
+ if (chIDX == chEnd || scratchChars.chars[chIDX] == SynonymMap.WORD_SEPARATOR) {
+ final int outputLen = chIDX - lastStart;
+ // Caller is not allowed to have empty string in
+ // the output:
+ assert outputLen > 0: "output contains empty string: " + scratchChars;
+ futureOutputs[outputUpto].add(scratchChars.chars, lastStart, outputLen);
+ //System.out.println(" " + new String(scratchChars.chars, lastStart, outputLen) + " outputUpto=" + outputUpto);
+ lastStart = 1+chIDX;
+ futureInputs[outputUpto].keepOrig |= keepOrig;
+ //System.out.println(" slot=" + outputUpto + " keepOrig=" + keepOrig);
+ outputUpto = rollIncr(outputUpto);
+ assert futureOutputs[outputUpto].posIncr == 1: "outputUpto=" + outputUpto + " vs nextWrite=" + nextWrite;
+ }
}
}
}
- private void pushTok(AttributeSource t) {
- if (buffer==null) buffer=new LinkedList<AttributeSource>();
- buffer.addFirst(t);
+ // ++ mod rollBufferSize
+ private int rollIncr(int count) {
+ count++;
+ if (count == rollBufferSize) {
+ return 0;
+ } else {
+ return count;
+ }
}
- private SynonymMap match(SynonymMap map) throws IOException {
- SynonymMap result = null;
+ // for testing
+ int getCaptureCount() {
+ return captureCount;
+ }
- if (map.submap != null) {
- AttributeSource tok = nextTok();
- if (tok != null) {
- // clone ourselves.
- if (tok == this)
- tok = cloneAttributes();
- // check for positionIncrement!=1? if>1, should not match, if==0, check multiple at this level?
- CharTermAttribute termAtt = tok.getAttribute(CharTermAttribute.class);
- SynonymMap subMap = map.submap.get(termAtt.buffer(), 0, termAtt.length());
-
- if (subMap != null) {
- // recurse
- result = match(subMap);
+ @Override
+ public boolean incrementToken() throws IOException {
+
+ //System.out.println("\nS: incrToken inputSkipCount=" + inputSkipCount + " nextRead=" + nextRead + " nextWrite=" + nextWrite);
+
+ while(true) {
+
+ // First play back any buffered future inputs/outputs
+ // w/o running parsing again:
+ while (inputSkipCount != 0) {
+
+ // At each position, we first output the original
+ // token
+
+ // TODO: maybe just a PendingState class, holding
+ // both input & outputs?
+ final PendingInput input = futureInputs[nextRead];
+ final PendingOutputs outputs = futureOutputs[nextRead];
+
+ //System.out.println(" cycle nextRead=" + nextRead + " nextWrite=" + nextWrite + " inputSkipCount="+ inputSkipCount + " input.keepOrig=" + input.keepOrig + " input.consumed=" + input.consumed + " input.state=" + input.state);
+
+ if (!input.consumed && (input.keepOrig || outputs.count == 0)) {
+ if (input.state != null) {
+ // Return a previously saved token (because we
+ // had to lookahead):
+ restoreState(input.state);
+ } else {
+ // Pass-through case: return token we just pulled
+ // but didn't capture:
+ assert inputSkipCount == 1: "inputSkipCount=" + inputSkipCount + " nextRead=" + nextRead;
+ }
+ input.reset();
+ if (outputs.count > 0) {
+ outputs.posIncr = 0;
+ } else {
+ nextRead = rollIncr(nextRead);
+ inputSkipCount--;
+ }
+ //System.out.println(" return token=" + termAtt.toString());
+ return true;
+ } else if (outputs.upto < outputs.count) {
+ // Still have pending outputs to replay at this
+ // position
+ input.reset();
+ final int posIncr = outputs.posIncr;
+ final CharsRef output = outputs.pullNext();
+ clearAttributes();
+ termAtt.copyBuffer(output.chars, output.offset, output.length);
+ typeAtt.setType(TYPE_SYNONYM);
+ offsetAtt.setOffset(input.startOffset, input.endOffset);
+ posIncrAtt.setPositionIncrement(posIncr);
+ if (outputs.count == 0) {
+ // Done with the buffered input and all outputs at
+ // this position
+ nextRead = rollIncr(nextRead);
+ inputSkipCount--;
+ }
+ //System.out.println(" return token=" + termAtt.toString());
+ return true;
+ } else {
+ // Done with the buffered input and all outputs at
+ // this position
+ input.reset();
+ nextRead = rollIncr(nextRead);
+ inputSkipCount--;
}
+ }
- if (result != null) {
- matched.addFirst(tok);
+ if (finished && nextRead == nextWrite) {
+ // End case: if any output syns went beyond end of
+ // input stream, enumerate them now:
+ final PendingOutputs outputs = futureOutputs[nextRead];
+ if (outputs.upto < outputs.count) {
+ final int posIncr = outputs.posIncr;
+ final CharsRef output = outputs.pullNext();
+ futureInputs[nextRead].reset();
+ if (outputs.count == 0) {
+ nextWrite = nextRead = rollIncr(nextRead);
+ }
+ clearAttributes();
+ termAtt.copyBuffer(output.chars, output.offset, output.length);
+ typeAtt.setType(TYPE_SYNONYM);
+ //System.out.println(" set posIncr=" + outputs.posIncr + " outputs=" + outputs);
+ posIncrAtt.setPositionIncrement(posIncr);
+ //System.out.println(" return token=" + termAtt.toString());
+ return true;
} else {
- // push back unmatched token
- pushTok(tok);
+ return false;
}
}
- }
- // if no longer sequence matched, so if this node has synonyms, it's the match.
- if (result==null && map.synonyms!=null) {
- result = map;
+ // Find new synonym matches:
+ parse();
}
-
- return result;
- }
-
- private void copy(AttributeSource target, AttributeSource source) {
- if (target != source)
- source.copyTo(target);
}
@Override
public void reset() throws IOException {
- input.reset();
- replacement = null;
- exhausted = false;
+ super.reset();
+ captureCount = 0;
+ finished = false;
+
+ // In normal usage these resets would not be needed,
+ // since they reset-as-they-are-consumed, but the app
+ // may not consume all input tokens in which case we
+ // have leftover state here:
+ for (PendingInput input : futureInputs) {
+ input.reset();
+ }
+ for (PendingOutputs output : futureOutputs) {
+ output.reset();
+ }
}
}
Modified: lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java (original)
+++ lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java Mon Jul 18 13:34:29 2011
@@ -1,3 +1,5 @@
+package org.apache.lucene.analysis.synonym;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -15,146 +17,301 @@
* limitations under the License.
*/
-package org.apache.lucene.analysis.synonym;
-
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.util.CharArrayMap;
-import org.apache.lucene.util.Version;
-
-import java.util.*;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.store.ByteArrayDataOutput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.util.fst.ByteSequenceOutputs;
+import org.apache.lucene.util.fst.FST;
-/** Mapping rules for use with {@link SynonymFilter}
+/**
+ * A map of synonyms, keys and values are phrases.
+ * @lucene.experimental
*/
public class SynonymMap {
- /** @lucene.internal */
- public CharArrayMap<SynonymMap> submap; // recursive: Map<String, SynonymMap>
- /** @lucene.internal */
- public Token[] synonyms;
- int flags;
-
- static final int INCLUDE_ORIG=0x01;
- static final int IGNORE_CASE=0x02;
-
- public SynonymMap() {}
- public SynonymMap(boolean ignoreCase) {
- if (ignoreCase) flags |= IGNORE_CASE;
+ /** for multiword support, you must separate words with this separator */
+ public static final char WORD_SEPARATOR = 0;
+ /** map<input word, list<ord>> */
+ public final FST<BytesRef> fst;
+ /** map<ord, outputword> */
+ public final BytesRefHash words;
+ /** maxHorizontalContext: maximum context we need on the tokenstream */
+ public final int maxHorizontalContext;
+
+ public SynonymMap(FST<BytesRef> fst, BytesRefHash words, int maxHorizontalContext) {
+ this.fst = fst;
+ this.words = words;
+ this.maxHorizontalContext = maxHorizontalContext;
}
-
- public boolean includeOrig() { return (flags & INCLUDE_ORIG) != 0; }
- public boolean ignoreCase() { return (flags & IGNORE_CASE) != 0; }
-
+
/**
- * @param singleMatch List<String>, the sequence of strings to match
- * @param replacement List<Token> the list of tokens to use on a match
- * @param includeOrig sets a flag on this mapping signaling the generation of matched tokens in addition to the replacement tokens
- * @param mergeExisting merge the replacement tokens with any other mappings that exist
+ * Builds an FSTSynonymMap.
+ * <p>
+ * Call add() until you have added all the mappings, then call build() to get an FSTSynonymMap
+ * @lucene.experimental
*/
- public void add(List<String> singleMatch, List<Token> replacement, boolean includeOrig, boolean mergeExisting) {
- SynonymMap currMap = this;
- for (String str : singleMatch) {
- if (currMap.submap==null) {
- // for now hardcode at 4.0, as its what the old code did.
- // would be nice to fix, but shouldn't store a version in each submap!!!
- currMap.submap = new CharArrayMap<SynonymMap>(Version.LUCENE_40, 1, ignoreCase());
- }
+ public static class Builder {
+ private final HashMap<CharsRef,MapEntry> workingSet = new HashMap<CharsRef,MapEntry>();
+ private final BytesRefHash words = new BytesRefHash();
+ private final BytesRef utf8Scratch = new BytesRef(8);
+ private int maxHorizontalContext;
+ private final boolean dedup;
+
+ /** If dedup is true then identical rules (same input,
+ * same output) will be added only once. */
+ public Builder(boolean dedup) {
+ this.dedup = dedup;
+ }
+
+ private static class MapEntry {
+ boolean includeOrig;
+ // we could sort for better sharing ultimately, but it could confuse people
+ ArrayList<Integer> ords = new ArrayList<Integer>();
+ }
- SynonymMap map = currMap.submap.get(str);
- if (map==null) {
- map = new SynonymMap();
- map.flags |= flags & IGNORE_CASE;
- currMap.submap.put(str, map);
+ /** Sugar: just joins the provided terms with {@link
+ * SynonymMap#WORD_SEPARATOR}. reuse and its chars
+ * must not be null. */
+ public static CharsRef join(String[] words, CharsRef reuse) {
+ int upto = 0;
+ char[] buffer = reuse.chars;
+ for(String word : words) {
+ if (upto > 0) {
+ if (upto >= buffer.length) {
+ reuse.grow(upto);
+ buffer = reuse.chars;
+ }
+ buffer[upto++] = SynonymMap.WORD_SEPARATOR;
+ }
+
+ final int wordLen = word.length();
+ final int needed = upto + wordLen;
+ if (needed > buffer.length) {
+ reuse.grow(needed);
+ buffer = reuse.chars;
+ }
+
+ word.getChars(0, wordLen, buffer, upto);
+ upto += wordLen;
}
- currMap = map;
+ return reuse;
}
-
- if (currMap.synonyms != null && !mergeExisting) {
- throw new RuntimeException("SynonymFilter: there is already a mapping for " + singleMatch);
+
+ /** Sugar: analyzes the text with the analyzer and
+ * separates by {@link SynonymMap#WORD_SEPARATOR}.
+ * reuse and its chars must not be null. */
+ public static CharsRef analyze(Analyzer analyzer, String text, CharsRef reuse) throws IOException {
+ TokenStream ts = analyzer.reusableTokenStream("", new StringReader(text));
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
+ ts.reset();
+ reuse.length = 0;
+ while (ts.incrementToken()) {
+ int length = termAtt.length();
+ if (length == 0) {
+ throw new IllegalArgumentException("term: " + text + " analyzed to a zero-length token");
+ }
+ if (posIncAtt.getPositionIncrement() != 1) {
+ throw new IllegalArgumentException("term: " + text + " analyzed to a token with posinc != 1");
+ }
+ reuse.grow(reuse.length + length + 1); /* current + word + separator */
+ int end = reuse.offset + reuse.length;
+ if (reuse.length > 0) {
+ reuse.chars[end++] = SynonymMap.WORD_SEPARATOR;
+ reuse.length++;
+ }
+ System.arraycopy(termAtt.buffer(), 0, reuse.chars, end, length);
+ reuse.length += length;
+ }
+ ts.end();
+ ts.close();
+ if (reuse.length == 0) {
+ throw new IllegalArgumentException("term: " + text + " was completely eliminated by analyzer");
+ }
+ return reuse;
}
- List<Token> superset = currMap.synonyms==null ? replacement :
- mergeTokens(Arrays.asList(currMap.synonyms), replacement);
- currMap.synonyms = superset.toArray(new Token[superset.size()]);
- if (includeOrig) currMap.flags |= INCLUDE_ORIG;
- }
+ /** only used for asserting! */
+ private boolean hasHoles(CharsRef chars) {
+ final int end = chars.offset + chars.length;
+ for(int idx=chars.offset+1;idx<end;idx++) {
+ if (chars.chars[idx] == SynonymMap.WORD_SEPARATOR && chars.chars[idx-1] == SynonymMap.WORD_SEPARATOR) {
+ return true;
+ }
+ }
+ if (chars.chars[chars.offset] == '\u0000') {
+ return true;
+ }
+ if (chars.chars[chars.offset + chars.length - 1] == '\u0000') {
+ return true;
+ }
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("<");
- if (synonyms!=null) {
- sb.append("[");
- for (int i=0; i<synonyms.length; i++) {
- if (i!=0) sb.append(',');
- sb.append(synonyms[i]);
- }
- if ((flags & INCLUDE_ORIG)!=0) {
- sb.append(",ORIG");
- }
- sb.append("],");
- }
- sb.append(submap);
- sb.append(">");
- return sb.toString();
- }
+ return false;
+ }
+ // NOTE: while it's tempting to make this public, since
+ // caller's parser likely knows the
+ // numInput/numOutputWords, sneaky exceptions, much later
+ // on, will result if these values are wrong; so we always
+ // recompute ourselves to be safe:
+ private void add(CharsRef input, int numInputWords, CharsRef output, int numOutputWords, boolean includeOrig) {
+ // first convert to UTF-8
+ if (numInputWords <= 0) {
+ throw new IllegalArgumentException("numInputWords must be > 0 (got " + numInputWords + ")");
+ }
+ if (input.length <= 0) {
+ throw new IllegalArgumentException("input.length must be > 0 (got " + input.length + ")");
+ }
+ if (numOutputWords <= 0) {
+ throw new IllegalArgumentException("numOutputWords must be > 0 (got " + numOutputWords + ")");
+ }
+ if (output.length <= 0) {
+ throw new IllegalArgumentException("output.length must be > 0 (got " + output.length + ")");
+ }
+ assert !hasHoles(input): "input has holes: " + input;
+ assert !hasHoles(output): "output has holes: " + output;
- /** Produces a List<Token> from a List<String> */
- public static List<Token> makeTokens(List<String> strings) {
- List<Token> ret = new ArrayList<Token>(strings.size());
- for (String str : strings) {
- //Token newTok = new Token(str,0,0,"SYNONYM");
- Token newTok = new Token(str, 0,0,"SYNONYM");
- ret.add(newTok);
+ //System.out.println("fmap.add input=" + input + " numInputWords=" + numInputWords + " output=" + output + " numOutputWords=" + numOutputWords);
+ final int hashCode = UnicodeUtil.UTF16toUTF8WithHash(output.chars, output.offset, output.length, utf8Scratch);
+ // lookup in hash
+ int ord = words.add(utf8Scratch, hashCode);
+ if (ord < 0) {
+ // already exists in our hash
+ ord = (-ord)-1;
+ //System.out.println(" output=" + output + " old ord=" + ord);
+ } else {
+ //System.out.println(" output=" + output + " new ord=" + ord);
+ }
+
+ MapEntry e = workingSet.get(input);
+ if (e == null) {
+ e = new MapEntry();
+ workingSet.put(new CharsRef(input), e); // make a copy, since we will keep around in our map
+ }
+
+ e.ords.add(ord);
+ e.includeOrig |= includeOrig;
+ maxHorizontalContext = Math.max(maxHorizontalContext, numInputWords);
+ maxHorizontalContext = Math.max(maxHorizontalContext, numOutputWords);
}
- return ret;
- }
+ private int countWords(CharsRef chars) {
+ int wordCount = 1;
+ int upto = chars.offset;
+ final int limit = chars.offset + chars.length;
+ while(upto < limit) {
+ if (chars.chars[upto++] == SynonymMap.WORD_SEPARATOR) {
+ wordCount++;
+ }
+ }
+ return wordCount;
+ }
+
+ /**
+ * Add a phrase->phrase synonym mapping.
+ * Phrases are character sequences where words are
+ * separated with character zero (\u0000). Empty words
+ * (two \u0000s in a row) are not allowed in the input nor
+ * the output!
+ *
+ * @param input input phrase
+ * @param output output phrase
+ * @param includeOrig true if the original should be included
+ */
+ public void add(CharsRef input, CharsRef output, boolean includeOrig) {
+ add(input, countWords(input), output, countWords(output), includeOrig);
+ }
+
+ /**
+ * Builds an {@link SynonymMap} and returns it.
+ */
+ public SynonymMap build() throws IOException {
+ ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
+ // TODO: are we using the best sharing options?
+ org.apache.lucene.util.fst.Builder<BytesRef> builder =
+ new org.apache.lucene.util.fst.Builder<BytesRef>(FST.INPUT_TYPE.BYTE4, outputs);
+
+ BytesRef scratch = new BytesRef(64);
+ ByteArrayDataOutput scratchOutput = new ByteArrayDataOutput();
+
+ final Set<Integer> dedupSet;
+
+ if (dedup) {
+ dedupSet = new HashSet<Integer>();
+ } else {
+ dedupSet = null;
+ }
- /**
- * Merge two lists of tokens, producing a single list with manipulated positionIncrements so that
- * the tokens end up at the same position.
- *
- * Example: [a b] merged with [c d] produces [a/b c/d] ('/' denotes tokens in the same position)
- * Example: [a,5 b,2] merged with [c d,4 e,4] produces [c a,5/d b,2 e,2] (a,n means a has posInc=n)
- *
- */
- public static List<Token> mergeTokens(List<Token> lst1, List<Token> lst2) {
- ArrayList<Token> result = new ArrayList<Token>();
- if (lst1 ==null || lst2 ==null) {
- if (lst2 != null) result.addAll(lst2);
- if (lst1 != null) result.addAll(lst1);
- return result;
- }
-
- int pos=0;
- Iterator<Token> iter1=lst1.iterator();
- Iterator<Token> iter2=lst2.iterator();
- Token tok1 = iter1.hasNext() ? iter1.next() : null;
- Token tok2 = iter2.hasNext() ? iter2.next() : null;
- int pos1 = tok1!=null ? tok1.getPositionIncrement() : 0;
- int pos2 = tok2!=null ? tok2.getPositionIncrement() : 0;
- while(tok1!=null || tok2!=null) {
- while (tok1 != null && (pos1 <= pos2 || tok2==null)) {
- Token tok = new Token(tok1.startOffset(), tok1.endOffset(), tok1.type());
- tok.copyBuffer(tok1.buffer(), 0, tok1.length());
- tok.setPositionIncrement(pos1-pos);
- result.add(tok);
- pos=pos1;
- tok1 = iter1.hasNext() ? iter1.next() : null;
- pos1 += tok1!=null ? tok1.getPositionIncrement() : 0;
- }
- while (tok2 != null && (pos2 <= pos1 || tok1==null)) {
- Token tok = new Token(tok2.startOffset(), tok2.endOffset(), tok2.type());
- tok.copyBuffer(tok2.buffer(), 0, tok2.length());
- tok.setPositionIncrement(pos2-pos);
- result.add(tok);
- pos=pos2;
- tok2 = iter2.hasNext() ? iter2.next() : null;
- pos2 += tok2!=null ? tok2.getPositionIncrement() : 0;
+ final byte[] spare = new byte[5];
+
+ Set<CharsRef> keys = workingSet.keySet();
+ CharsRef sortedKeys[] = keys.toArray(new CharsRef[keys.size()]);
+ Arrays.sort(sortedKeys, CharsRef.getUTF16SortedAsUTF8Comparator());
+
+ //System.out.println("fmap.build");
+ for (int keyIdx = 0; keyIdx < sortedKeys.length; keyIdx++) {
+ CharsRef input = sortedKeys[keyIdx];
+ MapEntry output = workingSet.get(input);
+
+ int numEntries = output.ords.size();
+ // output size, assume the worst case
+ int estimatedSize = 5 + numEntries * 5; // numEntries + one ord for each entry
+
+ scratch.grow(estimatedSize);
+ scratchOutput.reset(scratch.bytes, scratch.offset, scratch.bytes.length);
+ assert scratch.offset == 0;
+
+ // now write our output data:
+ int count = 0;
+ for (int i = 0; i < numEntries; i++) {
+ if (dedupSet != null) {
+ // box once
+ final Integer ent = output.ords.get(i);
+ if (dedupSet.contains(ent)) {
+ continue;
+ }
+ dedupSet.add(ent);
+ }
+ scratchOutput.writeVInt(output.ords.get(i));
+ count++;
+ }
+
+ final int pos = scratchOutput.getPosition();
+ scratchOutput.writeVInt(count << 1 | (output.includeOrig ? 0 : 1));
+ final int pos2 = scratchOutput.getPosition();
+ final int vIntLen = pos2-pos;
+
+ // Move the count + includeOrig to the front of the byte[]:
+ System.arraycopy(scratch.bytes, pos, spare, 0, vIntLen);
+ System.arraycopy(scratch.bytes, 0, scratch.bytes, vIntLen, pos);
+ System.arraycopy(spare, 0, scratch.bytes, 0, vIntLen);
+
+ if (dedupSet != null) {
+ dedupSet.clear();
+ }
+
+ scratch.length = scratchOutput.getPosition() - scratch.offset;
+ //System.out.println(" add input=" + input + " output=" + scratch + " offset=" + scratch.offset + " length=" + scratch.length + " count=" + count);
+ builder.add(input, new BytesRef(scratch));
}
+
+ FST<BytesRef> fst = builder.finish();
+ return new SynonymMap(fst, words, maxHorizontalContext);
}
- return result;
}
-
}
Modified: lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/util/FilteringTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/util/FilteringTokenFilter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/util/FilteringTokenFilter.java (original)
+++ lucene/dev/branches/positions/modules/analysis/common/src/java/org/apache/lucene/analysis/util/FilteringTokenFilter.java Mon Jul 18 13:34:29 2011
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.queryParser.QueryParser; // for javadoc
/**
* Abstract base class for TokenFilters that may remove tokens.
@@ -87,7 +86,7 @@ public abstract class FilteringTokenFilt
* token is incremented.
*
* <p> <b>NOTE</b>: be sure to also
- * set {@link QueryParser#setEnablePositionIncrements} if
+ * set org.apache.lucene.queryparser.classic.QueryParser#setEnablePositionIncrements if
* you use QueryParser to create queries.
*/
public void setEnablePositionIncrements(boolean enable) {