You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/08/20 21:20:48 UTC
svn commit: r1159905 [6/6] - in /lucene/dev/trunk:
lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/
lucene/src/java/org/apache/lucene/index/
lucene/src/java/org/apache/lucene/index/codecs/
lucene/src/java/org/apache/lucene/index/codec...
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java Sat Aug 20 19:20:45 2011
@@ -17,15 +17,32 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.BasicAutomata;
+import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.automaton.DaciukMihovAutomatonBuilder;
public class TestTermsEnum extends LuceneTestCase {
@@ -140,4 +157,563 @@ public class TestTermsEnum extends Lucen
r.close();
d.close();
}
+
+ private String randomString() {
+ //return _TestUtil.randomSimpleString(random);
+ return _TestUtil.randomRealisticUnicodeString(random);
+ }
+
+ private void addDoc(RandomIndexWriter w, Collection<String> terms, Map<BytesRef,Integer> termToID, int id) throws IOException {
+ Document doc = new Document();
+ doc.add(new NumericField("id").setIntValue(id));
+ if (VERBOSE) {
+ System.out.println("TEST: addDoc id:" + id + " terms=" + terms);
+ }
+ for (String s2 : terms) {
+ doc.add(newField("f", s2, Field.Index.NOT_ANALYZED));
+ termToID.put(new BytesRef(s2), id);
+ }
+ w.addDocument(doc);
+ terms.clear();
+ }
+
+ private boolean accepts(CompiledAutomaton c, BytesRef b) {
+ int state = c.runAutomaton.getInitialState();
+ for(int idx=0;idx<b.length;idx++) {
+ assertTrue(state != -1);
+ state = c.runAutomaton.step(state, b.bytes[b.offset+idx] & 0xff);
+ }
+ return c.runAutomaton.isAccept(state);
+ }
+
+ // Tests Terms.intersect
+ public void testIntersectRandom() throws IOException {
+
+ final Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random, dir);
+
+ final int numTerms = atLeast(1000);
+
+ final Set<String> terms = new HashSet<String>();
+ final Collection<String> pendingTerms = new ArrayList<String>();
+ final Map<BytesRef,Integer> termToID = new HashMap<BytesRef,Integer>();
+ int id = 0;
+ while(terms.size() != numTerms) {
+ final String s = randomString();
+ if (!terms.contains(s)) {
+ terms.add(s);
+ pendingTerms.add(s);
+ if (random.nextInt(20) == 7) {
+ addDoc(w, pendingTerms, termToID, id++);
+ }
+ }
+ }
+ addDoc(w, pendingTerms, termToID, id++);
+
+ final BytesRef[] termsArray = new BytesRef[terms.size()];
+ final Set<BytesRef> termsSet = new HashSet<BytesRef>();
+ {
+ int upto = 0;
+ for(String s : terms) {
+ final BytesRef b = new BytesRef(s);
+ termsArray[upto++] = b;
+ termsSet.add(b);
+ }
+ Arrays.sort(termsArray);
+ }
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: indexed terms (unicode order):");
+ for(BytesRef t : termsArray) {
+ System.out.println(" " + t.utf8ToString() + " -> id:" + termToID.get(t));
+ }
+ }
+
+ final IndexReader r = w.getReader();
+ w.close();
+
+ // NOTE: intentional insanity!!
+ final int[] docIDToID = FieldCache.DEFAULT.getInts(r, "id");
+
+ for(int iter=0;iter<10*RANDOM_MULTIPLIER;iter++) {
+
+ // TODO: can we also test infinite As here...?
+
+ // From the random terms, pick some ratio and compile an
+ // automaton:
+ final Set<String> acceptTerms = new HashSet<String>();
+ final TreeSet<BytesRef> sortedAcceptTerms = new TreeSet<BytesRef>();
+ final double keepPct = random.nextDouble();
+ Automaton a;
+ if (iter == 0) {
+ if (VERBOSE) {
+ System.out.println("\nTEST: empty automaton");
+ }
+ a = BasicAutomata.makeEmpty();
+ } else {
+ if (VERBOSE) {
+ System.out.println("\nTEST: keepPct=" + keepPct);
+ }
+ for (String s : terms) {
+ final String s2;
+ if (random.nextDouble() <= keepPct) {
+ s2 = s;
+ } else {
+ s2 = randomString();
+ }
+ acceptTerms.add(s2);
+ sortedAcceptTerms.add(new BytesRef(s2));
+ }
+ a = DaciukMihovAutomatonBuilder.build(sortedAcceptTerms);
+ }
+ final CompiledAutomaton c = new CompiledAutomaton(a, true, false);
+
+ final BytesRef[] acceptTermsArray = new BytesRef[acceptTerms.size()];
+ final Set<BytesRef> acceptTermsSet = new HashSet<BytesRef>();
+ int upto = 0;
+ for(String s : acceptTerms) {
+ final BytesRef b = new BytesRef(s);
+ acceptTermsArray[upto++] = b;
+ acceptTermsSet.add(b);
+ assertTrue(accepts(c, b));
+ }
+ Arrays.sort(acceptTermsArray);
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: accept terms (unicode order):");
+ for(BytesRef t : acceptTermsArray) {
+ System.out.println(" " + t.utf8ToString() + (termsSet.contains(t) ? " (exists)" : ""));
+ }
+ System.out.println(a.toDot());
+ }
+
+ for(int iter2=0;iter2<100;iter2++) {
+ final BytesRef startTerm = acceptTermsArray.length == 0 || random.nextBoolean() ? null : acceptTermsArray[random.nextInt(acceptTermsArray.length)];
+
+ final TermsEnum te = MultiFields.getTerms(r, "f").intersect(c, startTerm);
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: iter2=" + iter2 + " startTerm=" + (startTerm == null ? "<null>" : startTerm.utf8ToString()));
+ }
+
+ int loc;
+ if (startTerm == null) {
+ loc = 0;
+ } else {
+ loc = Arrays.binarySearch(termsArray, new BytesRef(startTerm));
+ if (loc < 0) {
+ loc = -(loc+1);
+ } else {
+ // startTerm exists in index
+ loc++;
+ }
+ }
+ while(loc < termsArray.length && !acceptTermsSet.contains(termsArray[loc])) {
+ loc++;
+ }
+
+ DocsEnum docsEnum = null;
+ while (loc < termsArray.length) {
+ final BytesRef expected = termsArray[loc];
+ final BytesRef actual = te.next();
+ if (VERBOSE) {
+ System.out.println("TEST: next() expected=" + expected.utf8ToString() + " actual=" + actual.utf8ToString());
+ }
+ assertEquals(expected, actual);
+ assertEquals(1, te.docFreq());
+ docsEnum = te.docs(null, docsEnum);
+ final int docID = docsEnum.nextDoc();
+ assertTrue(docID != DocsEnum.NO_MORE_DOCS);
+ assertEquals(docIDToID[docID], termToID.get(expected).intValue());
+ do {
+ loc++;
+ } while (loc < termsArray.length && !acceptTermsSet.contains(termsArray[loc]));
+ }
+
+ assertNull(te.next());
+ }
+ }
+
+ r.close();
+ dir.close();
+ }
+
+ private Directory d;
+ private IndexReader r;
+
+ private final String FIELD = "field";
+
+ private IndexReader makeIndex(String... terms) throws Exception {
+ d = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+
+ /*
+ CoreCodecProvider cp = new CoreCodecProvider();
+ cp.unregister(cp.lookup("Standard"));
+ cp.register(new StandardCodec(minTermsInBlock, maxTermsInBlock));
+ cp.setDefaultFieldCodec("Standard");
+ iwc.setCodecProvider(cp);
+ */
+
+ final RandomIndexWriter w = new RandomIndexWriter(random, d, iwc);
+ w.w.setInfoStream(VERBOSE ? System.out : null);
+ for(String term : terms) {
+ Document doc = new Document();
+ Field f = newField(FIELD, term, Field.Index.NOT_ANALYZED_NO_NORMS);
+ doc.add(f);
+ w.addDocument(doc);
+ }
+ if (r != null) {
+ close();
+ }
+ r = w.getReader();
+ w.close();
+ return r;
+ }
+
+ private void close() throws Exception {
+ final Directory d = ((SegmentReader) r.getSequentialSubReaders()[0]).directory();
+ r.close();
+ d.close();
+ }
+
+ private int docFreq(IndexReader r, String term) throws Exception {
+ return r.docFreq(new Term(FIELD, term));
+ }
+
+ public void testEasy() throws Exception {
+ // No floor arcs:
+ r = makeIndex("aa0", "aa1", "aa2", "aa3", "bb0", "bb1", "bb2", "bb3", "aa");
+
+ // First term in block:
+ assertEquals(1, docFreq(r, "aa0"));
+
+ // Scan forward to another term in same block
+ assertEquals(1, docFreq(r, "aa2"));
+
+ assertEquals(1, docFreq(r, "aa"));
+
+ // Reset same block then scan forwards
+ assertEquals(1, docFreq(r, "aa1"));
+
+ // Not found, in same block
+ assertEquals(0, docFreq(r, "aa5"));
+
+ // Found, in same block
+ assertEquals(1, docFreq(r, "aa2"));
+
+ // Not found in index:
+ assertEquals(0, docFreq(r, "b0"));
+
+ // Found:
+ assertEquals(1, docFreq(r, "aa2"));
+
+ // Found, rewind:
+ assertEquals(1, docFreq(r, "aa0"));
+
+
+ // First term in block:
+ assertEquals(1, docFreq(r, "bb0"));
+
+ // Scan forward to another term in same block
+ assertEquals(1, docFreq(r, "bb2"));
+
+ // Reset same block then scan forwards
+ assertEquals(1, docFreq(r, "bb1"));
+
+ // Not found, in same block
+ assertEquals(0, docFreq(r, "bb5"));
+
+ // Found, in same block
+ assertEquals(1, docFreq(r, "bb2"));
+
+ // Not found in index:
+ assertEquals(0, docFreq(r, "b0"));
+
+ // Found:
+ assertEquals(1, docFreq(r, "bb2"));
+
+ // Found, rewind:
+ assertEquals(1, docFreq(r, "bb0"));
+
+ close();
+ }
+
+ // tests:
+ // - test same prefix has non-floor block and floor block (ie, has 2 long outputs on same term prefix)
+ // - term that's entirely in the index
+
+ public void testFloorBlocks() throws Exception {
+ final String[] terms = new String[] {"aa0", "aa1", "aa2", "aa3", "aa4", "aa5", "aa6", "aa7", "aa8", "aa9", "aa", "xx"};
+ r = makeIndex(terms);
+ //r = makeIndex("aa0", "aa1", "aa2", "aa3", "aa4", "aa5", "aa6", "aa7", "aa8", "aa9");
+
+ // First term in first block:
+ assertEquals(1, docFreq(r, "aa0"));
+ assertEquals(1, docFreq(r, "aa4"));
+
+ // No block
+ assertEquals(0, docFreq(r, "bb0"));
+
+ // Second block
+ assertEquals(1, docFreq(r, "aa4"));
+
+ // Backwards to prior floor block:
+ assertEquals(1, docFreq(r, "aa0"));
+
+ // Forwards to last floor block:
+ assertEquals(1, docFreq(r, "aa9"));
+
+ assertEquals(0, docFreq(r, "a"));
+ assertEquals(1, docFreq(r, "aa"));
+ assertEquals(0, docFreq(r, "a"));
+ assertEquals(1, docFreq(r, "aa"));
+
+ // Forwards to last floor block:
+ assertEquals(1, docFreq(r, "xx"));
+ assertEquals(1, docFreq(r, "aa1"));
+ assertEquals(0, docFreq(r, "yy"));
+
+ assertEquals(1, docFreq(r, "xx"));
+ assertEquals(1, docFreq(r, "aa9"));
+
+ assertEquals(1, docFreq(r, "xx"));
+ assertEquals(1, docFreq(r, "aa4"));
+
+ final TermsEnum te = MultiFields.getTerms(r, FIELD).iterator();
+ while(te.next() != null) {
+ //System.out.println("TEST: next term=" + te.term().utf8ToString());
+ }
+
+ assertTrue(seekExact(te, "aa1"));
+ assertEquals("aa2", next(te));
+ assertTrue(seekExact(te, "aa8"));
+ assertEquals("aa9", next(te));
+ assertEquals("xx", next(te));
+
+ testRandomSeeks(r, terms);
+ close();
+ }
+
+ public void testZeroTerms() throws Exception {
+ d = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random, d);
+ w.w.setInfoStream(VERBOSE ? System.out : null);
+ Document doc = new Document();
+ doc.add(newField("field", "one two three", Field.Index.ANALYZED));
+ doc = new Document();
+ doc.add(newField("field2", "one two three", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ w.commit();
+ w.deleteDocuments(new Term("field", "one"));
+ w.optimize();
+ IndexReader r = w.getReader();
+ w.close();
+ assertEquals(1, r.numDocs());
+ assertEquals(1, r.maxDoc());
+ Terms terms = MultiFields.getTerms(r, "field");
+ if (terms != null) {
+ assertNull(terms.iterator().next());
+ }
+ r.close();
+ d.close();
+ }
+
+ private String getRandomString() {
+ //return _TestUtil.randomSimpleString(random);
+ return _TestUtil.randomRealisticUnicodeString(random);
+ }
+
+ public void testRandomTerms() throws Exception {
+ final String[] terms = new String[_TestUtil.nextInt(random, 1, atLeast(1000))];
+ final Set<String> seen = new HashSet<String>();
+
+ final boolean allowEmptyString = random.nextBoolean();
+
+ if (random.nextInt(10) == 7 && terms.length > 2) {
+ // Sometimes add a bunch of terms sharing a longish common prefix:
+ final int numTermsSamePrefix = random.nextInt(terms.length/2);
+ if (numTermsSamePrefix > 0) {
+ String prefix;
+ while(true) {
+ prefix = getRandomString();
+ if (prefix.length() < 5) {
+ continue;
+ } else {
+ break;
+ }
+ }
+ while(seen.size() < numTermsSamePrefix) {
+ final String t = prefix + getRandomString();
+ if (!seen.contains(t)) {
+ terms[seen.size()] = t;
+ seen.add(t);
+ }
+ }
+ }
+ }
+
+ while(seen.size() < terms.length) {
+ final String t = getRandomString();
+ if (!seen.contains(t) && (allowEmptyString || t.length() != 0)) {
+ terms[seen.size()] = t;
+ seen.add(t);
+ }
+ }
+ r = makeIndex(terms);
+ testRandomSeeks(r, terms);
+ close();
+ }
+
+ // sugar
+ private boolean seekExact(TermsEnum te, String term) throws IOException {
+ return te.seekExact(new BytesRef(term), random.nextBoolean());
+ }
+
+ // sugar
+ private String next(TermsEnum te) throws IOException {
+ final BytesRef br = te.next();
+ if (br == null) {
+ return null;
+ } else {
+ return br.utf8ToString();
+ }
+ }
+
+ private BytesRef getNonExistTerm(BytesRef[] terms) {
+ BytesRef t = null;
+ while(true) {
+ final String ts = getRandomString();
+ t = new BytesRef(ts);
+ if (Arrays.binarySearch(terms, t) < 0) {
+ return t;
+ }
+ }
+ }
+
+ private static class TermAndState {
+ public final BytesRef term;
+ public final TermState state;
+
+ public TermAndState(BytesRef term, TermState state) {
+ this.term = term;
+ this.state = state;
+ }
+ }
+
+ private void testRandomSeeks(IndexReader r, String... validTermStrings) throws IOException {
+ final BytesRef[] validTerms = new BytesRef[validTermStrings.length];
+ for(int termIDX=0;termIDX<validTermStrings.length;termIDX++) {
+ validTerms[termIDX] = new BytesRef(validTermStrings[termIDX]);
+ }
+ Arrays.sort(validTerms);
+ if (VERBOSE) {
+ System.out.println("TEST: " + validTerms.length + " terms:");
+ for(BytesRef t : validTerms) {
+ System.out.println(" " + t.utf8ToString() + " " + t);
+ }
+ }
+ final TermsEnum te = MultiFields.getTerms(r, FIELD).iterator();
+
+ final int END_LOC = -validTerms.length-1;
+
+ final List<TermAndState> termStates = new ArrayList<TermAndState>();
+
+ for(int iter=0;iter<100*RANDOM_MULTIPLIER;iter++) {
+
+ final BytesRef t;
+ int loc;
+ final TermState termState;
+ if (random.nextInt(6) == 4) {
+ // pick term that doens't exist:
+ t = getNonExistTerm(validTerms);
+ termState = null;
+ if (VERBOSE) {
+ System.out.println("\nTEST: invalid term=" + t.utf8ToString());
+ }
+ loc = Arrays.binarySearch(validTerms, t);
+ } else if (termStates.size() != 0 && random.nextInt(4) == 1) {
+ final TermAndState ts = termStates.get(random.nextInt(termStates.size()));
+ t = ts.term;
+ loc = Arrays.binarySearch(validTerms, t);
+ assertTrue(loc >= 0);
+ termState = ts.state;
+ if (VERBOSE) {
+ System.out.println("\nTEST: valid termState term=" + t.utf8ToString());
+ }
+ } else {
+ // pick valid term
+ loc = random.nextInt(validTerms.length);
+ t = new BytesRef(validTerms[loc]);
+ termState = null;
+ if (VERBOSE) {
+ System.out.println("\nTEST: valid term=" + t.utf8ToString());
+ }
+ }
+
+ // seekCeil or seekExact:
+ final boolean doSeekExact = random.nextBoolean();
+ if (termState != null) {
+ if (VERBOSE) {
+ System.out.println(" seekExact termState");
+ }
+ te.seekExact(t, termState);
+ } else if (doSeekExact) {
+ if (VERBOSE) {
+ System.out.println(" seekExact");
+ }
+ assertEquals(loc >= 0, te.seekExact(t, random.nextBoolean()));
+ } else {
+ if (VERBOSE) {
+ System.out.println(" seekCeil");
+ }
+
+ final TermsEnum.SeekStatus result = te.seekCeil(t, random.nextBoolean());
+ if (VERBOSE) {
+ System.out.println(" got " + result);
+ }
+
+ if (loc >= 0) {
+ assertEquals(TermsEnum.SeekStatus.FOUND, result);
+ } else if (loc == END_LOC) {
+ assertEquals(TermsEnum.SeekStatus.END, result);
+ } else {
+ assert loc >= -validTerms.length;
+ assertEquals(TermsEnum.SeekStatus.NOT_FOUND, result);
+ }
+ }
+
+ if (loc >= 0) {
+ assertEquals(t, te.term());
+ } else if (doSeekExact) {
+ // TermsEnum is unpositioned if seekExact returns false
+ continue;
+ } else if (loc == END_LOC) {
+ continue;
+ } else {
+ loc = -loc-1;
+ assertEquals(validTerms[loc], te.term());
+ }
+
+ // Do a bunch of next's after the seek
+ final int numNext = random.nextInt(validTerms.length);
+
+ for(int nextCount=0;nextCount<numNext;nextCount++) {
+ if (VERBOSE) {
+ System.out.println("\nTEST: next loc=" + loc + " of " + validTerms.length);
+ }
+ final BytesRef t2 = te.next();
+ loc++;
+ if (loc == validTerms.length) {
+ assertNull(t2);
+ break;
+ } else {
+ assertEquals(validTerms[loc], t2);
+ if (random.nextInt(40) == 17 && termStates.size() < 100) {
+ termStates.add(new TermAndState(validTerms[loc], te.termState()));
+ }
+ }
+ }
+ }
+ }
}
Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java?rev=1159905&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java (added)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java Sat Aug 20 19:20:45 2011
@@ -0,0 +1,175 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.TermsEnum.SeekStatus;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.search.AutomatonQuery;
+import org.apache.lucene.search.CheckHits;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.AutomatonTestUtil;
+import org.apache.lucene.util.automaton.BasicOperations;
+import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.automaton.DaciukMihovAutomatonBuilder;
+import org.apache.lucene.util.automaton.RegExp;
+import org.apache.lucene.util.automaton.SpecialOperations;
+
+public class TestTermsEnum2 extends LuceneTestCase {
+ private Directory dir;
+ private IndexReader reader;
+ private IndexSearcher searcher;
+ private SortedSet<BytesRef> terms; // the terms we put in the index
+ private Automaton termsAutomaton; // automata of the same
+ int numIterations;
+
+ public void setUp() throws Exception {
+ super.setUp();
+ // we generate aweful regexps: good for testing.
+ // but for preflex codec, the test can be very slow, so use less iterations.
+ numIterations = CodecProvider.getDefault().getFieldCodec("field").equals("PreFlex") ? 10 * RANDOM_MULTIPLIER : atLeast(50);
+ dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random, MockTokenizer.KEYWORD, false))
+ .setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000)));
+ Document doc = new Document();
+ Field field = newField("field", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
+ doc.add(field);
+ terms = new TreeSet<BytesRef>();
+
+ int num = atLeast(200);
+ for (int i = 0; i < num; i++) {
+ String s = _TestUtil.randomUnicodeString(random);
+ field.setValue(s);
+ terms.add(new BytesRef(s));
+ writer.addDocument(doc);
+ }
+
+ termsAutomaton = DaciukMihovAutomatonBuilder.build(terms);
+
+ reader = writer.getReader();
+ searcher = newSearcher(reader);
+ writer.close();
+ }
+
+ public void tearDown() throws Exception {
+ searcher.close();
+ reader.close();
+ dir.close();
+ super.tearDown();
+ }
+
+ /** tests a pre-intersected automaton against the original */
+ public void testFiniteVersusInfinite() throws Exception {
+ for (int i = 0; i < numIterations; i++) {
+ String reg = AutomatonTestUtil.randomRegexp(random);
+ Automaton automaton = new RegExp(reg, RegExp.NONE).toAutomaton();
+ final List<BytesRef> matchedTerms = new ArrayList<BytesRef>();
+ for(BytesRef t : terms) {
+ if (BasicOperations.run(automaton, t.utf8ToString())) {
+ matchedTerms.add(t);
+ }
+ }
+
+ Automaton alternate = DaciukMihovAutomatonBuilder.build(matchedTerms);
+ //System.out.println("match " + matchedTerms.size() + " " + alternate.getNumberOfStates() + " states, sigma=" + alternate.getStartPoints().length);
+ //AutomatonTestUtil.minimizeSimple(alternate);
+ //System.out.println("minmize done");
+ AutomatonQuery a1 = new AutomatonQuery(new Term("field", ""), automaton);
+ AutomatonQuery a2 = new AutomatonQuery(new Term("field", ""), alternate);
+ CheckHits.checkEqual(a1, searcher.search(a1, 25).scoreDocs, searcher.search(a2, 25).scoreDocs);
+ }
+ }
+
+ /** seeks to every term accepted by some automata */
+ public void testSeeking() throws Exception {
+ for (int i = 0; i < numIterations; i++) {
+ String reg = AutomatonTestUtil.randomRegexp(random);
+ Automaton automaton = new RegExp(reg, RegExp.NONE).toAutomaton();
+ TermsEnum te = MultiFields.getTerms(reader, "field").iterator();
+ ArrayList<BytesRef> unsortedTerms = new ArrayList<BytesRef>(terms);
+ Collections.shuffle(unsortedTerms, random);
+
+ for (BytesRef term : unsortedTerms) {
+ if (BasicOperations.run(automaton, term.utf8ToString())) {
+ // term is accepted
+ if (random.nextBoolean()) {
+ // seek exact
+ assertTrue(te.seekExact(term, random.nextBoolean()));
+ } else {
+ // seek ceil
+ assertEquals(SeekStatus.FOUND, te.seekCeil(term, random.nextBoolean()));
+ assertEquals(term, te.term());
+ }
+ }
+ }
+ }
+ }
+
+ /** mixes up seek and next for all terms */
+ public void testSeekingAndNexting() throws Exception {
+ for (int i = 0; i < numIterations; i++) {
+ TermsEnum te = MultiFields.getTerms(reader, "field").iterator();
+
+ for (BytesRef term : terms) {
+ int c = random.nextInt(3);
+ if (c == 0) {
+ assertEquals(term, te.next());
+ } else if (c == 1) {
+ assertEquals(SeekStatus.FOUND, te.seekCeil(term, random.nextBoolean()));
+ assertEquals(term, te.term());
+ } else {
+ assertTrue(te.seekExact(term, random.nextBoolean()));
+ }
+ }
+ }
+ }
+
+ /** tests intersect: TODO start at a random term! */
+ public void testIntersect() throws Exception {
+ for (int i = 0; i < numIterations; i++) {
+ String reg = AutomatonTestUtil.randomRegexp(random);
+ Automaton automaton = new RegExp(reg, RegExp.NONE).toAutomaton();
+ CompiledAutomaton ca = new CompiledAutomaton(automaton, SpecialOperations.isFinite(automaton), false);
+ TermsEnum te = MultiFields.getTerms(reader, "field").intersect(ca, null);
+ Automaton expected = BasicOperations.intersection(termsAutomaton, automaton);
+ TreeSet<BytesRef> found = new TreeSet<BytesRef>();
+ while (te.next() != null) {
+ found.add(new BytesRef(te.term()));
+ }
+
+ Automaton actual = DaciukMihovAutomatonBuilder.build(found);
+ assertTrue(BasicOperations.sameLanguage(expected, actual));
+ }
+ }
+}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java Sat Aug 20 19:20:45 2011
@@ -223,6 +223,7 @@ public class TestDocValuesIndexing exten
return cfg;
}
+ @SuppressWarnings("fallthrough")
public void runTestNumerics(IndexWriterConfig cfg, boolean withDeletions)
throws IOException {
Directory d = newDirectory();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java Sat Aug 20 19:20:45 2011
@@ -154,9 +154,7 @@ public class TestAutomatonQuery extends
assertEquals(a1, a2);
assertEquals(a1, a3);
-
- assertEquals(a1.toString(), a3.toString());
-
+
// different class
AutomatonQuery w1 = new WildcardQuery(newTerm("foobar"));
// different class
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java Sat Aug 20 19:20:45 2011
@@ -87,6 +87,7 @@ public class TestBooleanMinShouldMatch e
printHits(getName(), h, s);
}
assertEquals("result count", expected, h.length);
+ //System.out.println("TEST: now check");
QueryUtils.check(random, q,s);
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java Sat Aug 20 19:20:45 2011
@@ -75,6 +75,9 @@ public class TestFuzzyQuery2 extends Luc
}
public void assertFromTestData(int codePointTable[]) throws Exception {
+ if (VERBOSE) {
+ System.out.println("TEST: codePointTable=" + codePointTable);
+ }
InputStream stream = getClass().getResourceAsStream("fuzzyTestData.txt");
BufferedReader reader = new BufferedReader(new InputStreamReader(stream, "UTF-8"));
@@ -83,6 +86,8 @@ public class TestFuzzyQuery2 extends Luc
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.KEYWORD, false)).setMergePolicy(newLogMergePolicy()));
+
+ writer.w.setInfoStream(VERBOSE ? System.out : null);
Document doc = new Document();
Field field = newField("field", "", Field.Store.NO, Field.Index.ANALYZED);
@@ -95,6 +100,9 @@ public class TestFuzzyQuery2 extends Luc
IndexReader r = writer.getReader();
IndexSearcher searcher = newSearcher(r);
+ if (VERBOSE) {
+ System.out.println("TEST: searcher=" + searcher);
+ }
writer.close();
String line;
while ((line = reader.readLine()) != null) {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java Sat Aug 20 19:20:45 2011
@@ -94,12 +94,18 @@ public class TestMultiTermConstantScore
public static Query csrq(String f, String l, String h, boolean il, boolean ih) {
TermRangeQuery query = TermRangeQuery.newStringRange(f, l, h, il, ih);
query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
+ if (VERBOSE) {
+ System.out.println("TEST: query=" + query);
+ }
return query;
}
public static Query csrq(String f, String l, String h, boolean il, boolean ih, MultiTermQuery.RewriteMethod method) {
TermRangeQuery query = TermRangeQuery.newStringRange(f, l, h, il, ih);
query.setRewriteMethod(method);
+ if (VERBOSE) {
+ System.out.println("TEST: query=" + query + " method=" + method);
+ }
return query;
}
@@ -275,6 +281,10 @@ public class TestMultiTermConstantScore
IndexReader reader = signedIndexReader;
IndexSearcher search = newSearcher(reader);
+ if (VERBOSE) {
+ System.out.println("TEST: reader=" + reader);
+ }
+
int medId = ((maxId - minId) / 2);
String minIP = pad(minId);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java Sat Aug 20 19:20:45 2011
@@ -18,28 +18,27 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.ArrayList;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
@@ -143,6 +142,9 @@ public class TestRegexpRandom2 extends L
int num = CodecProvider.getDefault().getFieldCodec("field").equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : atLeast(1000);
for (int i = 0; i < num; i++) {
String reg = AutomatonTestUtil.randomRegexp(random);
+ if (VERBOSE) {
+ System.out.println("TEST: regexp=" + reg);
+ }
assertSame(reg);
}
}
@@ -153,18 +155,7 @@ public class TestRegexpRandom2 extends L
protected void assertSame(String regexp) throws IOException {
RegexpQuery smart = new RegexpQuery(new Term("field", regexp), RegExp.NONE);
DumbRegexpQuery dumb = new DumbRegexpQuery(new Term("field", regexp), RegExp.NONE);
-
- // we can't compare the two if automaton rewrites to a simpler enum.
- // for example: "a\uda07\udcc7?.*?" gets rewritten to a simpler query:
- // a\uda07* prefixquery. Prefixquery then does the "wrong" thing, which
- // isn't really wrong as the query was undefined to begin with... but not
- // automatically comparable.
-
- // TODO: does this check even matter anymore?!
- Terms terms = MultiFields.getTerms(searcher1.getIndexReader(), "field");
- if (!(smart.getTermsEnum(terms) instanceof AutomatonTermsEnum))
- return;
-
+
TopDocs smartDocs = searcher1.search(smart, 25);
TopDocs dumbDocs = searcher2.search(dumb, 25);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestWildcard.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestWildcard.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestWildcard.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestWildcard.java Sat Aug 20 19:20:45 2011
@@ -135,7 +135,7 @@ public class TestWildcard
wq = new WildcardQuery(new Term("field", "*"));
assertMatches(searcher, wq, 2);
assertFalse(wq.getTermsEnum(terms) instanceof PrefixTermsEnum);
- assertFalse(wq.getTermsEnum(terms) instanceof AutomatonTermsEnum);
+ assertFalse(wq.getTermsEnum(terms).getClass().getSimpleName().contains("AutomatonTermsEnum"));
searcher.close();
indexStore.close();
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java Sat Aug 20 19:20:45 2011
@@ -63,6 +63,9 @@ public class TestWildcardRandom extends
reader = writer.getReader();
searcher = newSearcher(reader);
writer.close();
+ if (VERBOSE) {
+ System.out.println("TEST: setUp searcher=" + searcher);
+ }
}
private char N() {
@@ -85,7 +88,11 @@ public class TestWildcardRandom extends
private void assertPatternHits(String pattern, int numHits) throws Exception {
// TODO: run with different rewrites
- Query wq = new WildcardQuery(new Term("field", fillPattern(pattern)));
+ final String filledPattern = fillPattern(pattern);
+ if (VERBOSE) {
+ System.out.println("TEST: run wildcard pattern=" + pattern + " filled=" + filledPattern);
+ }
+ Query wq = new WildcardQuery(new Term("field", filledPattern));
TopDocs docs = searcher.search(wq, 25);
assertEquals("Incorrect hits for pattern: " + pattern, numHits, docs.totalHits);
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Sat Aug 20 19:20:45 2011
@@ -152,11 +152,14 @@ public class TestPayloadNearQuery extend
}
for (int i=1;i<10;i++) {
query = newPhraseQuery("field", English.intToEnglish(i)+" hundred", true, new AveragePayloadFunction());
+ if (VERBOSE) {
+ System.out.println("TEST: run query=" + query);
+ }
// all should have score = 3 because adjacent terms have payloads of 2,4
// and all the similarity factors are set to 1
hits = searcher.search(query, null, 100);
assertTrue("hits is null and it shouldn't be", hits != null);
- assertTrue("should be 100 hits", hits.totalHits == 100);
+ assertEquals("should be 100 hits", 100, hits.totalHits);
for (int j = 0; j < hits.scoreDocs.length; j++) {
ScoreDoc doc = hits.scoreDocs[j];
// System.out.println("Doc: " + doc.toString());
Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java?rev=1159905&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java (added)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java Sat Aug 20 19:20:45 2011
@@ -0,0 +1,121 @@
+package org.apache.lucene.util.automaton;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+
+public class TestCompiledAutomaton extends LuceneTestCase {
+
+ private CompiledAutomaton build(String... strings) {
+ final List<Automaton> as = new ArrayList<Automaton>();
+ for(String s : strings) {
+ as.add(BasicAutomata.makeString(s));
+ }
+ Automaton a = BasicOperations.union(as);
+ a.determinize();
+ return new CompiledAutomaton(a, true, false);
+ }
+
+ private void testFloor(CompiledAutomaton c, String input, String expected) {
+ final BytesRef b = new BytesRef(input);
+ final BytesRef result = c.floor(b, b);
+ if (expected == null) {
+ assertNull(result);
+ } else {
+ assertNotNull(result);
+ assertEquals("actual=" + result.utf8ToString() + " vs expected=" + expected + " (input=" + input + ")",
+ result, new BytesRef(expected));
+ }
+ }
+
+ private void testTerms(String[] terms) throws Exception {
+ final CompiledAutomaton c = build(terms);
+ final BytesRef[] termBytes = new BytesRef[terms.length];
+ for(int idx=0;idx<terms.length;idx++) {
+ termBytes[idx] = new BytesRef(terms[idx]);
+ }
+ Arrays.sort(termBytes);
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: terms in unicode order");
+ for(BytesRef t : termBytes) {
+ System.out.println(" " + t.utf8ToString());
+ }
+ //System.out.println(c.utf8.toDot());
+ }
+
+ for(int iter=0;iter<100*RANDOM_MULTIPLIER;iter++) {
+ final String s = random.nextInt(10) == 1 ? terms[random.nextInt(terms.length)] : randomString();
+ if (VERBOSE) {
+ System.out.println("\nTEST: floor(" + s + ")");
+ }
+ int loc = Arrays.binarySearch(termBytes, new BytesRef(s));
+ final String expected;
+ if (loc >= 0) {
+ expected = s;
+ } else {
+ // term doesn't exist
+ loc = -(loc+1);
+ if (loc == 0) {
+ expected = null;
+ } else {
+ expected = termBytes[loc-1].utf8ToString();
+ }
+ }
+ if (VERBOSE) {
+ System.out.println(" expected=" + expected);
+ }
+ testFloor(c, s, expected);
+ }
+ }
+
+ public void testRandom() throws Exception {
+ final int numTerms = atLeast(1000);
+ final Set<String> terms = new HashSet<String>();
+ while(terms.size() != numTerms) {
+ terms.add(randomString());
+ }
+ testTerms(terms.toArray(new String[terms.size()]));
+ }
+
+ private String randomString() {
+ // return _TestUtil.randomSimpleString(random);
+ return _TestUtil.randomRealisticUnicodeString(random);
+ }
+
+ public void testBasic() throws Exception {
+ CompiledAutomaton c = build("foo", "fob", "goo");
+ testFloor(c, "goo", "goo");
+ testFloor(c, "ga", "foo");
+ testFloor(c, "g", "foo");
+ testFloor(c, "foc", "fob");
+ testFloor(c, "foz", "foo");
+ testFloor(c, "f", null);
+ testFloor(c, "", null);
+ testFloor(c, "aa", null);
+ testFloor(c, "zzz", "goo");
+ }
+}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java Sat Aug 20 19:20:45 2011
@@ -24,19 +24,25 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
+import java.io.StringWriter;
import java.io.Writer;
import java.util.*;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IndexInput;
@@ -456,7 +462,8 @@ public class TestFSTs extends LuceneTest
prune1==0 && prune2==0,
allowRandomSuffixSharing ? random.nextBoolean() : true,
allowRandomSuffixSharing ? _TestUtil.nextInt(random, 1, 10) : Integer.MAX_VALUE,
- outputs);
+ outputs,
+ null);
for(InputOutput<T> pair : pairs) {
if (pair.output instanceof UpToTwoPositiveIntOutputs.TwoLongs) {
@@ -872,15 +879,15 @@ public class TestFSTs extends LuceneTest
}
}
- //System.out.println("TEST: after prune");
- /*
- for(Map.Entry<BytesRef,CountMinOutput> ent : prefixes.entrySet()) {
- System.out.println(" " + inputToString(inputMode, ent.getKey()) + ": isLeaf=" + ent.getValue().isLeaf + " isFinal=" + ent.getValue().isFinal);
- if (ent.getValue().isFinal) {
- System.out.println(" finalOutput=" + outputs.outputToString(ent.getValue().finalOutput));
- }
+ if (VERBOSE) {
+ System.out.println("TEST: after prune");
+ for(Map.Entry<IntsRef,CountMinOutput<T>> ent : prefixes.entrySet()) {
+ System.out.println(" " + inputToString(inputMode, ent.getKey()) + ": isLeaf=" + ent.getValue().isLeaf + " isFinal=" + ent.getValue().isFinal);
+ if (ent.getValue().isFinal) {
+ System.out.println(" finalOutput=" + outputs.outputToString(ent.getValue().finalOutput));
+ }
}
- */
+ }
if (prefixes.size() <= 1) {
assertNull(fst);
@@ -1081,7 +1088,7 @@ public class TestFSTs extends LuceneTest
final BytesRef randomTerm = new BytesRef(getRandomString());
if (VERBOSE) {
- System.out.println("TEST: seek " + randomTerm.utf8ToString() + " " + randomTerm);
+ System.out.println("TEST: seek non-exist " + randomTerm.utf8ToString() + " " + randomTerm);
}
final TermsEnum.SeekStatus seekResult = termsEnum.seekCeil(randomTerm);
@@ -1133,10 +1140,10 @@ public class TestFSTs extends LuceneTest
assertEquals(termsEnum.term().utf8ToString() + " != " + fstEnum.current().input.utf8ToString(), termsEnum.term(), fstEnum.current().input);
if (storeOrd) {
// fst stored the ord
- assertEquals(termsEnum.ord(), ((Long) fstEnum.current().output).longValue());
+ assertEquals("term=" + termsEnum.term().utf8ToString() + " " + termsEnum.term(), termsEnum.ord(), ((Long) fstEnum.current().output).longValue());
} else {
// fst stored the docFreq
- assertEquals(termsEnum.docFreq(), (int) (((Long) fstEnum.current().output).longValue()));
+ assertEquals("term=" + termsEnum.term().utf8ToString() + " " + termsEnum.term(), termsEnum.docFreq(), (int) (((Long) fstEnum.current().output).longValue()));
}
}
}
@@ -1154,7 +1161,7 @@ public class TestFSTs extends LuceneTest
this.inputMode = inputMode;
this.outputs = outputs;
- builder = new Builder<T>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, 0, prune, prune == 0, true, Integer.MAX_VALUE, outputs);
+ builder = new Builder<T>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, 0, prune, prune == 0, true, Integer.MAX_VALUE, outputs, null);
}
protected abstract T getOutput(IntsRef input, int ord) throws IOException;
@@ -1254,7 +1261,7 @@ public class TestFSTs extends LuceneTest
}
}
- // java -cp build/classes/test:build/classes/java:build/classes/test-framework:lib/junit-4.7.jar org.apache.lucene.util.fst.TestFSTs /x/tmp/allTerms3.txt out
+ // java -cp build/classes/test:build/classes/test-framework:build/classes/java:lib/junit-4.7.jar org.apache.lucene.util.automaton.fst.TestFSTs /x/tmp/allTerms3.txt out
public static void main(String[] args) throws IOException {
int prune = 0;
int limit = Integer.MAX_VALUE;
@@ -1411,6 +1418,198 @@ public class TestFSTs extends LuceneTest
assertEquals(42, (long) seekResult.output);
}
+ public void testPrimaryKeys() throws Exception {
+ Directory dir = newDirectory();
+
+ for(int cycle=0;cycle<2;cycle++) {
+ if (VERBOSE) {
+ System.out.println("TEST: cycle=" + cycle);
+ }
+ RandomIndexWriter w = new RandomIndexWriter(random, dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+ Document doc = new Document();
+ Field idField = newField("id", "", Field.Index.NOT_ANALYZED);
+ doc.add(idField);
+
+ final int NUM_IDS = (int) (1000*RANDOM_MULTIPLIER*(1.0+random.nextDouble()));
+ //final int NUM_IDS = (int) (377 * (1.0+random.nextDouble()));
+ if (VERBOSE) {
+ System.out.println("TEST: NUM_IDS=" + NUM_IDS);
+ }
+ final Set<String> allIDs = new HashSet<String>();
+ for(int id=0;id<NUM_IDS;id++) {
+ String idString;
+ if (cycle == 0) {
+ // PKs are assigned sequentially
+ idString = String.format("%07d", id);
+ } else {
+ while(true) {
+ final String s = Long.toString(random.nextLong());
+ if (!allIDs.contains(s)) {
+ idString = s;
+ break;
+ }
+ }
+ }
+ allIDs.add(idString);
+ idField.setValue(idString);
+ w.addDocument(doc);
+ }
+
+ //w.optimize();
+
+ // turn writer into reader:
+ final IndexReader r = w.getReader();
+ final IndexSearcher s = new IndexSearcher(r);
+ w.close();
+
+ final List<String> allIDsList = new ArrayList<String>(allIDs);
+ final List<String> sortedAllIDsList = new ArrayList<String>(allIDsList);
+ Collections.sort(sortedAllIDsList);
+
+ // Sprinkle in some non-existent PKs:
+ Set<String> outOfBounds = new HashSet<String>();
+ for(int idx=0;idx<NUM_IDS/10;idx++) {
+ String idString;
+ if (cycle == 0) {
+ idString = String.format("%07d", (NUM_IDS + idx));
+ } else {
+ while(true) {
+ idString = Long.toString(random.nextLong());
+ if (!allIDs.contains(idString)) {
+ break;
+ }
+ }
+ }
+ outOfBounds.add(idString);
+ allIDsList.add(idString);
+ }
+
+ // Verify w/ TermQuery
+ for(int iter=0;iter<2*NUM_IDS;iter++) {
+ final String id = allIDsList.get(random.nextInt(allIDsList.size()));
+ final boolean exists = !outOfBounds.contains(id);
+ if (VERBOSE) {
+ System.out.println("TEST: TermQuery " + (exists ? "" : "non-exist ") + " id=" + id);
+ }
+ assertEquals((exists ? "" : "non-exist ") + "id=" + id, exists ? 1 : 0, s.search(new TermQuery(new Term("id", id)), 1).totalHits);
+ }
+
+ // Verify w/ MultiTermsEnum
+ final TermsEnum termsEnum = MultiFields.getTerms(r, "id").iterator();
+ for(int iter=0;iter<2*NUM_IDS;iter++) {
+ final String id;
+ final String nextID;
+ final boolean exists;
+
+ if (random.nextBoolean()) {
+ id = allIDsList.get(random.nextInt(allIDsList.size()));
+ exists = !outOfBounds.contains(id);
+ nextID = null;
+ if (VERBOSE) {
+ System.out.println("TEST: exactOnly " + (exists ? "" : "non-exist ") + "id=" + id);
+ }
+ } else {
+ // Pick ID between two IDs:
+ exists = false;
+ final int idv = random.nextInt(NUM_IDS-1);
+ if (cycle == 0) {
+ id = String.format("%07da", idv);
+ nextID = String.format("%07d", idv+1);
+ } else {
+ id = sortedAllIDsList.get(idv) + "a";
+ nextID = sortedAllIDsList.get(idv+1);
+ }
+ if (VERBOSE) {
+ System.out.println("TEST: not exactOnly id=" + id + " nextID=" + nextID);
+ }
+ }
+
+ final boolean useCache = random.nextBoolean();
+ if (VERBOSE) {
+ System.out.println(" useCache=" + useCache);
+ }
+
+ final TermsEnum.SeekStatus status;
+ if (nextID == null) {
+ if (termsEnum.seekExact(new BytesRef(id), useCache)) {
+ status = TermsEnum.SeekStatus.FOUND;
+ } else {
+ status = TermsEnum.SeekStatus.NOT_FOUND;
+ }
+ } else {
+ status = termsEnum.seekCeil(new BytesRef(id), useCache);
+ }
+
+ if (nextID != null) {
+ assertEquals(TermsEnum.SeekStatus.NOT_FOUND, status);
+ assertEquals("expected=" + nextID + " actual=" + termsEnum.term().utf8ToString(), new BytesRef(nextID), termsEnum.term());
+ } else if (!exists) {
+ assertTrue(status == TermsEnum.SeekStatus.NOT_FOUND ||
+ status == TermsEnum.SeekStatus.END);
+ } else {
+ assertEquals(TermsEnum.SeekStatus.FOUND, status);
+ }
+ }
+
+ r.close();
+ }
+ dir.close();
+ }
+
+ public void testRandomTermLookup() throws Exception {
+ Directory dir = newDirectory();
+
+ RandomIndexWriter w = new RandomIndexWriter(random, dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+ w.w.setInfoStream(VERBOSE ? System.out : null);
+
+ Document doc = new Document();
+ Field f = newField("field", "", Field.Index.NOT_ANALYZED);
+ doc.add(f);
+
+ final int NUM_TERMS = (int) (1000*RANDOM_MULTIPLIER * (1+random.nextDouble()));
+ if (VERBOSE) {
+ System.out.println("TEST: NUM_TERMS=" + NUM_TERMS);
+ }
+
+ final Set<String> allTerms = new HashSet<String>();
+ while(allTerms.size() < NUM_TERMS) {
+ allTerms.add(simpleRandomString(random));
+ }
+
+ for(String term : allTerms) {
+ f.setValue(term);
+ w.addDocument(doc);
+ }
+
+ // turn writer into reader:
+ if (VERBOSE) {
+ System.out.println("TEST: get reader");
+ }
+ IndexReader r = w.getReader();
+ if (VERBOSE) {
+ System.out.println("TEST: got reader=" + r);
+ }
+ IndexSearcher s = new IndexSearcher(r);
+ w.close();
+
+ final List<String> allTermsList = new ArrayList<String>(allTerms);
+ Collections.shuffle(allTermsList, random);
+
+ // verify exact lookup
+ for(String term : allTermsList) {
+ if (VERBOSE) {
+ System.out.println("TEST: term=" + term);
+ }
+ assertEquals("term=" + term, 1, s.search(new TermQuery(new Term("field", term)), 1).totalHits);
+ }
+
+ r.close();
+ dir.close();
+ }
+
+
/**
* Test state expansion (array format) on close-to-root states. Creates
* synthetic input that has one expanded state on each level.
@@ -1492,6 +1691,36 @@ public class TestFSTs extends LuceneTest
s.verifyStateAndBelow(fst, arc, 1);
}
+ public void testFinalOutputOnEndState() throws Exception {
+ final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
+
+ final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE4, 2, 0, true, true, Integer.MAX_VALUE, outputs, null);
+ builder.add("stat", outputs.get(17));
+ builder.add("station", outputs.get(10));
+ final FST<Long> fst = builder.finish();
+ //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot"));
+ StringWriter w = new StringWriter();
+ Util.toDot(fst, w, false, false);
+ w.close();
+ //System.out.println(w.toString());
+ assertTrue(w.toString().indexOf("label=\"t/[7]\"") != -1);
+ }
+
+ public void testInternalFinalState() throws Exception {
+ final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
+
+ final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null);
+ builder.add(new BytesRef("stat"), outputs.getNoOutput());
+ builder.add(new BytesRef("station"), outputs.getNoOutput());
+ final FST<Long> fst = builder.finish();
+ StringWriter w = new StringWriter();
+ //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot"));
+ Util.toDot(fst, w, false, false);
+ w.close();
+ //System.out.println(w.toString());
+ assertTrue(w.toString().indexOf("6 [shape=doublecircle") != -1);
+ }
+
// Make sure raw FST can differentiate between final vs
// non-final end nodes
public void testNonFinalStopNodes() throws Exception {
Modified: lucene/dev/trunk/modules/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java?rev=1159905&r1=1159904&r2=1159905&view=diff
==============================================================================
--- lucene/dev/trunk/modules/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java (original)
+++ lucene/dev/trunk/modules/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java Sat Aug 20 19:20:45 2011
@@ -400,7 +400,7 @@ public class DirectSpellChecker {
if (terms == null) {
return Collections.emptyList();
}
- FuzzyTermsEnum e = new FuzzyTermsEnum(terms.iterator(), atts, term, editDistance, Math.max(minPrefix, editDistance-1));
+ FuzzyTermsEnum e = new FuzzyTermsEnum(terms, atts, term, editDistance, Math.max(minPrefix, editDistance-1));
final PriorityQueue<ScoreTerm> stQueue = new PriorityQueue<ScoreTerm>();
BytesRef queryTerm = new BytesRef(term.text());