You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/05/02 15:51:22 UTC
svn commit: r1098566 [22/22] - in /lucene/dev/branches/docvalues: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ant/
dev-tools/idea/lucene/contrib/db/bdb-je/
dev-tools/idea/lucene/contrib/db/bdb/ dev-tools/idea/lucene/contri...
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/request/TestFaceting.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/request/TestFaceting.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/request/TestFaceting.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/request/TestFaceting.java Mon May 2 13:50:57 2011
@@ -17,14 +17,17 @@
package org.apache.solr.request;
+import java.util.Locale;
+import java.util.Random;
+
+import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.SolrTestCaseJ4;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.util.Locale;
-import java.util.Random;
/**
* @version $Id$
@@ -62,43 +65,47 @@ public class TestFaceting extends SolrTe
}
void doTermEnum(int size) throws Exception {
+ //System.out.println("doTermEnum size=" + size);
close();
createIndex(size);
req = lrf.makeRequest("q","*:*");
- TermIndex ti = new TermIndex(proto.field());
- NumberedTermsEnum te = ti.getEnumerator(req.getSearcher().getIndexReader());
+ UnInvertedField uif = new UnInvertedField(proto.field(), req.getSearcher());
- // iterate through first
- while(te.term() != null) te.next();
- assertEquals(size, te.getTermNumber());
- te.close();
+ assertEquals(size, uif.getNumTerms());
- te = ti.getEnumerator(req.getSearcher().getIndexReader());
+ TermsEnum te = uif.getOrdTermsEnum(req.getSearcher().getIndexReader());
+ assertEquals(size == 0, te == null);
Random r = new Random(size);
// test seeking by term string
for (int i=0; i<size*2+10; i++) {
int rnum = r.nextInt(size+2);
String s = t(rnum);
- BytesRef br = te.skipTo(new BytesRef(s));
+ //System.out.println("s=" + s);
+ final BytesRef br;
+ if (te == null) {
+ br = null;
+ } else {
+ TermsEnum.SeekStatus status = te.seek(new BytesRef(s));
+ if (status == TermsEnum.SeekStatus.END) {
+ br = null;
+ } else {
+ br = te.term();
+ }
+ }
assertEquals(br != null, rnum < size);
if (rnum < size) {
- assertEquals(rnum, te.pos);
+ assertEquals(rnum, (int) te.ord());
assertEquals(s, te.term().utf8ToString());
- } else {
- assertEquals(null, te.term());
- assertEquals(size, te.getTermNumber());
}
}
// test seeking before term
- assertEquals(size>0, te.skipTo(new BytesRef("000")) != null);
- assertEquals(0, te.getTermNumber());
if (size>0) {
+ assertEquals(size>0, te.seek(new BytesRef("000"), true) != TermsEnum.SeekStatus.END);
+ assertEquals(0, te.ord());
assertEquals(t(0), te.term().utf8ToString());
- } else {
- assertEquals(null, te.term());
}
if (size>0) {
@@ -106,9 +113,10 @@ public class TestFaceting extends SolrTe
for (int i=0; i<size*2+10; i++) {
int rnum = r.nextInt(size);
String s = t(rnum);
- BytesRef br = te.skipTo(rnum);
+ assertTrue(te.seek((long) rnum) != TermsEnum.SeekStatus.END);
+ BytesRef br = te.term();
assertNotNull(br);
- assertEquals(rnum, te.pos);
+ assertEquals(rnum, (int) te.ord());
assertEquals(s, te.term().utf8ToString());
}
}
@@ -118,11 +126,12 @@ public class TestFaceting extends SolrTe
public void testTermEnum() throws Exception {
doTermEnum(0);
doTermEnum(1);
- doTermEnum(TermIndex.interval - 1); // test boundaries around the block size
- doTermEnum(TermIndex.interval);
- doTermEnum(TermIndex.interval + 1);
- doTermEnum(TermIndex.interval * 2 + 2);
- // doTermEnum(TermIndex.interval * 3 + 3);
+ final int DEFAULT_INDEX_INTERVAL = 1 << DocTermOrds.DEFAULT_INDEX_INTERVAL_BITS;
+ doTermEnum(DEFAULT_INDEX_INTERVAL - 1); // test boundaries around the block size
+ doTermEnum(DEFAULT_INDEX_INTERVAL);
+ doTermEnum(DEFAULT_INDEX_INTERVAL + 1);
+ doTermEnum(DEFAULT_INDEX_INTERVAL * 2 + 2);
+ // doTermEnum(DEFAULT_INDEX_INTERVAL * 3 + 3);
}
@Test
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/CustomSimilarityFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/CustomSimilarityFactory.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/CustomSimilarityFactory.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/CustomSimilarityFactory.java Mon May 2 13:50:57 2011
@@ -16,11 +16,11 @@
*/
package org.apache.solr.schema;
-import org.apache.lucene.search.SimilarityProvider;
+import org.apache.lucene.search.Similarity;
public class CustomSimilarityFactory extends SimilarityFactory {
@Override
- public SimilarityProvider getSimilarityProvider() {
+ public Similarity getSimilarity() {
return new MockConfigurableSimilarity(params.get("echo"));
}
}
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/DateFieldTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/DateFieldTest.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/DateFieldTest.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/DateFieldTest.java Mon May 2 13:50:57 2011
@@ -17,6 +17,8 @@
package org.apache.solr.schema;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.schema.DateField;
import org.apache.solr.util.DateMathParser;
@@ -114,4 +116,13 @@ public class DateFieldTest extends Lucen
}
+ public void testCreateField() {
+ int props = FieldProperties.INDEXED ^ FieldProperties.STORED;
+ SchemaField sf = new SchemaField( "test", f, props, null );
+ Fieldable out = (Field)f.createField(sf, "1995-12-31T23:59:59Z", 1.0f );
+ assertEquals(820454399000l, f.toObject( out ).getTime() );
+
+ out = (Field)f.createField(sf, new Date(820454399000l), 1.0f );
+ assertEquals(820454399000l, f.toObject( out ).getTime() );
+ }
}
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/IndexSchemaTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/IndexSchemaTest.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/IndexSchemaTest.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/IndexSchemaTest.java Mon May 2 13:50:57 2011
@@ -81,11 +81,11 @@ public class IndexSchemaTest extends Sol
}
@Test
- public void testSimilarityFactory() {
+ public void testSimilarityProviderFactory() {
SolrCore core = h.getCore();
- SimilarityProvider similarity = core.getSchema().getSimilarityProvider();
- assertTrue("wrong class", similarity instanceof MockConfigurableSimilarity);
- assertEquals("is there an echo?", ((MockConfigurableSimilarity)similarity).getPassthrough());
+ SimilarityProvider similarityProvider = core.getSchema().getSimilarityProvider();
+ assertTrue("wrong class", similarityProvider instanceof MockConfigurableSimilarityProvider);
+ assertEquals("is there an echo?", ((MockConfigurableSimilarityProvider)similarityProvider).getPassthrough());
}
@Test
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java Mon May 2 13:50:57 2011
@@ -16,9 +16,9 @@
*/
package org.apache.solr.schema;
-import org.apache.lucene.search.DefaultSimilarityProvider;
+import org.apache.lucene.search.DefaultSimilarity;
-public class MockConfigurableSimilarity extends DefaultSimilarityProvider {
+public class MockConfigurableSimilarity extends DefaultSimilarity {
private String passthrough;
public MockConfigurableSimilarity(String passthrough) {
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/QueryParsingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/QueryParsingTest.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/QueryParsingTest.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/QueryParsingTest.java Mon May 2 13:50:57 2011
@@ -99,15 +99,15 @@ public class QueryParsingTest extends So
assertEquals(flds[0].getType(), SortField.CUSTOM);
//Not thrilled about the fragility of string matching here, but...
//the value sources get wrapped, so the out field is different than the input
- assertEquals(flds[0].getField(), "pow(float(weight),const(2.0))");
+ assertEquals(flds[0].getField(), "pow(float(weight),const(2))");
//test functions (more deep)
- sort = QueryParsing.parseSort("sum(product(r_f1,sum(d_f1,t_f1,1)),a_f1) asc", req);
+ sort = QueryParsing.parseSort("sum(product(r_f1,sum(d_f1,t_f1,1.0)),a_f1) asc", req);
flds = sort.getSort();
assertEquals(flds[0].getType(), SortField.CUSTOM);
assertEquals(flds[0].getField(), "sum(product(float(r_f1),sum(float(d_f1),float(t_f1),const(1.0))),float(a_f1))");
- sort = QueryParsing.parseSort("pow(weight, 2) desc", req);
+ sort = QueryParsing.parseSort("pow(weight, 2.0) desc", req);
flds = sort.getSort();
assertEquals(flds[0].getType(), SortField.CUSTOM);
//Not thrilled about the fragility of string matching here, but...
@@ -115,7 +115,7 @@ public class QueryParsingTest extends So
assertEquals(flds[0].getField(), "pow(float(weight),const(2.0))");
- sort = QueryParsing.parseSort("pow(weight, 2) desc, weight desc, bday asc", req);
+ sort = QueryParsing.parseSort("pow(weight, 2.0) desc, weight desc, bday asc", req);
flds = sort.getSort();
assertEquals(flds[0].getType(), SortField.CUSTOM);
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/TestExtendedDismaxParser.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/TestExtendedDismaxParser.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/TestExtendedDismaxParser.java Mon May 2 13:50:57 2011
@@ -49,8 +49,13 @@ public class TestExtendedDismaxParser ex
"name", "The Zapper"));
assertU(adoc("id", "45", "trait_ss", "Chauvinist",
"title", "25 star General"));
- assertU(adoc("id", "46", "trait_ss", "Obnoxious",
- "subject", "Defeated the pacifists op the Gandhi nebula"));
+ assertU(adoc("id", "46",
+ "trait_ss", "Obnoxious",
+ "subject", "Defeated the pacifists op the Gandhi nebula",
+ "t_special", "literal:colon value",
+ "movies_t", "first is Mission: Impossible, second is Terminator 2: Judgement Day. Terminator:3 ok...",
+ "foo_i", "8"
+ ));
assertU(adoc("id", "47", "trait_ss", "Pig",
"text", "line up and fly directly at the enemy death cannons, clogging them with wreckage!"));
assertU(adoc("id", "48", "text_sw", "this has gigabyte potential", "foo_i","100"));
@@ -64,6 +69,11 @@ public class TestExtendedDismaxParser ex
String twor = "*[count(//doc)=2]";
String nor = "*[count(//doc)=0]";
+ assertQ("expected doc is missing (using un-escaped edismax w/qf)",
+ req("q", "literal:colon",
+ "qf", "t_special",
+ "defType", "edismax"),
+ "//doc[1]/str[@name='id'][.='46']");
assertQ("standard request handler returns all matches",
req(allq),
@@ -164,6 +174,58 @@ public class TestExtendedDismaxParser ex
"q","the big"), oner
);
+ // searching for a literal colon value when clearly not used for a field
+ assertQ("expected doc is missing (using standard)",
+ req("q", "t_special:literal\\:colon"),
+ "//doc[1]/str[@name='id'][.='46']");
+ assertQ("expected doc is missing (using escaped edismax w/field)",
+ req("q", "t_special:literal\\:colon",
+ "defType", "edismax"),
+ "//doc[1]/str[@name='id'][.='46']");
+ assertQ("expected doc is missing (using un-escaped edismax w/field)",
+ req("q", "t_special:literal:colon",
+ "defType", "edismax"),
+ "//doc[1]/str[@name='id'][.='46']");
+ assertQ("expected doc is missing (using escaped edismax w/qf)",
+ req("q", "literal\\:colon",
+ "qf", "t_special",
+ "defType", "edismax"),
+ "//doc[1]/str[@name='id'][.='46']");
+ assertQ("expected doc is missing (using un-escaped edismax w/qf)",
+ req("q", "literal:colon",
+ "qf", "t_special",
+ "defType", "edismax"),
+ "//doc[1]/str[@name='id'][.='46']");
+
+ assertQ(req("defType","edismax", "mm","100%", "q","terminator:3", "qf","movies_t"),
+ oner);
+ assertQ(req("defType","edismax", "mm","100%", "q","Mission:Impossible", "qf","movies_t"),
+ oner);
+ assertQ(req("defType","edismax", "mm","100%", "q","Mission : Impossible", "qf","movies_t"),
+ oner);
+ assertQ(req("defType","edismax", "mm","100%", "q","Mission: Impossible", "qf","movies_t"),
+ oner);
+ assertQ(req("defType","edismax", "mm","100%", "q","Terminator 2: Judgement Day", "qf","movies_t"),
+ oner);
+
+ // make sure the clause wasn't eliminated
+ assertQ(req("defType","edismax", "mm","100%", "q","Terminator 10: Judgement Day", "qf","movies_t"),
+ nor);
+
+ // throw in a numeric field
+ assertQ(req("defType","edismax", "mm","0", "q","Terminator: 100", "qf","movies_t foo_i"),
+ twor);
+
+ assertQ(req("defType","edismax", "mm","100%", "q","Terminator: 100", "qf","movies_t foo_i"),
+ nor);
+
+ assertQ(req("defType","edismax", "mm","100%", "q","Terminator: 8", "qf","movies_t foo_i"),
+ oner);
+
+ assertQ(req("defType","edismax", "mm","0", "q","movies_t:Terminator 100", "qf","movies_t foo_i"),
+ twor);
+
+
/** stopword removal in conjunction with multi-word synonyms at query time
* break this test.
// multi-word synonyms
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/TestQueryTypes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/TestQueryTypes.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/TestQueryTypes.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/TestQueryTypes.java Mon May 2 13:50:57 2011
@@ -43,6 +43,7 @@ public class TestQueryTypes extends Abst
public void testQueryTypes() {
+ assertU(adoc("id","0"));
assertU(adoc("id","1", "v_t","Hello Dude"));
assertU(adoc("id","2", "v_t","Hello Yonik"));
assertU(adoc("id","3", "v_s","{!literal}"));
@@ -120,6 +121,7 @@ public class TestQueryTypes extends Abst
);
// function query... just make sure it doesn't throw an exception
+ if ("v_s".equals(f)) continue; // in this context, functions must be able to be interpreted as a float
assertQ(req( "q", "+id:999 _val_:\"" + f + "\"")
,"//result[@numFound='1']"
);
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java Mon May 2 13:50:57 2011
@@ -24,6 +24,7 @@ import org.apache.lucene.search.Similari
import org.apache.solr.SolrTestCaseJ4;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.Ignore;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
@@ -71,16 +72,14 @@ public class TestFunctionQuery extends S
// replace \0 with the field name and create a parseable string
public String func(String field, String template) {
- StringBuilder sb = new StringBuilder("_val_:\"");
+ StringBuilder sb = new StringBuilder("{!func}");
for (char ch : template.toCharArray()) {
if (ch=='\0') {
sb.append(field);
continue;
}
- if (ch=='"') sb.append('\\');
sb.append(ch);
}
- sb.append('"');
return sb.toString();
}
@@ -520,5 +519,66 @@ public class TestFunctionQuery extends S
dofunc("atan2(.25,.5)", Math.atan2(.25,.5));
}
+ /**
+ * verify that both the field("...") value source parser as well as
+ * ExternalFileField work with esoteric field names
+ */
+ @Test
+ public void testExternalFieldValueSourceParser() {
+ clearIndex();
+
+ String field = "CoMpleX fieldName _extf";
+ String fieldAsFunc = "field(\"CoMpleX fieldName _extf\")";
+
+ float[] ids = {100,-4,0,10,25,5,77,23,55,-78,-45,-24,63,78,94,22,34,54321,261,-627};
+
+ createIndex(null,ids);
+
+ // Unsorted field, largest first
+ makeExternalFile(field, "54321=543210\n0=-999\n25=250","UTF-8");
+ // test identity (straight field value)
+ singleTest(fieldAsFunc, "\0", 54321, 543210, 0,-999, 25,250, 100, 1);
+ Object orig = FileFloatSource.onlyForTesting;
+ singleTest(fieldAsFunc, "log(\0)");
+ // make sure the values were cached
+ assertTrue(orig == FileFloatSource.onlyForTesting);
+ singleTest(fieldAsFunc, "sqrt(\0)");
+ assertTrue(orig == FileFloatSource.onlyForTesting);
+
+ makeExternalFile(field, "0=1","UTF-8");
+ assertU(adoc("id", "10000")); // will get same reader if no index change
+ assertU(commit());
+ singleTest(fieldAsFunc, "sqrt(\0)");
+ assertTrue(orig != FileFloatSource.onlyForTesting);
+
+ purgeFieldCache(FieldCache.DEFAULT); // avoid FC insanity
+ }
+
+ /**
+ * some platforms don't allow quote characters in filenames, so
+ * in addition to testExternalFieldValueSourceParser above, test a field
+ * name with quotes in it that does NOT use ExternalFileField
+ * @see #testExternalFieldValueSourceParser
+ */
+ @Test
+ public void testFieldValueSourceParser() {
+ clearIndex();
+
+ String field = "CoMpleX \" fieldName _f";
+ String fieldAsFunc = "field(\"CoMpleX \\\" fieldName _f\")";
+
+ float[] ids = {100,-4,0,10,25,5,77,1};
+
+ createIndex(field, ids);
+
+ // test identity (straight field value)
+ singleTest(fieldAsFunc, "\0",
+ 100,100, -4,-4, 0,0, 10,10, 25,25, 5,5, 77,77, 1,1);
+ singleTest(fieldAsFunc, "sqrt(\0)",
+ 100,10, 25,5, 0,0, 1,1);
+ singleTest(fieldAsFunc, "log(\0)", 1,0);
+
+ purgeFieldCache(FieldCache.DEFAULT); // avoid FC insanity
+ }
}
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/servlet/NoCacheHeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/servlet/NoCacheHeaderTest.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/servlet/NoCacheHeaderTest.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/servlet/NoCacheHeaderTest.java Mon May 2 13:50:57 2011
@@ -31,7 +31,7 @@ import org.junit.Test;
public class NoCacheHeaderTest extends CacheHeaderTestBase {
@BeforeClass
public static void beforeTest() throws Exception {
- createJetty(TEST_HOME, "solr/conf/solrconfig-nocache.xml", null);
+ createJetty(TEST_HOME(), "solr/conf/solrconfig-nocache.xml", null);
}
// The tests
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/spelling/suggest/PersistenceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/spelling/suggest/PersistenceTest.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/spelling/suggest/PersistenceTest.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/spelling/suggest/PersistenceTest.java Mon May 2 13:50:57 2011
@@ -19,62 +19,74 @@ package org.apache.solr.spelling.suggest
import java.io.File;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.spelling.suggest.fst.FSTLookup;
import org.apache.solr.spelling.suggest.jaspell.JaspellLookup;
import org.apache.solr.spelling.suggest.tst.TSTLookup;
import org.junit.Test;
public class PersistenceTest extends SolrTestCaseJ4 {
-
- public static final String[] keys = new String[] {
- "one",
- "two",
- "three",
- "four",
- "oneness",
- "onerous",
- "onesimus",
- "twofold",
- "twonk",
- "thrive",
- "through",
- "threat",
- "foundation",
- "fourier",
- "fourty"
- };
+ public final String[] keys = new String[] {
+ "one",
+ "two",
+ "three",
+ "four",
+ "oneness",
+ "onerous",
+ "onesimus",
+ "twofold",
+ "twonk",
+ "thrive",
+ "through",
+ "threat",
+ "foundation",
+ "fourier",
+ "fourty"};
@Test
public void testTSTPersistence() throws Exception {
- TSTLookup lookup = new TSTLookup();
- for (String k : keys) {
- lookup.add(k, new Float(k.length()));
- }
- File storeDir = new File(TEST_HOME);
- lookup.store(storeDir);
- lookup = new TSTLookup();
- lookup.load(storeDir);
- for (String k : keys) {
- Float val = (Float)lookup.get(k);
- assertNotNull(k, val);
- assertEquals(k, k.length(), val.intValue());
- }
+ runTest(TSTLookup.class, true);
}
@Test
public void testJaspellPersistence() throws Exception {
- JaspellLookup lookup = new JaspellLookup();
- for (String k : keys) {
- lookup.add(k, new Float(k.length()));
- }
- File storeDir = new File(TEST_HOME);
+ runTest(JaspellLookup.class, true);
+ }
+
+ @Test
+ public void testFSTPersistence() throws Exception {
+ runTest(FSTLookup.class, false);
+ }
+
+ private void runTest(Class<? extends Lookup> lookupClass,
+ boolean supportsExactWeights) throws Exception {
+
+ // Add all input keys.
+ Lookup lookup = lookupClass.newInstance();
+ TermFreq[] keys = new TermFreq[this.keys.length];
+ for (int i = 0; i < keys.length; i++)
+ keys[i] = new TermFreq(this.keys[i], (float) i);
+ lookup.build(new TermFreqArrayIterator(keys));
+
+ // Store the suggester.
+ File storeDir = new File(TEST_HOME());
lookup.store(storeDir);
- lookup = new JaspellLookup();
+
+ // Re-read it from disk.
+ lookup = lookupClass.newInstance();
lookup.load(storeDir);
- for (String k : keys) {
- Float val = (Float)lookup.get(k);
- assertNotNull(k, val);
- assertEquals(k, k.length(), val.intValue());
+
+ // Assert validity.
+ float previous = Float.NEGATIVE_INFINITY;
+ for (TermFreq k : keys) {
+ Float val = (Float) lookup.get(k.term);
+ assertNotNull(k.term, val);
+
+ if (supportsExactWeights) {
+ assertEquals(k.term, Float.valueOf(k.v), val);
+ } else {
+ assertTrue(val + ">=" + previous, val >= previous);
+ previous = val.floatValue();
+ }
}
}
-
}
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/spelling/suggest/SuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/spelling/suggest/SuggesterTest.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/spelling/suggest/SuggesterTest.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/spelling/suggest/SuggesterTest.java Mon May 2 13:50:57 2011
@@ -17,23 +17,19 @@
package org.apache.solr.spelling.suggest;
-import org.apache.lucene.util.RamUsageEstimator;
+import java.io.File;
+
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.SpellingParams;
-import org.apache.solr.spelling.suggest.Lookup.LookupResult;
-import org.apache.solr.spelling.suggest.jaspell.JaspellLookup;
-import org.apache.solr.spelling.suggest.tst.TSTLookup;
-import org.apache.solr.util.TermFreqIterator;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.File;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
public class SuggesterTest extends SolrTestCaseJ4 {
+ /**
+ * Expected URI at which the given suggester will live.
+ */
+ protected String requestUri = "/suggest";
+
@BeforeClass
public static void beforeClass() throws Exception {
initCore("solrconfig-spellchecker.xml","schema-spellchecker.xml");
@@ -54,10 +50,9 @@ public class SuggesterTest extends SolrT
@Test
public void testSuggestions() throws Exception {
addDocs();
-
assertU(commit()); // configured to do a rebuild on commit
- assertQ(req("qt","/suggest", "q","ac", SpellingParams.SPELLCHECK_COUNT, "2", SpellingParams.SPELLCHECK_ONLY_MORE_POPULAR, "true"),
+ assertQ(req("qt", requestUri, "q", "ac", SpellingParams.SPELLCHECK_COUNT, "2", SpellingParams.SPELLCHECK_ONLY_MORE_POPULAR, "true"),
"//lst[@name='spellcheck']/lst[@name='suggestions']/lst[@name='ac']/int[@name='numFound'][.='2']",
"//lst[@name='spellcheck']/lst[@name='suggestions']/lst[@name='ac']/arr[@name='suggestion']/str[1][.='acquire']",
"//lst[@name='spellcheck']/lst[@name='suggestions']/lst[@name='ac']/arr[@name='suggestion']/str[2][.='accommodate']"
@@ -77,12 +72,12 @@ public class SuggesterTest extends SolrT
dataDir = data;
configString = config;
initCore();
- assertQ(req("qt","/suggest", "q","ac", SpellingParams.SPELLCHECK_COUNT, "2", SpellingParams.SPELLCHECK_ONLY_MORE_POPULAR, "true"),
+ assertQ(req("qt", requestUri, "q", "ac", SpellingParams.SPELLCHECK_COUNT, "2", SpellingParams.SPELLCHECK_ONLY_MORE_POPULAR, "true"),
"//lst[@name='spellcheck']/lst[@name='suggestions']/lst[@name='ac']/int[@name='numFound'][.='2']",
"//lst[@name='spellcheck']/lst[@name='suggestions']/lst[@name='ac']/arr[@name='suggestion']/str[1][.='acquire']",
"//lst[@name='spellcheck']/lst[@name='suggestions']/lst[@name='ac']/arr[@name='suggestion']/str[2][.='accommodate']"
);
-
+
// restore the property
System.setProperty("solr.test.leavedatadir", leaveData);
}
@@ -91,136 +86,13 @@ public class SuggesterTest extends SolrT
public void testRebuild() throws Exception {
addDocs();
assertU(commit());
- assertQ(req("qt","/suggest", "q","ac", SpellingParams.SPELLCHECK_COUNT, "2", SpellingParams.SPELLCHECK_ONLY_MORE_POPULAR, "true"),
- "//lst[@name='spellcheck']/lst[@name='suggestions']/lst[@name='ac']/int[@name='numFound'][.='2']");
+ assertQ(req("qt", requestUri, "q", "ac", SpellingParams.SPELLCHECK_COUNT, "2", SpellingParams.SPELLCHECK_ONLY_MORE_POPULAR, "true"),
+ "//lst[@name='spellcheck']/lst[@name='suggestions']/lst[@name='ac']/int[@name='numFound'][.='2']");
assertU(adoc("id", "4",
"text", "actually"
));
assertU(commit());
- assertQ(req("qt","/suggest", "q","ac", SpellingParams.SPELLCHECK_COUNT, "2", SpellingParams.SPELLCHECK_ONLY_MORE_POPULAR, "true"),
- "//lst[@name='spellcheck']/lst[@name='suggestions']/lst[@name='ac']/int[@name='numFound'][.='2']");
- }
-
-
- private TermFreqIterator getTFIT() {
- final int count = 100000;
- TermFreqIterator tfit = new TermFreqIterator() {
- Random r = new Random(1234567890L);
- Random r1 = new Random(1234567890L);
- int pos;
-
- public float freq() {
- return r1.nextInt(4);
- }
-
- public boolean hasNext() {
- return pos < count;
- }
-
- public String next() {
- pos++;
- return Long.toString(r.nextLong());
- }
-
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- return tfit;
- }
-
- private void _benchmark(Lookup lookup, Map<String,Integer> ref, boolean estimate, Bench bench) throws Exception {
- long start = System.currentTimeMillis();
- lookup.build(getTFIT());
- long buildTime = System.currentTimeMillis() - start;
- TermFreqIterator tfit = getTFIT();
- long elapsed = 0;
- while (tfit.hasNext()) {
- String key = tfit.next();
- // take only the first part of the key
- int len = key.length() > 4 ? key.length() / 3 : 2;
- String prefix = key.substring(0, len);
- start = System.nanoTime();
- List<LookupResult> res = lookup.lookup(prefix, true, 10);
- elapsed += System.nanoTime() - start;
- assertTrue(res.size() > 0);
- for (LookupResult lr : res) {
- assertTrue(lr.key.startsWith(prefix));
- }
- if (ref != null) { // verify the counts
- Integer Cnt = ref.get(key);
- if (Cnt == null) { // first pass
- ref.put(key, res.size());
- } else {
- assertEquals(key + ", prefix: " + prefix, Cnt.intValue(), res.size());
- }
- }
- }
- if (estimate) {
- RamUsageEstimator rue = new RamUsageEstimator();
- long size = rue.estimateRamUsage(lookup);
- System.err.println(lookup.getClass().getSimpleName() + " - size=" + size);
- }
- if (bench != null) {
- bench.buildTime += buildTime;
- bench.lookupTime += elapsed;
- }
- }
-
- class Bench {
- long buildTime;
- long lookupTime;
- }
-
- @Test
- public void testBenchmark() throws Exception {
- // this benchmark is very time consuming
- boolean doTest = false;
- if (!doTest) {
- return;
- }
- Map<String,Integer> ref = new HashMap<String,Integer>();
- JaspellLookup jaspell = new JaspellLookup();
- TSTLookup tst = new TSTLookup();
-
- _benchmark(tst, ref, true, null);
- _benchmark(jaspell, ref, true, null);
- jaspell = null;
- tst = null;
- int count = 100;
- Bench b = runBenchmark(JaspellLookup.class, count);
- System.err.println(JaspellLookup.class.getSimpleName() + ": buildTime[ms]=" + (b.buildTime / count) +
- " lookupTime[ms]=" + (b.lookupTime / count / 1000000));
- b = runBenchmark(TSTLookup.class, count);
- System.err.println(TSTLookup.class.getSimpleName() + ": buildTime[ms]=" + (b.buildTime / count) +
- " lookupTime[ms]=" + (b.lookupTime / count / 1000000));
- }
-
- private Bench runBenchmark(Class<? extends Lookup> cls, int count) throws Exception {
- System.err.println("* Running " + count + " iterations for " + cls.getSimpleName() + " ...");
- System.err.println(" - warm-up 10 iterations...");
- for (int i = 0; i < 10; i++) {
- System.runFinalization();
- System.gc();
- Lookup lookup = cls.newInstance();
- _benchmark(lookup, null, false, null);
- lookup = null;
- }
- Bench b = new Bench();
- System.err.print(" - main iterations:"); System.err.flush();
- for (int i = 0; i < count; i++) {
- System.runFinalization();
- System.gc();
- Lookup lookup = cls.newInstance();
- _benchmark(lookup, null, false, b);
- lookup = null;
- if (i > 0 && (i % 10 == 0)) {
- System.err.print(" " + i);
- System.err.flush();
- }
- }
- System.err.println();
- return b;
+ assertQ(req("qt", requestUri, "q", "ac", SpellingParams.SPELLCHECK_COUNT, "2", SpellingParams.SPELLCHECK_ONLY_MORE_POPULAR, "true"),
+ "//lst[@name='spellcheck']/lst[@name='suggestions']/lst[@name='ac']/int[@name='numFound'][.='2']");
}
}
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java Mon May 2 13:50:57 2011
@@ -43,7 +43,7 @@ import org.junit.Test;
public class SignatureUpdateProcessorFactoryTest extends SolrTestCaseJ4 {
/** modified by tests as needed */
- private String processor = "dedupe";
+ private String chain = "dedupe";
@BeforeClass
public static void beforeClass() throws Exception {
@@ -56,7 +56,7 @@ public class SignatureUpdateProcessorFac
super.setUp();
clearIndex();
assertU(commit());
- processor = "dedupe"; // set the default that most tests expect
+ chain = "dedupe"; // set the default that most tests expect
}
void checkNumDocs(int n) {
@@ -203,7 +203,7 @@ public class SignatureUpdateProcessorFac
checkNumDocs(0);
- processor = "stored_sig";
+ chain = "stored_sig";
addDoc(adoc("id", "2a", "v_t", "Hello Dude man!", "name", "ali babi'"));
addDoc(adoc("id", "2b", "v_t", "Hello Dude man!", "name", "ali babi'"));
addDoc(commit());
@@ -232,7 +232,7 @@ public class SignatureUpdateProcessorFac
private void addDoc(String doc) throws Exception {
Map<String, String[]> params = new HashMap<String, String[]>();
MultiMapSolrParams mmparams = new MultiMapSolrParams(params);
- params.put(UpdateParams.UPDATE_PROCESSOR, new String[] { processor });
+ params.put(UpdateParams.UPDATE_CHAIN, new String[] { chain });
SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(),
(SolrParams) mmparams) {
};
Modified: lucene/dev/branches/docvalues/solr/src/webapp/web/admin/analysis.jsp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/webapp/web/admin/analysis.jsp?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/webapp/web/admin/analysis.jsp (original)
+++ lucene/dev/branches/docvalues/solr/src/webapp/web/admin/analysis.jsp Mon May 2 13:50:57 2011
@@ -222,6 +222,7 @@
public boolean incrementToken() throws IOException {
if (iter.hasNext()) {
+ clearAttributes();
AttributeSource token = iter.next();
Iterator<Class<? extends Attribute>> atts = token.getAttributeClassesIterator();
while (atts.hasNext()) // make sure all att impls in the token exist here
Modified: lucene/dev/branches/docvalues/solr/src/webapp/web/admin/form.jsp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/webapp/web/admin/form.jsp?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/webapp/web/admin/form.jsp (original)
+++ lucene/dev/branches/docvalues/solr/src/webapp/web/admin/form.jsp Mon May 2 13:50:57 2011
@@ -72,7 +72,7 @@
<strong>Query Type</strong>
</td>
<td>
- <input name="qt" type="text" value="standard">
+ <input name="qt" type="text">
</td>
</tr>
<tr>
@@ -80,7 +80,7 @@
<strong>Output Type</strong>
</td>
<td>
- <input name="wt" type="text" value="standard">
+ <input name="wt" type="text" value="">
</td>
</tr>
<tr>
@@ -121,7 +121,7 @@
<td>
</td>
<td>
- <input class="stdbutton" type="submit" value="search" onclick="if (queryForm.q.value.length==0) { alert('no empty queries, please'); return false; } else { queryForm.submit(); } ">
+ <input class="stdbutton" type="submit" value="search" onclick="if (queryForm.q.value.length==0) { alert('no empty queries, please'); return false; } else { queryForm.submit(); return false;} ">
</td>
</tr>
</table>
Modified: lucene/dev/branches/docvalues/solr/src/webapp/web/admin/index.jsp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/webapp/web/admin/index.jsp?rev=1098566&r1=1098565&r2=1098566&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/webapp/web/admin/index.jsp (original)
+++ lucene/dev/branches/docvalues/solr/src/webapp/web/admin/index.jsp Mon May 2 13:50:57 2011
@@ -125,7 +125,7 @@
<input name="rows" type="hidden" value="10">
<input name="indent" type="hidden" value="on">
<br><input class="stdbutton" type="submit" value="search"
- onclick="if (queryForm.q.value.length==0) { alert('no empty queries, please'); return false; } else { queryForm.submit(); } ">
+ onclick="if (queryForm.q.value.length==0) { alert('no empty queries, please'); return false; } else { queryForm.submit(); return false;} ">
</form>
</td>
</tr>