You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/05/27 22:16:00 UTC
svn commit: r1128442 - in /lucene/dev/branches/branch_3x/solr/src:
test-framework/org/apache/solr/SolrTestCaseJ4.java
webapp/src/org/apache/solr/servlet/DirectSolrConnection.java
Author: mikemccand
Date: Fri May 27 20:15:59 2011
New Revision: 1128442
URL: http://svn.apache.org/viewvc?rev=1128442&view=rev
Log:
SOLR-2543: backport test infra improvements from trunk
Modified:
lucene/dev/branches/branch_3x/solr/src/test-framework/org/apache/solr/SolrTestCaseJ4.java
lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java
Modified: lucene/dev/branches/branch_3x/solr/src/test-framework/org/apache/solr/SolrTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/test-framework/org/apache/solr/SolrTestCaseJ4.java?rev=1128442&r1=1128441&r2=1128442&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/test-framework/org/apache/solr/SolrTestCaseJ4.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/test-framework/org/apache/solr/SolrTestCaseJ4.java Fri May 27 20:15:59 2011
@@ -20,22 +20,26 @@ package org.apache.solr;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.noggit.CharArr;
+import org.apache.noggit.JSONUtil;
+import org.apache.noggit.ObjectBuilder;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.XML;
import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.JsonUpdateRequestHandler;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
-import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.search.DocIterator;
-import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.servlet.DirectSolrConnection;
import org.apache.solr.util.TestHarness;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -47,9 +51,7 @@ import javax.xml.xpath.XPathExpressionEx
import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
+import java.util.*;
/**
* A junit4 Solr test harness that extends LuceneTestCaseJ4.
@@ -77,7 +79,7 @@ public abstract class SolrTestCaseJ4 ext
@Override
public void tearDown() throws Exception {
- log.info("###Ending " + getName());
+ log.info("###Ending " + getName());
super.tearDown();
}
@@ -114,7 +116,7 @@ public abstract class SolrTestCaseJ4 ext
if (endNumOpens-numOpens != endNumCloses-numCloses) {
String msg = "ERROR: SolrIndexSearcher opens=" + (endNumOpens-numOpens) + " closes=" + (endNumCloses-numCloses);
log.error(msg);
- // fail(msg);
+ fail(msg);
}
}
@@ -127,7 +129,7 @@ public abstract class SolrTestCaseJ4 ext
public static void resetExceptionIgnores() {
SolrException.ignorePatterns = null;
- ignoreException("ignore_exception"); // always ignore "ignore_exception"
+ ignoreException("ignore_exception"); // always ignore "ignore_exception"
}
protected static String getClassName() {
@@ -221,7 +223,6 @@ public abstract class SolrTestCaseJ4 ext
if (factoryProp == null) {
System.setProperty("solr.directoryFactory","solr.RAMDirectoryFactory");
}
-
if (dataDir == null) {
createTempDir();
}
@@ -238,7 +239,7 @@ public abstract class SolrTestCaseJ4 ext
solrConfig,
getSchemaFile());
lrf = h.getRequestFactory
- ("standard",0,20,"version","2.2");
+ ("standard",0,20,CommonParams.VERSION,"2.2");
}
log.info("####initCore end");
}
@@ -282,7 +283,7 @@ public abstract class SolrTestCaseJ4 ext
if (factoryProp == null) {
System.clearProperty("solr.directoryFactory");
}
-
+
dataDir = null;
solrConfig = null;
h = null;
@@ -373,7 +374,7 @@ public abstract class SolrTestCaseJ4 ext
}
}
- /**
+ /**
* Validates a query matches some JSON test expressions using the default double delta tollerance.
* @see JSONTestUtil#DEFAULT_DELTA
* @see #assertJQ(SolrQueryRequest,double,String...)
@@ -381,14 +382,14 @@ public abstract class SolrTestCaseJ4 ext
public static void assertJQ(SolrQueryRequest req, String... tests) throws Exception {
assertJQ(req, JSONTestUtil.DEFAULT_DELTA, tests);
}
- /**
- * Validates a query matches some JSON test expressions and closes the
- * query. The text expression is of the form path:JSON. To facilitate
- * easy embedding in Java strings, the JSON can have double quotes
+ /**
+ * Validates a query matches some JSON test expressions and closes the
+ * query. The text expression is of the form path:JSON. To facilitate
+ * easy embedding in Java strings, the JSON can have double quotes
* replaced with single quotes.
* <p>
- * Please use this with care: this makes it easy to match complete
- * structures, but doing so can result in fragile tests if you are
+ * Please use this with care: this makes it easy to match complete
+ * structures, but doing so can result in fragile tests if you are
* matching more than what you want to test.
* </p>
* @param req Solr request to execute
@@ -434,7 +435,7 @@ public abstract class SolrTestCaseJ4 ext
}
} finally {
if (failed) {
- log.error("JSON query validation threw an exception." +
+ log.error("JSON query validation threw an exception." +
"\n expected =" + testJSON +
"\n response = " + response +
"\n request = " + req.getParamString()
@@ -446,7 +447,7 @@ public abstract class SolrTestCaseJ4 ext
// restore the params
if (params != null && params != req.getParams()) req.setParams(params);
}
- }
+ }
/** Makes sure a query throws a SolrException with the listed response code */
@@ -494,7 +495,7 @@ public abstract class SolrTestCaseJ4 ext
* @see #doc
*/
public static String adoc(String... fieldsAndValues) {
- Doc d = doc(fieldsAndValues);
+ XmlDoc d = doc(fieldsAndValues);
return add(d);
}
@@ -522,7 +523,7 @@ public abstract class SolrTestCaseJ4 ext
* @see #add
* @see #doc
*/
- public static String add(Doc doc, String... args) {
+ public static String add(XmlDoc doc, String... args) {
try {
StringWriter r = new StringWriter();
@@ -565,8 +566,8 @@ public abstract class SolrTestCaseJ4 ext
* @param fieldsAndValues 0th and Even numbered args are fields names, Odds are field values.
* @see TestHarness#makeSimpleDoc
*/
- public static Doc doc(String... fieldsAndValues) {
- Doc d = new Doc();
+ public static XmlDoc doc(String... fieldsAndValues) {
+ XmlDoc d = new XmlDoc();
d.xml = h.makeSimpleDoc(fieldsAndValues).toString();
return d;
}
@@ -615,7 +616,7 @@ public abstract class SolrTestCaseJ4 ext
}
/** Neccessary to make method signatures un-ambiguous */
- public static class Doc {
+ public static class XmlDoc {
public String xml;
@Override
public String toString() { return xml; }
@@ -632,11 +633,446 @@ public abstract class SolrTestCaseJ4 ext
}
return f.delete();
}
-
+
public void clearIndex() {
assertU(delQ("*:*"));
}
+ /** Send JSON update commands */
+ public static String updateJ(String json, SolrParams args) throws Exception {
+ SolrCore core = h.getCore();
+ DirectSolrConnection connection = new DirectSolrConnection(core);
+ SolrRequestHandler handler = core.getRequestHandler("/udate/json");
+ if (handler == null) {
+ handler = new JsonUpdateRequestHandler();
+ handler.init(null);
+ }
+ return connection.request(handler, args, json);
+ }
+
+
+ /////////////////////////////////////////////////////////////////////////////////////
+ //////////////////////////// random document / index creation ///////////////////////
+ /////////////////////////////////////////////////////////////////////////////////////
+
+ public abstract static class Vals {
+ public abstract Comparable get();
+ public String toJSON(Comparable val) {
+ return JSONUtil.toJSON(val);
+ }
+
+ protected int between(int min, int max) {
+ return min != max ? random.nextInt(max-min+1) + min : min;
+ }
+ }
+
+ public abstract static class IVals extends Vals {
+ public abstract int getInt();
+ }
+
+ public static class IRange extends IVals {
+ final int min;
+ final int max;
+ public IRange(int min, int max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ @Override
+ public int getInt() {
+ return between(min,max);
+ }
+
+ @Override
+ public Comparable get() {
+ return getInt();
+ }
+ }
+
+ public static class FVal extends Vals {
+ final float min;
+ final float max;
+ public FVal(float min, float max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ public float getFloat() {
+ if (min >= max) return min;
+ return min + random.nextFloat() * (max - min);
+ }
+
+ @Override
+ public Comparable get() {
+ return getFloat();
+ }
+ }
+
+ public static class SVal extends Vals {
+ char start;
+ char end;
+ int minLength;
+ int maxLength;
+
+ public SVal() {
+ this('a','z',1,10);
+ }
+
+ public SVal(char start, char end, int minLength, int maxLength) {
+ this.start = start;
+ this.end = end;
+ this.minLength = minLength;
+ this.maxLength = maxLength;
+ }
+
+ @Override
+ public Comparable get() {
+ char[] arr = new char[between(minLength,maxLength)];
+ for (int i=0; i<arr.length; i++) {
+ arr[i] = (char)between(start, end);
+ }
+ return new String(arr);
+ }
+ }
+
+ public static final IRange ZERO_ONE = new IRange(0,1);
+ public static final IRange ZERO_TWO = new IRange(0,2);
+ public static final IRange ONE_ONE = new IRange(1,1);
+
+ public static class Doc implements Comparable{
+ public Comparable id;
+ public List<Fld> fields;
+ public int order; // the order this document was added to the index
+
+
+ @Override
+ public String toString() {
+ return "Doc("+order+"):"+fields.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Doc)) return false;
+ Doc other = (Doc)o;
+ return this==other || id != null && id.equals(other.id);
+ }
+
+ public int compareTo(Object o) {
+ if (!(o instanceof Doc)) return this.getClass().hashCode() - o.getClass().hashCode();
+ Doc other = (Doc)o;
+ return this.id.compareTo(other.id);
+ }
+
+ public List<Comparable> getValues(String field) {
+ for (Fld fld : fields) {
+ if (fld.ftype.fname.equals(field)) return fld.vals;
+ }
+ return null;
+ }
+
+ public Comparable getFirstValue(String field) {
+ List<Comparable> vals = getValues(field);
+ return vals==null || vals.size()==0 ? null : vals.get(0);
+ }
+
+ public Map<String,Object> toObject(IndexSchema schema) {
+ Map<String,Object> result = new HashMap<String,Object>();
+ for (Fld fld : fields) {
+ SchemaField sf = schema.getField(fld.ftype.fname);
+ if (!sf.multiValued()) {
+ result.put(fld.ftype.fname, fld.vals.get(0));
+ } else {
+ result.put(fld.ftype.fname, fld.vals);
+ }
+ }
+ return result;
+ }
+
+ }
+
+ public static class Fld {
+ public FldType ftype;
+ public List<Comparable> vals;
+ @Override
+ public String toString() {
+ return ftype.fname + "=" + (vals.size()==1 ? vals.get(0).toString() : vals.toString());
+ }
+ }
+
+ class FldType {
+ public String fname;
+ public IRange numValues;
+ public Vals vals;
+
+ public FldType(String fname, Vals vals) {
+ this(fname, ZERO_ONE, vals);
+ }
+
+ public FldType(String fname, IRange numValues, Vals vals) {
+ this.fname = fname;
+ this.numValues = numValues;
+ this.vals = vals;
+ }
+
+ public Comparable createValue() {
+ return vals.get();
+ }
+
+ public List<Comparable> createValues() {
+ int nVals = numValues.getInt();
+ if (nVals <= 0) return null;
+ List<Comparable> vals = new ArrayList<Comparable>(nVals);
+ for (int i=0; i<nVals; i++)
+ vals.add(createValue());
+ return vals;
+ }
+
+ public Fld createField() {
+ List<Comparable> vals = createValues();
+ if (vals == null) return null;
+
+ Fld fld = new Fld();
+ fld.ftype = this;
+ fld.vals = vals;
+ return fld;
+ }
+
+ }
+
+ public Map<Comparable,Doc> indexDocs(List<FldType> descriptor, Map<Comparable,Doc> model, int nDocs) throws Exception {
+ if (model == null) {
+ model = new LinkedHashMap<Comparable,Doc>();
+ }
+
+ // commit an average of 10 times for large sets, or 10% of the time for small sets
+ int commitOneOutOf = Math.max(nDocs/10, 10);
+
+ for (int i=0; i<nDocs; i++) {
+ Doc doc = createDoc(descriptor);
+ // doc.order = order++;
+ updateJ(toJSON(doc), null);
+ model.put(doc.id, doc);
+
+ // commit 10% of the time
+ if (random.nextInt(commitOneOutOf)==0) {
+ assertU(commit());
+ }
+
+ // duplicate 10% of the docs
+ if (random.nextInt(10)==0) {
+ updateJ(toJSON(doc), null);
+ model.put(doc.id, doc);
+ }
+ }
+
+ // optimize 10% of the time
+ if (random.nextInt(10)==0) {
+ assertU(optimize());
+ } else {
+ assertU(commit());
+ }
+
+ // merging segments no longer selects just adjacent segments hence ids (doc.order) can be shuffled.
+ // we need to look at the index to determine the order.
+ String responseStr = h.query(req("q","*:*", "fl","id", "sort","_docid_ asc", "rows",Integer.toString(model.size()*2), "wt","json", "indent","true"));
+ Object response = ObjectBuilder.fromJSON(responseStr);
+
+ response = ((Map)response).get("response");
+ response = ((Map)response).get("docs");
+ List<Map> docList = (List<Map>)response;
+ int order = 0;
+ for (Map doc : docList) {
+ Object id = doc.get("id");
+ Doc modelDoc = model.get(id);
+ if (modelDoc == null) continue; // may be some docs in the index that aren't modeled
+ modelDoc.order = order++;
+ }
+
+ // make sure we updated the order of all docs in the model
+ assertEquals(order, model.size());
+
+ return model;
+ }
+
+ public static Doc createDoc(List<FldType> descriptor) {
+ Doc doc = new Doc();
+ doc.fields = new ArrayList<Fld>();
+ for (FldType ftype : descriptor) {
+ Fld fld = ftype.createField();
+ if (fld != null) {
+ doc.fields.add(fld);
+ if ("id".equals(ftype.fname))
+ doc.id = fld.vals.get(0);
+ }
+ }
+ return doc;
+ }
+
+ public static Comparator<Doc> createSort(IndexSchema schema, List<FldType> fieldTypes, String[] out) {
+ StringBuilder sortSpec = new StringBuilder();
+ int nSorts = random.nextInt(4);
+ List<Comparator<Doc>> comparators = new ArrayList<Comparator<Doc>>();
+ for (int i=0; i<nSorts; i++) {
+ if (i>0) sortSpec.append(',');
+
+ int which = random.nextInt(fieldTypes.size()+2);
+ boolean asc = random.nextBoolean();
+ if (which == fieldTypes.size()) {
+ // sort by score
+ sortSpec.append("score").append(asc ? " asc" : " desc");
+ comparators.add(createComparator("score", asc, false, false, false));
+ } else if (which == fieldTypes.size() + 1) {
+ // sort by docid
+ sortSpec.append("_docid_").append(asc ? " asc" : " desc");
+ comparators.add(createComparator("_docid_", asc, false, false, false));
+ } else {
+ String field = fieldTypes.get(which).fname;
+ sortSpec.append(field).append(asc ? " asc" : " desc");
+ SchemaField sf = schema.getField(field);
+ comparators.add(createComparator(field, asc, sf.sortMissingLast(), sf.sortMissingFirst(), !(sf.sortMissingLast()||sf.sortMissingFirst()) ));
+ }
+ }
+
+ out[0] = sortSpec.length() > 0 ? sortSpec.toString() : null;
+
+ if (comparators.size() == 0) {
+ // default sort is by score desc
+ comparators.add(createComparator("score", false, false, false, false));
+ }
+
+ return createComparator(comparators);
+ }
+
+ public static Comparator<Doc> createComparator(final String field, final boolean asc, final boolean sortMissingLast, final boolean sortMissingFirst, final boolean sortMissingAsZero) {
+ final int mul = asc ? 1 : -1;
+
+ if (field.equals("_docid_")) {
+ return new Comparator<Doc>() {
+ public int compare(Doc o1, Doc o2) {
+ return (o1.order - o2.order) * mul;
+ }
+ };
+ }
+
+ if (field.equals("score")) {
+ return createComparator("score_f", asc, sortMissingLast, sortMissingFirst, sortMissingAsZero);
+ }
+
+ return new Comparator<Doc>() {
+ private Comparable zeroVal(Comparable template) {
+ if (template == null) return null;
+ if (template instanceof String) return null; // fast-path for string
+ if (template instanceof Integer) return 0;
+ if (template instanceof Long) return (long)0;
+ if (template instanceof Float) return (float)0;
+ if (template instanceof Double) return (double)0;
+ if (template instanceof Short) return (short)0;
+ if (template instanceof Byte) return (byte)0;
+ if (template instanceof Character) return (char)0;
+ return null;
+ }
+
+ public int compare(Doc o1, Doc o2) {
+ Comparable v1 = o1.getFirstValue(field);
+ Comparable v2 = o2.getFirstValue(field);
+
+ v1 = v1 == null ? zeroVal(v2) : v1;
+ v2 = v2 == null ? zeroVal(v1) : v2;
+
+ int c = 0;
+ if (v1 == v2) {
+ c = 0;
+ } else if (v1 == null) {
+ if (sortMissingLast) c = mul;
+ else if (sortMissingFirst) c = -mul;
+ else c = -1;
+ } else if (v2 == null) {
+ if (sortMissingLast) c = -mul;
+ else if (sortMissingFirst) c = mul;
+ else c = 1;
+ } else {
+ c = v1.compareTo(v2);
+ }
+
+ c = c * mul;
+
+ return c;
+ }
+ };
+ }
+
+ public static Comparator<Doc> createComparator(final List<Comparator<Doc>> comparators) {
+ return new Comparator<Doc>() {
+ public int compare(Doc o1, Doc o2) {
+ int c = 0;
+ for (Comparator<Doc> comparator : comparators) {
+ c = comparator.compare(o1, o2);
+ if (c!=0) return c;
+ }
+ return o1.order - o2.order;
+ }
+ };
+ }
+
+
+ public static String toJSON(Doc doc) {
+ CharArr out = new CharArr();
+ try {
+ out.append("{\"add\":{\"doc\":{");
+ boolean firstField = true;
+ for (Fld fld : doc.fields) {
+ if (firstField) firstField=false;
+ else out.append(',');
+ JSONUtil.writeString(fld.ftype.fname, 0, fld.ftype.fname.length(), out);
+ out.append(':');
+ if (fld.vals.size() > 1) {
+ out.append('[');
+ }
+ boolean firstVal = true;
+ for (Comparable val : fld.vals) {
+ if (firstVal) firstVal=false;
+ else out.append(',');
+ out.append(JSONUtil.toJSON(val));
+ }
+ if (fld.vals.size() > 1) {
+ out.append(']');
+ }
+ }
+ out.append("}}}");
+ } catch (IOException e) {
+ // should never happen
+ }
+ return out.toString();
+ }
+
+ /** Return a Map from field value to a list of document ids */
+ Map<Comparable, List<Comparable>> invertField(Map<Comparable, Doc> model, String field) {
+ Map<Comparable, List<Comparable>> value_to_id = new HashMap<Comparable, List<Comparable>>();
+
+ // invert field
+ for (Comparable key : model.keySet()) {
+ Doc doc = model.get(key);
+ List<Comparable> vals = doc.getValues(field);
+ if (vals == null) continue;
+ for (Comparable val : vals) {
+ List<Comparable> ids = value_to_id.get(val);
+ if (ids == null) {
+ ids = new ArrayList<Comparable>(2);
+ value_to_id.put(val, ids);
+ }
+ ids.add(key);
+ }
+ }
+
+ return value_to_id;
+ }
+
+
/** Gets a resource from the context classloader as {@link File}. This method should only be used,
* if a real file is needed. To get a stream, code should prefer
* {@link Class#getResourceAsStream} using {@code this.getClass()}.
@@ -653,7 +1089,7 @@ public abstract class SolrTestCaseJ4 ext
throw new RuntimeException("Cannot find resource: " + name);
}
}
-
+
public static String TEST_HOME() {
return getFile("solr/conf").getParent();
}
Modified: lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java?rev=1128442&r1=1128441&r2=1128442&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java Fri May 27 20:15:59 2011
@@ -155,16 +155,25 @@ public class DirectSolrConnection
}
}
}
+
if( handler == null ) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "unknown handler: "+path );
}
-
+
+ return request(handler, params, body);
+ }
+
+ public String request(SolrRequestHandler handler, SolrParams params, String body ) throws Exception {
+
+ if (params == null)
+ params = new MapSolrParams( new HashMap<String, String>() );
+
// Make a stream for the 'body' content
List<ContentStream> streams = new ArrayList<ContentStream>( 1 );
if( body != null && body.length() > 0 ) {
streams.add( new ContentStreamBase.StringStream( body ) );
}
-
+
SolrQueryRequest req = null;
try {
req = parser.buildRequestFrom( core, params, streams );
@@ -173,7 +182,7 @@ public class DirectSolrConnection
if( rsp.getException() != null ) {
throw rsp.getException();
}
-
+
// Now write it out
QueryResponseWriter responseWriter = core.getQueryResponseWriter(req);
StringWriter out = new StringWriter();