You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/11/17 16:43:10 UTC
svn commit: r1036080 [4/4] - in /lucene/dev/branches/docvalues: ./ lucene/
lucene/contrib/ lucene/contrib/highlighter/src/test/
lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/
lucene/contrib/misc/src/java/org/apache/lucene/in...
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/Grouping.java?rev=1036080&r1=1036079&r2=1036080&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/Grouping.java Wed Nov 17 15:43:06 2010
@@ -19,10 +19,14 @@ package org.apache.solr.search;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.*;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.schema.StrFieldSource;
import org.apache.solr.search.function.DocValues;
+import org.apache.solr.search.function.StringIndexDocValues;
import org.apache.solr.search.function.ValueSource;
+import org.apache.solr.util.SentinelIntSet;
import java.io.IOException;
import java.util.*;
@@ -141,6 +145,9 @@ public class Grouping {
Collector createCollector() throws IOException {
maxGroupToFind = getMax(offset, numGroups, maxDoc);
+ // if we aren't going to return any groups, disregard the offset
+ if (numGroups == 0) maxGroupToFind = 0;
+
if (compareSorts(sort, groupSort)) {
collector = new TopGroupCollector(groupBy, context, normalizeSort(sort), maxGroupToFind);
} else {
@@ -151,10 +158,15 @@ public class Grouping {
@Override
Collector createNextCollector() throws IOException {
+ if (numGroups == 0) return null;
+
int docsToCollect = getMax(groupOffset, docsPerGroup, maxDoc);
- if (docsToCollect < 0 || docsToCollect > maxDoc) docsToCollect = maxDoc;
- collector2 = new Phase2GroupCollector(collector, groupBy, context, groupSort, docsToCollect, needScores, offset);
+ if (false && groupBy instanceof StrFieldSource) {
+ collector2 = new Phase2StringGroupCollector(collector, groupBy, context, groupSort, docsToCollect, needScores, offset);
+ } else {
+ collector2 = new Phase2GroupCollector(collector, groupBy, context, groupSort, docsToCollect, needScores, offset);
+ }
return collector2;
}
@@ -162,11 +174,16 @@ public class Grouping {
void finish() throws IOException {
NamedList groupResult = commonResponse();
- if (collector.orderedGroups == null) collector.buildSet();
-
List groupList = new ArrayList();
groupResult.add("groups", groupList); // grouped={ key={ groups=[
+ // handle case of rows=0
+ if (numGroups == 0) return;
+
+ if (collector.orderedGroups == null) collector.buildSet();
+
+
+
int skipCount = offset;
for (SearchGroup group : collector.orderedGroups) {
if (skipCount > 0) {
@@ -411,7 +428,7 @@ class TopGroupCollector extends GroupCol
public TopGroupCollector(ValueSource groupByVS, Map vsContext, Sort sort, int nGroups) throws IOException {
this.vs = groupByVS;
this.context = vsContext;
- this.nGroups = nGroups;
+ this.nGroups = nGroups = Math.max(1,nGroups); // we need a minimum of 1 for this collector
SortField[] sortFields = sort.getSort();
this.comparators = new FieldComparator[sortFields.length];
@@ -538,16 +555,27 @@ class TopGroupCollector extends GroupCol
// remove before updating the group since lookup is done via comparators
// TODO: optimize this
- if (orderedGroups != null)
+
+ SearchGroup prevLast = null;
+ if (orderedGroups != null) {
+ prevLast = orderedGroups.last();
orderedGroups.remove(group);
+ }
group.topDoc = docBase + doc;
// group.topDocScore = scorer.score();
int tmp = spareSlot; spareSlot = group.comparatorSlot; group.comparatorSlot=tmp; // swap slots
// re-add the changed group
- if (orderedGroups != null)
+ if (orderedGroups != null) {
orderedGroups.add(group);
+ SearchGroup newLast = orderedGroups.last();
+ // if we changed the value of the last group, or changed which group was last, then update bottom
+ if (group == newLast || prevLast != newLast) {
+ for (FieldComparator fc : comparators)
+ fc.setBottom(newLast.comparatorSlot);
+ }
+ }
}
void buildSet() {
@@ -662,16 +690,16 @@ class TopGroupSortCollector extends TopG
buildSet();
}
- SearchGroup leastSignificantGroup = orderedGroups.last();
+ // see if this new group would be competitive if this doc was the top doc
for (int i = 0;; i++) {
- final int c = leastSignificantGroup.sortGroupReversed[i] * leastSignificantGroup.sortGroupComparators[i].compareBottom(doc);
+ final int c = reversed[i] * comparators[i].compareBottom(doc);
if (c < 0) {
- // Definitely not competitive.
+ // Definitely not competitive. So don't even bother to continue
return;
} else if (c > 0) {
// Definitely competitive.
break;
- } else if (i == leastSignificantGroup.sortGroupComparators.length - 1) {
+ } else if (i == comparators.length - 1) {
// Here c=0. If we're at the last comparator, this doc is not
// competitive, since docs are visited in doc Id order, which means
// this doc cannot compete with any other document in the queue.
@@ -698,10 +726,9 @@ class TopGroupSortCollector extends TopG
groupMap.put(smallest.groupValue, smallest);
orderedGroups.add(smallest);
+ int lastSlot = orderedGroups.last().comparatorSlot;
for (FieldComparator fc : comparators)
- fc.setBottom(orderedGroups.last().comparatorSlot);
- for (FieldComparator fc : smallest.sortGroupComparators)
- fc.setBottom(0);
+ fc.setBottom(lastSlot);
return;
}
@@ -839,3 +866,52 @@ class SearchGroupDocs {
TopDocsCollector collector;
}
+
+
+class Phase2StringGroupCollector extends Phase2GroupCollector {
+ FieldCache.DocTermsIndex index;
+ SentinelIntSet ordSet;
+ SearchGroupDocs[] groups;
+ BytesRef spare;
+
+ public Phase2StringGroupCollector(TopGroupCollector topGroups, ValueSource groupByVS, Map vsContext, Sort sort, int docsPerGroup, boolean getScores, int offset) throws IOException {
+ super(topGroups, groupByVS, vsContext,sort,docsPerGroup,getScores,offset);
+ ordSet = new SentinelIntSet(groupMap.size(), -1);
+ groups = new SearchGroupDocs[ordSet.keys.length];
+ }
+
+ @Override
+ public void setScorer(Scorer scorer) throws IOException {
+ this.scorer = scorer;
+ for (SearchGroupDocs group : groupMap.values())
+ group.collector.setScorer(scorer);
+ }
+
+ @Override
+ public void collect(int doc) throws IOException {
+ int slot = ordSet.find(index.getOrd(doc));
+ if (slot >= 0) {
+ groups[slot].collector.collect(doc);
+ }
+ }
+
+ @Override
+ public void setNextReader(IndexReader reader, int docBase) throws IOException {
+ super.setNextReader(reader, docBase);
+ index = ((StringIndexDocValues)docValues).getDocTermsIndex();
+
+ ordSet.clear();
+ for (SearchGroupDocs group : groupMap.values()) {
+ int ord = index.binarySearchLookup(((MutableValueStr)group.groupValue).value, spare);
+ if (ord > 0) {
+ int slot = ordSet.put(ord);
+ groups[slot] = group;
+ }
+ }
+ }
+
+ @Override
+ public boolean acceptsDocsOutOfOrder() {
+ return false;
+ }
+}
\ No newline at end of file
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java?rev=1036080&r1=1036079&r2=1036080&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java Wed Nov 17 15:43:06 2010
@@ -189,12 +189,11 @@ class TermOrdValComparator_SML extends F
bottomOrd = index;
// exact value match
bottomSameReader = true;
+ readerGen[bottomSlot] = currentReaderGen;
+ ords[bottomSlot] = bottomOrd;
}
}
}
- if (bottomSameReader) {
- readerGen[bottomSlot] = currentReaderGen;
- }
}
@Override
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/function/StringIndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/function/StringIndexDocValues.java?rev=1036080&r1=1036079&r2=1036080&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/function/StringIndexDocValues.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/function/StringIndexDocValues.java Wed Nov 17 15:43:06 2010
@@ -41,6 +41,10 @@ public abstract class StringIndexDocValu
}
this.vs = vs;
}
+
+ public FieldCache.DocTermsIndex getDocTermsIndex() {
+ return termsIndex;
+ }
protected abstract String toTerm(String readableValue);
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/JSONTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/JSONTestUtil.java?rev=1036080&r1=1036079&r2=1036080&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/JSONTestUtil.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/JSONTestUtil.java Wed Nov 17 15:43:06 2010
@@ -121,9 +121,13 @@ class CollectionTester {
}
boolean match() {
- if (expected == null && val == null) {
+ if (expected == val) {
return true;
}
+ if (expected == null || val == null) {
+ setErr("mismatch: '" + expected + "'!='" + val + "'");
+ return false;
+ }
if (expected instanceof List) {
return matchList();
}
@@ -133,8 +137,20 @@ class CollectionTester {
// generic fallback
if (!expected.equals(val)) {
- setErr("mismatch: '" + expected + "'!='" + val + "'");
- return false;
+
+ // make an exception for some numerics
+ if (expected instanceof Integer && val instanceof Long || expected instanceof Long && val instanceof Integer
+ && ((Number)expected).longValue() == ((Number)val).longValue())
+ {
+ // OK
+ } else if (expected instanceof Float && val instanceof Double || expected instanceof Double && val instanceof Float
+ && ((Number)expected).doubleValue() == ((Number)val).doubleValue())
+ {
+ // OK
+ } else {
+ setErr("mismatch: '" + expected + "'!='" + val + "'");
+ return false;
+ }
}
// setErr("unknown expected type " + expected.getClass().getName());
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/SolrTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/SolrTestCaseJ4.java?rev=1036080&r1=1036079&r2=1036080&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/SolrTestCaseJ4.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/SolrTestCaseJ4.java Wed Nov 17 15:43:06 2010
@@ -20,6 +20,10 @@ package org.apache.solr;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.apache.noggit.CharArr;
+import org.apache.noggit.JSONUtil;
+import org.apache.noggit.JSONWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
@@ -27,9 +31,20 @@ import org.apache.solr.common.params.Mod
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.XML;
import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.JsonUpdateRequestHandler;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.request.BinaryQueryResponseWriter;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.servlet.DirectSolrConnection;
+import org.apache.solr.servlet.SolrRequestParsers;
import org.apache.solr.util.TestHarness;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -40,10 +55,9 @@ import org.xml.sax.SAXException;
import javax.xml.xpath.XPathExpressionException;
import java.io.File;
import java.io.IOException;
+import java.io.PrintWriter;
import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
+import java.util.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -476,7 +490,7 @@ public abstract class SolrTestCaseJ4 ext
* @see #doc
*/
public static String adoc(String... fieldsAndValues) {
- Doc d = doc(fieldsAndValues);
+ XmlDoc d = doc(fieldsAndValues);
return add(d);
}
@@ -504,7 +518,7 @@ public abstract class SolrTestCaseJ4 ext
* @see #add
* @see #doc
*/
- public static String add(Doc doc, String... args) {
+ public static String add(XmlDoc doc, String... args) {
try {
StringWriter r = new StringWriter();
@@ -547,8 +561,8 @@ public abstract class SolrTestCaseJ4 ext
* @param fieldsAndValues 0th and Even numbered args are fields names, Odds are field values.
* @see TestHarness#makeSimpleDoc
*/
- public static Doc doc(String... fieldsAndValues) {
- Doc d = new Doc();
+ public static XmlDoc doc(String... fieldsAndValues) {
+ XmlDoc d = new XmlDoc();
d.xml = h.makeSimpleDoc(fieldsAndValues).toString();
return d;
}
@@ -597,7 +611,7 @@ public abstract class SolrTestCaseJ4 ext
}
/** Neccessary to make method signatures un-ambiguous */
- public static class Doc {
+ public static class XmlDoc {
public String xml;
public String toString() { return xml; }
}
@@ -617,4 +631,394 @@ public abstract class SolrTestCaseJ4 ext
public void clearIndex() {
assertU(delQ("*:*"));
}
+
+ /** Send JSON update commands */
+ public static String updateJ(String json, SolrParams args) throws Exception {
+ SolrCore core = h.getCore();
+ DirectSolrConnection connection = new DirectSolrConnection(core);
+ SolrRequestHandler handler = core.getRequestHandler("/udate/json");
+ if (handler == null) {
+ handler = new JsonUpdateRequestHandler();
+ handler.init(null);
+ }
+ return connection.request(handler, args, json);
+ }
+
+
+ /////////////////////////////////////////////////////////////////////////////////////
+ //////////////////////////// random document / index creation ///////////////////////
+ /////////////////////////////////////////////////////////////////////////////////////
+
+ public abstract static class Vals {
+ public abstract Comparable get();
+ public String toJSON(Comparable val) {
+ return JSONUtil.toJSON(val);
+ }
+
+ protected int between(int min, int max) {
+ return min != max ? random.nextInt(max-min+1) + min : min;
+ }
+ }
+
+ public abstract static class IVals extends Vals {
+ public abstract int getInt();
+ }
+
+ public static class IRange extends IVals {
+ final int min;
+ final int max;
+ public IRange(int min, int max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ @Override
+ public int getInt() {
+ return between(min,max);
+ }
+
+ @Override
+ public Comparable get() {
+ return getInt();
+ }
+ }
+
+ public static class FVal extends Vals {
+ final float min;
+ final float max;
+ public FVal(float min, float max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ public float getFloat() {
+ if (min >= max) return min;
+ return min + random.nextFloat() * (max - min);
+ }
+
+ @Override
+ public Comparable get() {
+ return getFloat();
+ }
+ }
+
+ public static class SVal extends Vals {
+ char start;
+ char end;
+ int minLength;
+ int maxLength;
+
+ public SVal() {
+ this('a','z',1,10);
+ }
+
+ public SVal(char start, char end, int minLength, int maxLength) {
+ this.start = start;
+ this.end = end;
+ this.minLength = minLength;
+ this.maxLength = maxLength;
+ }
+
+ @Override
+ public Comparable get() {
+ char[] arr = new char[between(minLength,maxLength)];
+ for (int i=0; i<arr.length; i++) {
+ arr[i] = (char)between(start, end);
+ }
+ return new String(arr);
+ }
+ }
+
+ public static final IRange ZERO_ONE = new IRange(0,1);
+ public static final IRange ONE_ONE = new IRange(1,1);
+
+ public static class Doc implements Comparable{
+ public Comparable id;
+ public List<Fld> fields;
+ public int order; // the order this document was added to the index
+
+
+ public String toString() {
+ return "Doc("+order+"):"+fields.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Doc)) return false;
+ Doc other = (Doc)o;
+ return this==other || id != null && id.equals(other.id);
+ }
+
+ @Override
+ public int compareTo(Object o) {
+ if (!(o instanceof Doc)) return this.getClass().hashCode() - o.getClass().hashCode();
+ Doc other = (Doc)o;
+ return this.id.compareTo(other.id);
+ }
+
+ public List<Comparable> getValues(String field) {
+ for (Fld fld : fields) {
+ if (fld.ftype.fname.equals(field)) return fld.vals;
+ }
+ return null;
+ }
+
+ public Comparable getFirstValue(String field) {
+ List<Comparable> vals = getValues(field);
+ return vals==null || vals.size()==0 ? null : vals.get(0);
+ }
+
+ public Map<String,Object> toObject(IndexSchema schema) {
+ Map<String,Object> result = new HashMap<String,Object>();
+ for (Fld fld : fields) {
+ SchemaField sf = schema.getField(fld.ftype.fname);
+ if (!sf.multiValued()) {
+ result.put(fld.ftype.fname, fld.vals.get(0));
+ } else {
+ result.put(fld.ftype.fname, fld.vals);
+ }
+ }
+ return result;
+ }
+
+ }
+
+ public static class Fld {
+ public FldType ftype;
+ public List<Comparable> vals;
+ public String toString() {
+ return ftype.fname + "=" + (vals.size()==1 ? vals.get(0).toString() : vals.toString());
+ }
+ }
+
+ class FldType {
+ public String fname;
+ public IRange numValues;
+ public Vals vals;
+
+ public FldType(String fname, Vals vals) {
+ this(fname, ZERO_ONE, vals);
+ }
+
+ public FldType(String fname, IRange numValues, Vals vals) {
+ this.fname = fname;
+ this.numValues = numValues;
+ this.vals = vals;
+ }
+
+ public Comparable createValue() {
+ return vals.get();
+ }
+
+ public List<Comparable> createValues() {
+ int nVals = numValues.getInt();
+ if (nVals <= 0) return null;
+ List<Comparable> vals = new ArrayList<Comparable>(nVals);
+ for (int i=0; i<nVals; i++)
+ vals.add(createValue());
+ return vals;
+ }
+
+ public Fld createField() {
+ List<Comparable> vals = createValues();
+ if (vals == null) return null;
+
+ Fld fld = new Fld();
+ fld.ftype = this;
+ fld.vals = vals;
+ return fld;
+ }
+
+ }
+
+
+ public Map<Comparable,Doc> indexDocs(List<FldType> descriptor, Map<Comparable,Doc> model, int nDocs) throws Exception {
+ if (model == null) {
+ model = new LinkedHashMap<Comparable,Doc>();
+ }
+
+ // commit an average of 10 times for large sets, or 10% of the time for small sets
+ int commitOneOutOf = Math.max(nDocs/10, 10);
+
+
+ // find the max order (docid) and start from there
+ int order = -1;
+ for (Doc doc : model.values()) {
+ order = Math.max(order, doc.order);
+ }
+ order++;
+
+ for (int i=0; i<nDocs; i++) {
+ Doc doc = createDoc(descriptor);
+ doc.order = order++;
+ updateJ(toJSON(doc), null);
+ model.put(doc.id, doc);
+
+ // commit 10% of the time
+ if (random.nextInt(commitOneOutOf)==0) {
+ assertU(commit());
+ }
+
+ // duplicate 10% of the docs
+ if (random.nextInt(10)==0) {
+ updateJ(toJSON(doc), null);
+ model.put(doc.id, doc);
+ }
+ }
+
+ // optimize 10% of the time
+ if (random.nextInt(10)==0) {
+ assertU(optimize());
+ } else {
+ assertU(commit());
+ }
+
+ return model;
+ }
+
+ public static Doc createDoc(List<FldType> descriptor) {
+ Doc doc = new Doc();
+ doc.fields = new ArrayList<Fld>();
+ for (FldType ftype : descriptor) {
+ Fld fld = ftype.createField();
+ if (fld != null) {
+ doc.fields.add(fld);
+ if ("id".equals(ftype.fname))
+ doc.id = fld.vals.get(0);
+ }
+ }
+ return doc;
+ }
+
+ public static Comparator<Doc> createSort(IndexSchema schema, List<FldType> fieldTypes, String[] out) {
+ StringBuilder sortSpec = new StringBuilder();
+ int nSorts = random.nextInt(4);
+ List<Comparator<Doc>> comparators = new ArrayList<Comparator<Doc>>();
+ for (int i=0; i<nSorts; i++) {
+ if (i>0) sortSpec.append(',');
+
+ int which = random.nextInt(fieldTypes.size()+2);
+ boolean asc = random.nextBoolean();
+ if (which == fieldTypes.size()) {
+ // sort by score
+ sortSpec.append("score").append(asc ? " asc" : " desc");
+ comparators.add(createComparator("score", asc, false, false));
+ } else if (which == fieldTypes.size() + 1) {
+ // sort by docid
+ sortSpec.append("_docid_").append(asc ? " asc" : " desc");
+ comparators.add(createComparator("_docid_", asc, false, false));
+ } else {
+ String field = fieldTypes.get(which).fname;
+ sortSpec.append(field).append(asc ? " asc" : " desc");
+ SchemaField sf = schema.getField(field);
+ comparators.add(createComparator(field, asc, sf.sortMissingLast(), sf.sortMissingFirst()));
+ }
+ }
+
+ out[0] = sortSpec.length() > 0 ? sortSpec.toString() : null;
+
+ if (comparators.size() == 0) {
+ // default sort is by score desc
+ comparators.add(createComparator("score", false, false, false));
+ }
+
+ return createComparator(comparators);
+ }
+
+ public static Comparator<Doc> createComparator(final String field, final boolean asc, final boolean sortMissingLast, final boolean sortMissingFirst) {
+ final int mul = asc ? 1 : -1;
+
+ if (field.equals("_docid_")) {
+ return new Comparator<Doc>() {
+ @Override
+ public int compare(Doc o1, Doc o2) {
+ return (o1.order - o2.order) * mul;
+ }
+ };
+ }
+
+ if (field.equals("score")) {
+ return createComparator("score_f", asc, sortMissingLast, sortMissingFirst);
+ }
+
+ return new Comparator<Doc>() {
+ @Override
+ public int compare(Doc o1, Doc o2) {
+ Comparable v1 = o1.getFirstValue(field);
+ Comparable v2 = o2.getFirstValue(field);
+
+ int c = 0;
+ if (v1 == v2) {
+ c = 0;
+ } else if (v1 == null) {
+ if (sortMissingLast) c = mul;
+ else if (sortMissingFirst) c = -mul;
+ else c = -1;
+ } else if (v2 == null) {
+ if (sortMissingLast) c = -mul;
+ else if (sortMissingFirst) c = mul;
+ else c = 1;
+ } else {
+ c = v1.compareTo(v2);
+ }
+
+ c = c * mul;
+
+ return c;
+ }
+ };
+ }
+
+ public static Comparator<Doc> createComparator(final List<Comparator<Doc>> comparators) {
+ return new Comparator<Doc>() {
+ @Override
+ public int compare(Doc o1, Doc o2) {
+ int c = 0;
+ for (Comparator<Doc> comparator : comparators) {
+ c = comparator.compare(o1, o2);
+ if (c!=0) return c;
+ }
+ return o1.order - o2.order;
+ }
+ };
+ }
+
+
+ public static String toJSON(Doc doc) {
+ CharArr out = new CharArr();
+ try {
+ out.append("{\"add\":{\"doc\":{");
+ boolean firstField = true;
+ for (Fld fld : doc.fields) {
+ if (firstField) firstField=false;
+ else out.append(',');
+ JSONUtil.writeString(fld.ftype.fname, 0, fld.ftype.fname.length(), out);
+ out.append(':');
+ if (fld.vals.size() > 1) {
+ out.append('[');
+ }
+ boolean firstVal = true;
+ for (Comparable val : fld.vals) {
+ if (firstVal) firstVal=false;
+ else out.append(',');
+ out.append(JSONUtil.toJSON(val));
+ }
+ if (fld.vals.size() > 1) {
+ out.append(']');
+ }
+ }
+ out.append("}}}");
+ } catch (IOException e) {
+ // should never happen
+ }
+ return out.toString();
+ }
+
+
+
}
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/TestGroupingSearch.java?rev=1036080&r1=1036079&r2=1036080&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/TestGroupingSearch.java Wed Nov 17 15:43:06 2010
@@ -18,10 +18,17 @@
package org.apache.solr;
import org.apache.lucene.search.FieldCache;
+import org.apache.noggit.JSONUtil;
+import org.apache.noggit.ObjectBuilder;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.util.*;
+
public class TestGroupingSearch extends SolrTestCaseJ4 {
@BeforeClass
@@ -301,4 +308,184 @@ public class TestGroupingSearch extends
+ @Test
+ public void testRandomGrouping() throws Exception {
+ /**
+ updateJ("{\"add\":{\"doc\":{\"id\":\"77\"}}}", params("commit","true"));
+ assertJQ(req("q","id:77"), "/response/numFound==1");
+
+ Doc doc = createDocObj(types);
+ updateJ(toJSON(doc), params("commit","true"));
+
+ assertJQ(req("q","id:"+doc.id), "/response/numFound==1");
+ **/
+
+ int indexIter=0; // make >0 to enable test
+ int queryIter=1000;
+
+ while (--indexIter >= 0) {
+
+ List<FldType> types = new ArrayList<FldType>();
+ types.add(new FldType("id",ONE_ONE, new SVal('A','Z',2,2)));
+ types.add(new FldType("score_f",ONE_ONE, new FVal(1,100))); // field used to score
+ types.add(new FldType("foo_i",ONE_ONE, new IRange(0,10)));
+ types.add(new FldType("foo_s",ONE_ONE, new SVal('a','z',1,2)));
+
+ Map<Comparable, Doc> model = indexDocs(types, null, 2);
+ System.out.println("############### model=" + model);
+
+
+ for (int qiter=0; qiter<queryIter; qiter++) {
+ String groupField = types.get(random.nextInt(types.size())).fname;
+
+ Map<Comparable, Grp> groups = groupBy(model.values(), groupField);
+ int rows = random.nextInt(11)-1;
+ int start = random.nextInt(5)==0 ? random.nextInt(model.size()+2) : random.nextInt(5); // pick a small start normally for better coverage
+ int group_limit = random.nextInt(11)-1;
+group_limit = random.nextInt(10)+1;
+ int group_offset = random.nextInt(10)==0 ? random.nextInt(model.size()+2) : random.nextInt(2); // pick a small start normally for better coverage
+
+ // sort each group
+ String[] stringSortA = new String[1];
+ Comparator<Doc> groupComparator = createSort(h.getCore().getSchema(), types, stringSortA);
+ String groupSortStr = stringSortA[0];
+
+ // Test specific sort
+ /***
+ groupComparator = createComparator("_docid_", false, false, false);
+ stringSort = "_docid_ desc";
+ ***/
+
+ // first sort the docs in each group
+ for (Grp grp : groups.values()) {
+ Collections.sort(grp.docs, groupComparator);
+ }
+
+ // now sort the groups by the first doc in that group
+ Comparator<Doc> sortComparator = random.nextBoolean() ? groupComparator : createSort(h.getCore().getSchema(), types, stringSortA);
+ String sortStr = stringSortA[0];
+
+ List<Grp> sortedGroups = new ArrayList(groups.values());
+ Collections.sort(sortedGroups, createFirstDocComparator(sortComparator));
+
+ Object modelResponse = buildGroupedResult(h.getCore().getSchema(), sortedGroups, start, rows, group_offset, group_limit);
+
+ // TODO: create a random filter too
+
+ SolrQueryRequest req = req("group","true","wt","json","indent","true", "q","{!func}score_f", "group.field",groupField
+ ,sortStr==null ? "nosort":"sort", sortStr ==null ? "": sortStr
+ ,(groupSortStr==null || groupSortStr==sortStr) ? "nosort":"group.sort", groupSortStr==null ? "": groupSortStr
+ ,"rows",""+rows, "start",""+start, "group.offset",""+group_offset, "group.limit",""+group_limit
+ );
+
+ String strResponse = h.query(req);
+
+ Object realResponse = ObjectBuilder.fromJSON(strResponse);
+ String err = JSONTestUtil.matchObj("/grouped/"+groupField, realResponse, modelResponse);
+ if (err != null) {
+ log.error("GROUPING MISMATCH: " + err
+ + "\n\tresult="+strResponse
+ + "\n\texpected="+ JSONUtil.toJSON(modelResponse)
+ + "\n\tsorted_model="+ sortedGroups
+ );
+
+ fail(err);
+ }
+ } // end query iter
+ } // end index iter
+
+ }
+
+ public static Object buildGroupedResult(IndexSchema schema, List<Grp> sortedGroups, int start, int rows, int group_offset, int group_limit) {
+ Map<String,Object> result = new LinkedHashMap<String,Object>();
+
+ long matches = 0;
+ for (Grp grp : sortedGroups) {
+ matches += grp.docs.size();
+ }
+ result.put("matches", matches);
+ List groupList = new ArrayList();
+ result.put("groups", groupList);
+
+ for (int i=start; i<sortedGroups.size(); i++) {
+ if (rows != -1 && groupList.size() >= rows) break; // directly test rather than calculating, so we can catch any calc errors in the real code
+ Map<String,Object> group = new LinkedHashMap<String,Object>();
+ groupList.add(group);
+
+ Grp grp = sortedGroups.get(i);
+ group.put("groupValue", grp.groupValue);
+
+ Map<String,Object> resultSet = new LinkedHashMap<String,Object>();
+ group.put("doclist", resultSet);
+ resultSet.put("numFound", grp.docs.size());
+ resultSet.put("start", start);
+
+ List docs = new ArrayList();
+ resultSet.put("docs", docs);
+ for (int j=group_offset; j<grp.docs.size(); j++) {
+ if (group_offset != -1 && docs.size() >= group_limit) break;
+ docs.add( grp.docs.get(j).toObject(schema) );
+ }
+ }
+
+ return result;
+ }
+
+
+ public static Comparator<Grp> createFirstDocComparator(final Comparator<Doc> docComparator) {
+ return new Comparator<Grp>() {
+ @Override
+ public int compare(Grp o1, Grp o2) {
+ // all groups should have at least one doc
+ Doc d1 = o1.docs.get(0);
+ Doc d2 = o2.docs.get(0);
+ return docComparator.compare(d1, d2);
+ }
+ };
+ }
+
+
+
+ public static Map<Comparable, Grp> groupBy(Collection<Doc> docs, String field) {
+ Map<Comparable, Grp> groups = new HashMap<Comparable, Grp>();
+ for (Doc doc : docs) {
+ List<Comparable> vals = doc.getValues(field);
+ if (vals == null) {
+ Grp grp = groups.get(null);
+ if (grp == null) {
+ grp = new Grp();
+ grp.groupValue = null;
+ grp.docs = new ArrayList<Doc>();
+ groups.put(null, grp);
+ }
+ grp.docs.add(doc);
+ } else {
+ for (Comparable val : vals) {
+
+ Grp grp = groups.get(val);
+ if (grp == null) {
+ grp = new Grp();
+ grp.groupValue = val;
+ grp.docs = new ArrayList<Doc>();
+ groups.put(grp.groupValue, grp);
+ }
+ grp.docs.add(doc);
+ }
+ }
+ }
+ return groups;
+ }
+
+
+ public static class Grp {
+ public Comparable groupValue;
+ public List<SolrTestCaseJ4.Doc> docs;
+
+ @Override
+ public String toString() {
+ return "{groupValue="+groupValue+",docs="+docs+"}";
+ }
+ }
}
+
+
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java?rev=1036080&r1=1036079&r2=1036080&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java Wed Nov 17 15:43:06 2010
@@ -161,14 +161,13 @@ public class CloudStateUpdateTest extend
// slight pause - TODO: takes an oddly long amount of time to schedule tasks
// with almost no delay ...
- Thread.sleep(5000);
CloudState cloudState2 = null;
Map<String,Slice> slices = null;
- for (int i = 30; i > 0; i--) {
+ for (int i = 75; i > 0; i--) {
cloudState2 = zkController2.getCloudState();
slices = cloudState2.getSlices("testcore");
- if (slices.containsKey(host + ":1661_solr_testcore")) {
+ if (slices != null && slices.containsKey(host + ":1661_solr_testcore")) {
break;
}
Thread.sleep(500);
@@ -198,12 +197,12 @@ public class CloudStateUpdateTest extend
container3.shutdown();
- // slight pause for watch to trigger
- for(int i = 0; i < 30; i++) {
+ // slight pause (15s timeout) for watch to trigger
+ for(int i = 0; i < (5 * 15); i++) {
if(zkController2.getCloudState().getLiveNodes().size() == 2) {
break;
}
- Thread.sleep(50);
+ Thread.sleep(200);
}
assertEquals(2, zkController2.getCloudState().getLiveNodes().size());
@@ -217,7 +216,7 @@ public class CloudStateUpdateTest extend
container2 = init2.initialize();
// pause for watch to trigger
- for(int i = 0; i < 100; i++) {
+ for(int i = 0; i < 200; i++) {
if (container1.getZkController().getCloudState().liveNodesContain(
container2.getZkController().getNodeName())) {
break;
Modified: lucene/dev/branches/docvalues/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java?rev=1036080&r1=1036079&r2=1036080&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java (original)
+++ lucene/dev/branches/docvalues/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java Wed Nov 17 15:43:06 2010
@@ -143,11 +143,19 @@ public class DirectSolrConnection
path= pathAndParams;
params = new MapSolrParams( new HashMap<String, String>() );
}
-
+
+ return request(path, params, body);
+ }
+
+
+ public String request(String path, SolrParams params, String body) throws Exception
+ {
// Extract the handler from the path or params
SolrRequestHandler handler = core.getRequestHandler( path );
if( handler == null ) {
if( "/select".equals( path ) || "/select/".equalsIgnoreCase( path) ) {
+ if (params == null)
+ params = new MapSolrParams( new HashMap<String, String>() );
String qt = params.get( CommonParams.QT );
handler = core.getRequestHandler( qt );
if( handler == null ) {
@@ -158,13 +166,21 @@ public class DirectSolrConnection
if( handler == null ) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "unknown handler: "+path );
}
-
+
+ return request(handler, params, body);
+ }
+
+ public String request(SolrRequestHandler handler, SolrParams params, String body) throws Exception
+ {
+ if (params == null)
+ params = new MapSolrParams( new HashMap<String, String>() );
+
// Make a stream for the 'body' content
List<ContentStream> streams = new ArrayList<ContentStream>( 1 );
if( body != null && body.length() > 0 ) {
streams.add( new ContentStreamBase.StringStream( body ) );
}
-
+
SolrQueryRequest req = null;
try {
req = parser.buildRequestFrom( core, params, streams );
@@ -173,7 +189,7 @@ public class DirectSolrConnection
if( rsp.getException() != null ) {
throw rsp.getException();
}
-
+
// Now write it out
QueryResponseWriter responseWriter = core.getQueryResponseWriter(req);
StringWriter out = new StringWriter();
@@ -185,7 +201,9 @@ public class DirectSolrConnection
}
}
}
-
+
+
+
/**
* Use this method to close the underlying SolrCore.
*