You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/12/19 18:48:57 UTC
svn commit: r1552377 [10/15] - in /lucene/dev/branches/lucene5339: ./
dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/benchmark/src/ dev-tools/idea/lucene/demo/
dev-tools/idea/lucene/facet/ dev-tools/idea/solr/con...
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Thu Dec 19 17:48:47 2013
@@ -34,7 +34,6 @@ import org.apache.lucene.search.grouping
import org.apache.lucene.search.grouping.TopGroups;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
@@ -47,6 +46,7 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.response.ResultContext;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
@@ -190,7 +190,7 @@ public class QueryComponent extends Sear
// groupSort defaults to sort
String groupSortStr = params.get(GroupParams.GROUP_SORT);
//TODO: move weighting of sort
- Sort sortWithinGroup = groupSortStr == null ? groupSort : searcher.weightSort(QueryParsing.parseSort(groupSortStr, req));
+ Sort sortWithinGroup = groupSortStr == null ? groupSort : searcher.weightSort(QueryParsing.parseSortSpec(groupSortStr, req).getSort());
if (sortWithinGroup == null) {
sortWithinGroup = Sort.RELEVANCE;
}
@@ -449,16 +449,12 @@ public class QueryComponent extends Sear
{
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
- final CharsRef spare = new CharsRef();
// The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't
// currently have an option to return sort field values. Because of this, we
// take the documents given and re-derive the sort values.
boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES,false);
if(fsv){
- Sort sort = searcher.weightSort(rb.getSortSpec().getSort());
- SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort();
NamedList<Object[]> sortVals = new NamedList<Object[]>(); // order is important for the sort fields
- Field field = new StringField("dummy", "", Field.Store.NO); // a dummy Field
IndexReaderContext topReaderContext = searcher.getTopReaderContext();
List<AtomicReaderContext> leaves = topReaderContext.leaves();
AtomicReaderContext currentLeaf = null;
@@ -479,18 +475,22 @@ public class QueryComponent extends Sear
}
Arrays.sort(sortedIds);
+ SortSpec sortSpec = rb.getSortSpec();
+ Sort sort = searcher.weightSort(sortSpec.getSort());
+ SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort();
+ List<SchemaField> schemaFields = sortSpec.getSchemaFields();
+
+ for (int fld = 0; fld < schemaFields.size(); fld++) {
+ SchemaField schemaField = schemaFields.get(fld);
+ FieldType ft = null == schemaField? null : schemaField.getType();
+ SortField sortField = sortFields[fld];
- for (SortField sortField: sortFields) {
SortField.Type type = sortField.getType();
+ // :TODO: would be simpler to always serialize every position of SortField[]
if (type==SortField.Type.SCORE || type==SortField.Type.DOC) continue;
FieldComparator comparator = null;
-
- String fieldname = sortField.getField();
- FieldType ft = fieldname==null ? null : searcher.getSchema().getFieldTypeNoEx(fieldname);
-
Object[] vals = new Object[nDocs];
-
int lastIdx = -1;
int idx = 0;
@@ -516,31 +516,11 @@ public class QueryComponent extends Sear
doc -= currentLeaf.docBase; // adjust for what segment this is in
comparator.copy(0, doc);
Object val = comparator.value(0);
-
- // Sortable float, double, int, long types all just use a string
- // comparator. For these, we need to put the type into a readable
- // format. One reason for this is that XML can't represent all
- // string values (or even all unicode code points).
- // indexedToReadable() should be a no-op and should
- // thus be harmless anyway (for all current ways anyway)
- if (val instanceof String) {
- field.setStringValue((String)val);
- val = ft.toObject(field);
- }
-
- // Must do the same conversion when sorting by a
- // String field in Lucene, which returns the terms
- // data as BytesRef:
- if (val instanceof BytesRef) {
- UnicodeUtil.UTF8toUTF16((BytesRef)val, spare);
- field.setStringValue(spare.toString());
- val = ft.toObject(field);
- }
-
+ if (null != ft) val = ft.marshalSortValue(val);
vals[position] = val;
}
- sortVals.add(fieldname, vals);
+ sortVals.add(sortField.getField(), vals);
}
rsp.add("sort_values", sortVals);
@@ -778,7 +758,8 @@ public class QueryComponent extends Sear
sortFields = new SortField[]{SortField.FIELD_SCORE};
}
- SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField();
+ IndexSchema schema = rb.req.getSchema();
+ SchemaField uniqueKeyField = schema.getUniqueKeyField();
// id to shard mapping, to eliminate any accidental dups
@@ -787,7 +768,7 @@ public class QueryComponent extends Sear
// Merge the docs via a priority queue so we don't have to sort *all* of the
// documents... we only need to order the top (rows+start)
ShardFieldSortedHitQueue queue;
- queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount());
+ queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount(), rb.req.getSearcher());
NamedList<Object> shardInfo = null;
if(rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
@@ -813,11 +794,15 @@ public class QueryComponent extends Sear
StringWriter trace = new StringWriter();
t.printStackTrace(new PrintWriter(trace));
nl.add("trace", trace.toString() );
+ if (srsp.getShardAddress() != null) {
+ nl.add("shardAddress", srsp.getShardAddress());
+ }
}
else {
docs = (SolrDocumentList)srsp.getSolrResponse().getResponse().get("response");
nl.add("numFound", docs.getNumFound());
nl.add("maxScore", docs.getMaxScore());
+ nl.add("shardAddress", srsp.getShardAddress());
}
if(srsp.getSolrResponse()!=null) {
nl.add("time", srsp.getSolrResponse().getElapsedTime());
@@ -882,7 +867,7 @@ public class QueryComponent extends Sear
}
}
- shardDoc.sortFieldValues = sortFieldValues;
+ shardDoc.sortFieldValues = unmarshalSortValues(ss, sortFieldValues, schema);
queue.insertWithOverflow(shardDoc);
} // end for-each-doc-in-response
@@ -924,6 +909,47 @@ public class QueryComponent extends Sear
}
}
+ private NamedList unmarshalSortValues(SortSpec sortSpec,
+ NamedList sortFieldValues,
+ IndexSchema schema) {
+ NamedList unmarshalledSortValsPerField = new NamedList();
+
+ if (0 == sortFieldValues.size()) return unmarshalledSortValsPerField;
+
+ List<SchemaField> schemaFields = sortSpec.getSchemaFields();
+ SortField[] sortFields = sortSpec.getSort().getSort();
+
+ int marshalledFieldNum = 0;
+ for (int sortFieldNum = 0; sortFieldNum < sortFields.length; sortFieldNum++) {
+ final SortField sortField = sortFields[sortFieldNum];
+ final SortField.Type type = sortField.getType();
+
+ // :TODO: would be simpler to always serialize every position of SortField[]
+ if (type==SortField.Type.SCORE || type==SortField.Type.DOC) continue;
+
+ final String sortFieldName = sortField.getField();
+ final String valueFieldName = sortFieldValues.getName(marshalledFieldNum);
+ assert sortFieldName.equals(valueFieldName)
+ : "sortFieldValues name key does not match expected SortField.getField";
+
+ List sortVals = (List)sortFieldValues.getVal(marshalledFieldNum);
+
+ final SchemaField schemaField = schemaFields.get(sortFieldNum);
+ if (null == schemaField) {
+ unmarshalledSortValsPerField.add(sortField.getField(), sortVals);
+ } else {
+ FieldType fieldType = schemaField.getType();
+ List unmarshalledSortVals = new ArrayList();
+ for (Object sortVal : sortVals) {
+ unmarshalledSortVals.add(fieldType.unmarshalSortValue(sortVal));
+ }
+ unmarshalledSortValsPerField.add(sortField.getField(), unmarshalledSortVals);
+ }
+ marshalledFieldNum++;
+ }
+ return unmarshalledSortValsPerField;
+ }
+
private void createRetrieveDocs(ResponseBuilder rb) {
// TODO: in a system with nTiers > 2, we could be passed "ids" here
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java Thu Dec 19 17:48:47 2013
@@ -208,7 +208,7 @@ public class QueryElevationComponent ext
ZkController zkController = core.getCoreDescriptor().getCoreContainer().getZkController();
if (zkController != null) {
// TODO : shouldn't have to keep reading the config name when it has been read before
- exists = zkController.configFileExists(zkController.readConfigName(core.getCoreDescriptor().getCloudDescriptor().getCollectionName()), f);
+ exists = zkController.configFileExists(zkController.getZkStateReader().readConfigName(core.getCoreDescriptor().getCloudDescriptor().getCollectionName()), f);
} else {
File fC = new File(core.getResourceLoader().getConfigDir(), f);
File fD = new File(core.getDataDir(), f);
@@ -419,16 +419,16 @@ public class QueryElevationComponent ext
// insert documents in their proper place
SortSpec sortSpec = rb.getSortSpec();
if (sortSpec.getSort() == null) {
- sortSpec.setSort(new Sort(new SortField[]{
- new SortField("_elevate_", comparator, true),
- new SortField(null, SortField.Type.SCORE, false)
- }));
+ sortSpec.setSortAndFields(new Sort(new SortField[]{
+ new SortField("_elevate_", comparator, true),
+ new SortField(null, SortField.Type.SCORE, false)
+ }),
+ Arrays.asList(new SchemaField[2]));
} else {
// Check if the sort is based on score
- SortField[] current = sortSpec.getSort().getSort();
- Sort modified = this.modifySort(current, force, comparator);
- if(modified != null) {
- sortSpec.setSort(modified);
+ SortSpec modSortSpec = this.modifySortSpec(sortSpec, force, comparator);
+ if (null != modSortSpec) {
+ rb.setSortSpec(modSortSpec);
}
}
@@ -470,22 +470,43 @@ public class QueryElevationComponent ext
}
private Sort modifySort(SortField[] current, boolean force, ElevationComparatorSource comparator) {
+ SortSpec tmp = new SortSpec(new Sort(current), Arrays.asList(new SchemaField[current.length]));
+ tmp = modifySortSpec(tmp, force, comparator);
+ return null == tmp ? null : tmp.getSort();
+ }
+
+ private SortSpec modifySortSpec(SortSpec current, boolean force, ElevationComparatorSource comparator) {
boolean modify = false;
- ArrayList<SortField> sorts = new ArrayList<SortField>(current.length + 1);
+ SortField[] currentSorts = current.getSort().getSort();
+ List<SchemaField> currentFields = current.getSchemaFields();
+
+ ArrayList<SortField> sorts = new ArrayList<SortField>(currentSorts.length + 1);
+ List<SchemaField> fields = new ArrayList<SchemaField>(currentFields.size() + 1);
+
// Perhaps force it to always sort by score
- if (force && current[0].getType() != SortField.Type.SCORE) {
+ if (force && currentSorts[0].getType() != SortField.Type.SCORE) {
sorts.add(new SortField("_elevate_", comparator, true));
+ fields.add(null);
modify = true;
}
- for (SortField sf : current) {
+ for (int i = 0; i < currentSorts.length; i++) {
+ SortField sf = currentSorts[i];
if (sf.getType() == SortField.Type.SCORE) {
sorts.add(new SortField("_elevate_", comparator, !sf.getReverse()));
+ fields.add(null);
modify = true;
}
sorts.add(sf);
+ fields.add(currentFields.get(i));
}
-
- return modify ? new Sort(sorts.toArray(new SortField[sorts.size()])) : null;
+ if (modify) {
+ SortSpec newSpec = new SortSpec(new Sort(sorts.toArray(new SortField[sorts.size()])),
+ fields);
+ newSpec.setOffset(current.getOffset());
+ newSpec.setCount(current.getCount());
+ return newSpec;
+ }
+ return null;
}
@Override
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java Thu Dec 19 17:48:47 2013
@@ -17,27 +17,29 @@
package org.apache.solr.handler.component;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
-import org.apache.solr.util.RTimer;
+import org.apache.solr.common.util.ContentStream;
import org.apache.solr.core.CloseHook;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.util.RTimer;
import org.apache.solr.util.SolrPluginUtils;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
/**
*
@@ -69,6 +71,7 @@ public class SearchHandler extends Reque
names.add( HighlightComponent.COMPONENT_NAME );
names.add( StatsComponent.COMPONENT_NAME );
names.add( DebugComponent.COMPONENT_NAME );
+ names.add( AnalyticsComponent.COMPONENT_NAME );
return names;
}
@@ -164,6 +167,10 @@ public class SearchHandler extends Reque
{
// int sleep = req.getParams().getInt("sleep",0);
// if (sleep > 0) {log.error("SLEEPING for " + sleep); Thread.sleep(sleep);}
+ if (req.getContentStreams() != null && req.getContentStreams().iterator().hasNext()) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Search requests cannot accept content streams");
+ }
+
ResponseBuilder rb = new ResponseBuilder(req, rsp, components);
if (rb.requestInfo != null) {
rb.requestInfo.setResponseBuilder(rb);
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java Thu Dec 19 17:48:47 2013
@@ -16,18 +16,21 @@
*/
package org.apache.solr.handler.component;
-import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.PriorityQueue;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.search.MissingStringLastComparatorSource;
+import org.apache.solr.search.SolrIndexSearcher;
-import java.text.Collator;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
-import java.util.Locale;
+
+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
public class ShardDoc extends FieldDoc {
public String shard;
@@ -101,7 +104,7 @@ public class ShardDoc extends FieldDoc {
class ShardFieldSortedHitQueue extends PriorityQueue<ShardDoc> {
/** Stores a comparator corresponding to each field being sorted by */
- protected Comparator[] comparators;
+ protected Comparator<ShardDoc>[] comparators;
/** Stores the sort criteria being used. */
protected SortField[] fields;
@@ -109,9 +112,10 @@ class ShardFieldSortedHitQueue extends P
/** The order of these fieldNames should correspond to the order of sort field values retrieved from the shard */
protected List<String> fieldNames = new ArrayList<String>();
- public ShardFieldSortedHitQueue(SortField[] fields, int size) {
+ public ShardFieldSortedHitQueue(SortField[] fields, int size, IndexSearcher searcher) {
super(size);
final int n = fields.length;
+ //noinspection unchecked
comparators = new Comparator[n];
this.fields = new SortField[n];
for (int i = 0; i < n; ++i) {
@@ -123,8 +127,7 @@ class ShardFieldSortedHitQueue extends P
}
String fieldname = fields[i].getField();
- comparators[i] = getCachedComparator(fieldname, fields[i]
- .getType(), fields[i].getComparatorSource());
+ comparators[i] = getCachedComparator(fields[i], searcher);
if (fields[i].getType() == SortField.Type.STRING) {
this.fields[i] = new SortField(fieldname, SortField.Type.STRING,
@@ -169,47 +172,36 @@ class ShardFieldSortedHitQueue extends P
return c < 0;
}
- Comparator getCachedComparator(String fieldname, SortField.Type type, FieldComparatorSource factory) {
- Comparator comparator = null;
- switch (type) {
- case SCORE:
- comparator = comparatorScore(fieldname);
- break;
- case STRING:
- comparator = comparatorNatural(fieldname);
- break;
- case CUSTOM:
- if (factory instanceof MissingStringLastComparatorSource){
- comparator = comparatorMissingStringLast(fieldname);
- } else {
- // TODO: support other types such as random... is there a way to
- // support generically? Perhaps just comparing Object
- comparator = comparatorNatural(fieldname);
- // throw new RuntimeException("Custom sort not supported factory is "+factory.getClass());
- }
- break;
- case DOC:
- // TODO: we can support this!
- throw new RuntimeException("Doc sort not supported");
- default:
- comparator = comparatorNatural(fieldname);
- break;
- }
- return comparator;
- }
-
- class ShardComparator implements Comparator {
- String fieldName;
- int fieldNum;
- public ShardComparator(String fieldName) {
- this.fieldName = fieldName;
- this.fieldNum=0;
+ Comparator<ShardDoc> getCachedComparator(SortField sortField, IndexSearcher searcher) {
+ SortField.Type type = sortField.getType();
+ if (type == SortField.Type.SCORE) {
+ return comparatorScore();
+ } else if (type == SortField.Type.REWRITEABLE) {
+ try {
+ sortField = sortField.rewrite(searcher);
+ } catch (IOException e) {
+ throw new SolrException(SERVER_ERROR, "Exception rewriting sort field " + sortField, e);
+ }
+ }
+ return comparatorFieldComparator(sortField);
+ }
+
+ abstract class ShardComparator implements Comparator<ShardDoc> {
+ final SortField sortField;
+ final String fieldName;
+ final int fieldNum;
+
+ public ShardComparator(SortField sortField) {
+ this.sortField = sortField;
+ this.fieldName = sortField.getField();
+ int fieldNum = 0;
for (int i=0; i<fieldNames.size(); i++) {
if (fieldNames.get(i).equals(fieldName)) {
- this.fieldNum = i;
+ fieldNum = i;
break;
}
}
+ this.fieldNum = fieldNum;
}
Object sortVal(ShardDoc shardDoc) {
@@ -217,22 +209,14 @@ class ShardFieldSortedHitQueue extends P
List lst = (List)shardDoc.sortFieldValues.getVal(fieldNum);
return lst.get(shardDoc.orderInShard);
}
-
- @Override
- public int compare(Object o1, Object o2) {
- return 0;
- }
}
- static Comparator comparatorScore(final String fieldName) {
- return new Comparator() {
+ static Comparator<ShardDoc> comparatorScore() {
+ return new Comparator<ShardDoc>() {
@Override
- public final int compare(final Object o1, final Object o2) {
- ShardDoc e1 = (ShardDoc) o1;
- ShardDoc e2 = (ShardDoc) o2;
-
- final float f1 = e1.score;
- final float f2 = e2.score;
+ public final int compare(final ShardDoc o1, final ShardDoc o2) {
+ final float f1 = o1.score;
+ final float f2 = o2.score;
if (f1 < f2)
return -1;
if (f1 > f2)
@@ -242,71 +226,24 @@ class ShardFieldSortedHitQueue extends P
};
}
- // The lucene natural sort ordering corresponds to numeric
- // and string natural sort orderings (ascending). Since
- // the PriorityQueue keeps the biggest elements by default,
- // we need to reverse the natural compare ordering so that the
- // smallest elements are kept instead of the largest... hence
- // the negative sign on the final compareTo().
- Comparator comparatorNatural(String fieldName) {
- return new ShardComparator(fieldName) {
- @Override
- public final int compare(final Object o1, final Object o2) {
- ShardDoc sd1 = (ShardDoc) o1;
- ShardDoc sd2 = (ShardDoc) o2;
- Comparable v1 = (Comparable)sortVal(sd1);
- Comparable v2 = (Comparable)sortVal(sd2);
- if (v1==v2)
- return 0;
- if (v1==null)
- return 1;
- if(v2==null)
- return -1;
- return -v1.compareTo(v2);
- }
- };
- }
-
-
- Comparator comparatorStringLocale(final String fieldName,
- Locale locale) {
- final Collator collator = Collator.getInstance(locale);
- return new ShardComparator(fieldName) {
- @Override
- public final int compare(final Object o1, final Object o2) {
- ShardDoc sd1 = (ShardDoc) o1;
- ShardDoc sd2 = (ShardDoc) o2;
- Comparable v1 = (Comparable)sortVal(sd1);
- Comparable v2 = (Comparable)sortVal(sd2);
- if (v1==v2)
- return 0;
- if (v1==null)
- return 1;
- if(v2==null)
- return -1;
- return -collator.compare(v1,v2);
- }
- };
- }
-
+ Comparator<ShardDoc> comparatorFieldComparator(SortField sortField) {
+ final FieldComparator fieldComparator;
+ try {
+ fieldComparator = sortField.getComparator(0, 0);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to get FieldComparator for sortField " + sortField);
+ }
- Comparator comparatorMissingStringLast(final String fieldName) {
- return new ShardComparator(fieldName) {
+ return new ShardComparator(sortField) {
+ // Since the PriorityQueue keeps the biggest elements by default,
+ // we need to reverse the field compare ordering so that the
+ // smallest elements are kept instead of the largest... hence
+ // the negative sign.
@Override
- public final int compare(final Object o1, final Object o2) {
- ShardDoc sd1 = (ShardDoc) o1;
- ShardDoc sd2 = (ShardDoc) o2;
- Comparable v1 = (Comparable)sortVal(sd1);
- Comparable v2 = (Comparable)sortVal(sd2);
- if (v1==v2)
- return 0;
- if (v1==null)
- return -1;
- if(v2==null)
- return 1;
- return -v1.compareTo(v2);
+ public int compare(final ShardDoc o1, final ShardDoc o2) {
+ //noinspection unchecked
+ return -fieldComparator.compareValues(sortVal(o1), sortVal(o2));
}
};
}
-
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java Thu Dec 19 17:48:47 2013
@@ -320,7 +320,16 @@ public class SpellCheckComponent extends
if (maxResultsForSuggest==null || !isCorrectlySpelled) {
for (ShardRequest sreq : rb.finished) {
for (ShardResponse srsp : sreq.responses) {
- NamedList nl = (NamedList) srsp.getSolrResponse().getResponse().get("spellcheck");
+ NamedList nl = null;
+ try {
+ nl = (NamedList) srsp.getSolrResponse().getResponse().get("spellcheck");
+ } catch (Exception e) {
+ if (rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false)) {
+ continue; // looks like a shard did not return anything
+ }
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Unable to read spelling info for shard: " + srsp.getShard(), e);
+ }
LOG.info(srsp.getShard() + " " + nl);
if (nl != null) {
mergeData.totalNumberShardResponses++;
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java Thu Dec 19 17:48:47 2013
@@ -102,7 +102,16 @@ public class StatsComponent extends Sear
StatsInfo si = rb._statsInfo;
for (ShardResponse srsp : sreq.responses) {
- NamedList stats = (NamedList) srsp.getSolrResponse().getResponse().get("stats");
+ NamedList stats = null;
+ try {
+ stats = (NamedList) srsp.getSolrResponse().getResponse().get("stats");
+ } catch (Exception e) {
+ if (rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false)) {
+ continue; // looks like a shard did not return anything
+ }
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Unable to read stats info for shard: " + srsp.getShard(), e);
+ }
NamedList stats_fields = (NamedList) stats.get("stats_fields");
if (stats_fields != null) {
@@ -167,8 +176,9 @@ class StatsInfo {
String[] statsFs = params.getParams(StatsParams.STATS_FIELD);
if (statsFs != null) {
for (String field : statsFs) {
+ boolean calcDistinct = params.getFieldBool(field, StatsParams.STATS_CALC_DISTINCT, false);
SchemaField sf = rb.req.getSchema().getField(field);
- statsFields.put(field, StatsValuesFactory.createStatsValues(sf));
+ statsFields.put(field, StatsValuesFactory.createStatsValues(sf, calcDistinct));
}
}
}
@@ -207,6 +217,7 @@ class SimpleStats {
if (null != statsFs) {
final IndexSchema schema = searcher.getSchema();
for (String f : statsFs) {
+ boolean calcDistinct = params.getFieldBool(f, StatsParams.STATS_CALC_DISTINCT, false);
String[] facets = params.getFieldParams(f, StatsParams.STATS_FACET);
if (facets == null) {
facets = new String[0]; // make sure it is something...
@@ -218,9 +229,9 @@ class SimpleStats {
if (sf.multiValued() || ft.multiValuedFieldCache()) {
//use UnInvertedField for multivalued fields
UnInvertedField uif = UnInvertedField.getUnInvertedField(f, searcher);
- stv = uif.getStats(searcher, docs, facets).getStatsValues();
+ stv = uif.getStats(searcher, docs, calcDistinct, facets).getStatsValues();
} else {
- stv = getFieldCacheStats(f, facets);
+ stv = getFieldCacheStats(f, calcDistinct, facets);
}
if (isShard == true || (Long) stv.get("count") > 0) {
res.add(f, stv);
@@ -232,11 +243,11 @@ class SimpleStats {
return res;
}
- public NamedList<?> getFieldCacheStats(String fieldName, String[] facet) throws IOException {
+ public NamedList<?> getFieldCacheStats(String fieldName, boolean calcDistinct, String[] facet) throws IOException {
IndexSchema schema = searcher.getSchema();
final SchemaField sf = schema.getField(fieldName);
- final StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
+ final StatsValues allstats = StatsValuesFactory.createStatsValues(sf, calcDistinct);
List<FieldFacetStats> facetStats = new ArrayList<FieldFacetStats>();
for( String facetField : facet ) {
@@ -247,7 +258,7 @@ class SimpleStats {
"Stats can only facet on single-valued fields, not: " + facetField );
}
- facetStats.add(new FieldFacetStats(searcher, facetField, sf, fsf));
+ facetStats.add(new FieldFacetStats(searcher, facetField, sf, fsf, calcDistinct));
}
final Iterator<AtomicReaderContext> ctxIt = searcher.getIndexReader().leaves().iterator();
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java Thu Dec 19 17:48:47 2013
@@ -18,10 +18,7 @@
package org.apache.solr.handler.component;
import java.io.IOException;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Map;
-import java.util.HashMap;
+import java.util.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
@@ -44,7 +41,7 @@ public class StatsValuesFactory {
* @param sf SchemaField for the field whose statistics will be created by the resulting StatsValues
* @return Instance of StatsValues that will create statistics from values from a field of the given type
*/
- public static StatsValues createStatsValues(SchemaField sf) {
+ public static StatsValues createStatsValues(SchemaField sf, boolean calcDistinct) {
// TODO: allow for custom field types
FieldType fieldType = sf.getType();
if (DoubleField.class.isInstance(fieldType) ||
@@ -56,13 +53,13 @@ public class StatsValuesFactory {
SortableIntField.class.isInstance(fieldType) ||
SortableLongField.class.isInstance(fieldType) ||
SortableFloatField.class.isInstance(fieldType)) {
- return new NumericStatsValues(sf);
+ return new NumericStatsValues(sf, calcDistinct);
} else if (DateField.class.isInstance(fieldType)) {
- return new DateStatsValues(sf);
+ return new DateStatsValues(sf, calcDistinct);
} else if (StrField.class.isInstance(fieldType)) {
- return new StringStatsValues(sf);
+ return new StringStatsValues(sf, calcDistinct);
} else if (sf.getType().getClass().equals(EnumField.class)) {
- return new EnumStatsValues(sf);
+ return new EnumStatsValues(sf, calcDistinct);
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field type " + fieldType + " is not currently supported");
}
@@ -84,15 +81,20 @@ abstract class AbstractStatsValues<T> im
protected T min;
protected long missing;
protected long count;
+ protected long countDistinct;
+ protected Set<T> distinctValues;
private ValueSource valueSource;
protected FunctionValues values;
+ protected boolean calcDistinct = false;
// facetField facetValue
protected Map<String, Map<String, StatsValues>> facets = new HashMap<String, Map<String, StatsValues>>();
- protected AbstractStatsValues(SchemaField sf) {
+ protected AbstractStatsValues(SchemaField sf, boolean calcDistinct) {
this.sf = sf;
this.ft = sf.getType();
+ this.distinctValues = new TreeSet<T>();
+ this.calcDistinct = calcDistinct;
}
/**
@@ -102,6 +104,10 @@ abstract class AbstractStatsValues<T> im
public void accumulate(NamedList stv) {
count += (Long) stv.get("count");
missing += (Long) stv.get("missing");
+ if (calcDistinct) {
+ distinctValues.addAll((Collection<T>) stv.get("distinctValues"));
+ countDistinct = distinctValues.size();
+ }
updateMinMax((T) stv.get("min"), (T) stv.get("max"));
updateTypeSpecificStats(stv);
@@ -123,7 +129,7 @@ abstract class AbstractStatsValues<T> im
String val = vals.getName(j);
StatsValues vvals = addTo.get(val);
if (vvals == null) {
- vvals = StatsValuesFactory.createStatsValues(sf);
+ vvals = StatsValuesFactory.createStatsValues(sf, calcDistinct);
addTo.put(val, vvals);
}
vvals.accumulate((NamedList) vals.getVal(j));
@@ -142,6 +148,10 @@ abstract class AbstractStatsValues<T> im
public void accumulate(T value, int count) {
this.count += count;
+ if (calcDistinct) {
+ distinctValues.add(value);
+ countDistinct = distinctValues.size();
+ }
updateMinMax(value, value);
updateTypeSpecificStats(value, count);
}
@@ -181,6 +191,11 @@ abstract class AbstractStatsValues<T> im
res.add("max", max);
res.add("count", count);
res.add("missing", missing);
+ if (calcDistinct) {
+ res.add("distinctValues", distinctValues);
+ res.add("countDistinct", countDistinct);
+ }
+
addTypeSpecificStats(res);
// add the facet stats
@@ -242,8 +257,8 @@ class NumericStatsValues extends Abstrac
double sum;
double sumOfSquares;
- public NumericStatsValues(SchemaField sf) {
- super(sf);
+ public NumericStatsValues(SchemaField sf, boolean calcDistinct) {
+ super(sf, calcDistinct);
min = Double.POSITIVE_INFINITY;
max = Double.NEGATIVE_INFINITY;
}
@@ -317,8 +332,8 @@ class NumericStatsValues extends Abstrac
*/
class EnumStatsValues extends AbstractStatsValues<EnumFieldValue> {
- public EnumStatsValues(SchemaField sf) {
- super(sf);
+ public EnumStatsValues(SchemaField sf, boolean calcDistinct) {
+ super(sf, calcDistinct);
}
/**
@@ -386,8 +401,8 @@ class DateStatsValues extends AbstractSt
private long sum = -1;
double sumOfSquares = 0;
- public DateStatsValues(SchemaField sf) {
- super(sf);
+ public DateStatsValues(SchemaField sf, boolean calcDistinct) {
+ super(sf, calcDistinct);
}
@Override
@@ -404,8 +419,11 @@ class DateStatsValues extends AbstractSt
*/
@Override
protected void updateTypeSpecificStats(NamedList stv) {
- sum += ((Date) stv.get("sum")).getTime();
- sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
+ Date date = (Date) stv.get("sum");
+ if (date != null) {
+ sum += date.getTime();
+ sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
+ }
}
/**
@@ -469,8 +487,8 @@ class DateStatsValues extends AbstractSt
*/
class StringStatsValues extends AbstractStatsValues<String> {
- public StringStatsValues(SchemaField sf) {
- super(sf);
+ public StringStatsValues(SchemaField sf, boolean calcDistinct) {
+ super(sf, calcDistinct);
}
@Override
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java Thu Dec 19 17:48:47 2013
@@ -232,9 +232,20 @@ public class DocValuesFacets {
return res;
}
- /** accumulates per-segment single-valued facet counts, mapping to global ordinal space */
- // specialized since the single-valued case is different
+ /** accumulates per-segment single-valued facet counts */
static void accumSingle(int counts[], int startTermIndex, SortedDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
+ if (startTermIndex == -1 && (map == null || si.getValueCount() < disi.cost()*10)) {
+ // no prefixing, not too many unique values wrt matching docs (lucene/facets heuristic):
+ // collect separately per-segment, then map to global ords
+ accumSingleSeg(counts, si, disi, subIndex, map);
+ } else {
+ // otherwise: do collect+map on the fly
+ accumSingleGeneric(counts, startTermIndex, si, disi, subIndex, map);
+ }
+ }
+
+ /** accumulates per-segment single-valued facet counts, mapping to global ordinal space on-the-fly */
+ static void accumSingleGeneric(int counts[], int startTermIndex, SortedDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
int term = si.getOrd(doc);
@@ -246,8 +257,41 @@ public class DocValuesFacets {
}
}
- /** accumulates per-segment multi-valued facet counts, mapping to global ordinal space */
+ /** "typical" single-valued faceting: not too many unique values, no prefixing. maps to global ordinals as a separate step */
+ static void accumSingleSeg(int counts[], SortedDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
+ // First count in seg-ord space:
+ final int segCounts[];
+ if (map == null) {
+ segCounts = counts;
+ } else {
+ segCounts = new int[1+si.getValueCount()];
+ }
+
+ int doc;
+ while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ segCounts[1+si.getOrd(doc)]++;
+ }
+
+ // migrate to global ords (if necessary)
+ if (map != null) {
+ migrateGlobal(counts, segCounts, subIndex, map);
+ }
+ }
+
+ /** accumulates per-segment multi-valued facet counts */
static void accumMulti(int counts[], int startTermIndex, SortedSetDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
+ if (startTermIndex == -1 && (map == null || si.getValueCount() < disi.cost()*10)) {
+ // no prefixing, not too many unique values wrt matching docs (lucene/facets heuristic):
+ // collect separately per-segment, then map to global ords
+ accumMultiSeg(counts, si, disi, subIndex, map);
+ } else {
+ // otherwise: do collect+map on the fly
+ accumMultiGeneric(counts, startTermIndex, si, disi, subIndex, map);
+ }
+ }
+
+ /** accumulates per-segment multi-valued facet counts, mapping to global ordinal space on-the-fly */
+ static void accumMultiGeneric(int counts[], int startTermIndex, SortedSetDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
si.setDocument(doc);
@@ -269,4 +313,47 @@ public class DocValuesFacets {
} while ((term = (int) si.nextOrd()) >= 0);
}
}
+
+ /** "typical" multi-valued faceting: not too many unique values, no prefixing. maps to global ordinals as a separate step */
+ static void accumMultiSeg(int counts[], SortedSetDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
+ // First count in seg-ord space:
+ final int segCounts[];
+ if (map == null) {
+ segCounts = counts;
+ } else {
+ segCounts = new int[1+(int)si.getValueCount()];
+ }
+
+ int doc;
+ while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ si.setDocument(doc);
+ int term = (int) si.nextOrd();
+ if (term < 0) {
+ counts[0]++; // missing
+ } else {
+ do {
+ segCounts[1+term]++;
+ } while ((term = (int)si.nextOrd()) >= 0);
+ }
+ }
+
+ // migrate to global ords (if necessary)
+ if (map != null) {
+ migrateGlobal(counts, segCounts, subIndex, map);
+ }
+ }
+
+ /** folds counts in segment ordinal space (segCounts) into global ordinal space (counts) */
+ static void migrateGlobal(int counts[], int segCounts[], int subIndex, OrdinalMap map) {
+ // missing count
+ counts[0] += segCounts[0];
+
+ // migrate actual ordinals
+ for (int ord = 1; ord < segCounts.length; ord++) {
+ int count = segCounts[ord];
+ if (count != 0) {
+ counts[1+(int) map.getGlobalOrd(subIndex, ord-1)] += count;
+ }
+ }
+ }
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Thu Dec 19 17:48:47 2013
@@ -566,7 +566,7 @@ public class SimpleFacets {
throw se;
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR,
- "Exception during facet.field: " + workerFacetValue, e.getCause());
+ "Exception during facet.field: " + workerFacetValue, e);
} finally {
semaphore.release();
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/UnInvertedField.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/UnInvertedField.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/request/UnInvertedField.java Thu Dec 19 17:48:47 2013
@@ -464,11 +464,12 @@ public class UnInvertedField extends Doc
*
* @param searcher The Searcher to use to gather the statistics
* @param baseDocs The {@link org.apache.solr.search.DocSet} to gather the stats on
+ * @param calcDistinct whether distinct values should be collected and counted
* @param facet One or more fields to facet on.
* @return The {@link org.apache.solr.handler.component.StatsValues} collected
* @throws IOException If there is a low-level I/O error.
*/
- public StatsValues getStats(SolrIndexSearcher searcher, DocSet baseDocs, String[] facet) throws IOException {
+ public StatsValues getStats(SolrIndexSearcher searcher, DocSet baseDocs, boolean calcDistinct, String[] facet) throws IOException {
//this function is ripped off nearly wholesale from the getCounts function to use
//for multiValued fields within the StatsComponent. may be useful to find common
//functionality between the two and refactor code somewhat
@@ -477,7 +478,7 @@ public class UnInvertedField extends Doc
SchemaField sf = searcher.getSchema().getField(field);
// FieldType ft = sf.getType();
- StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
+ StatsValues allstats = StatsValuesFactory.createStatsValues(sf, calcDistinct);
DocSet docs = baseDocs;
@@ -494,7 +495,7 @@ public class UnInvertedField extends Doc
SortedDocValues si;
for (String f : facet) {
SchemaField facet_sf = searcher.getSchema().getField(f);
- finfo[i] = new FieldFacetStats(searcher, f, sf, facet_sf);
+ finfo[i] = new FieldFacetStats(searcher, f, sf, facet_sf, calcDistinct);
i++;
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/CollationField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/CollationField.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/CollationField.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/CollationField.java Thu Dec 19 17:48:47 2013
@@ -22,6 +22,9 @@ import java.io.InputStream;
import java.text.Collator;
import java.text.ParseException;
import java.text.RuleBasedCollator;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -30,7 +33,12 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.collation.CollationKeyAnalyzer;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.StorableField;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DocTermOrdsRangeFilter;
+import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermRangeQuery;
@@ -39,6 +47,7 @@ import org.apache.lucene.util.Version;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.Base64;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
@@ -138,8 +147,7 @@ public class CollationField extends Fiel
else
throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid decomposition: " + decomposition);
}
- // we use 4.0 because it ensures we just encode the pure byte[] keys.
- analyzer = new CollationKeyAnalyzer(Version.LUCENE_40, collator);
+ analyzer = new CollationKeyAnalyzer(Version.LUCENE_CURRENT, collator);
}
/**
@@ -209,30 +217,81 @@ public class CollationField extends Fiel
* its just that all methods are synced), this keeps things
* simple (we already have a threadlocal clone in the reused TS)
*/
- private BytesRef analyzeRangePart(String field, String part) {
- try (TokenStream source = analyzer.tokenStream(field, part)) {
+ private BytesRef getCollationKey(String field, String text) {
+ try (TokenStream source = analyzer.tokenStream(field, text)) {
source.reset();
TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
BytesRef bytes = termAtt.getBytesRef();
// we control the analyzer here: most errors are impossible
if (!source.incrementToken())
- throw new IllegalArgumentException("analyzer returned no terms for range part: " + part);
+ throw new IllegalArgumentException("analyzer returned no terms for text: " + text);
termAtt.fillBytesRef();
assert !source.incrementToken();
source.end();
return BytesRef.deepCopyOf(bytes);
} catch (IOException e) {
- throw new RuntimeException("Unable to analyze range part: " + part, e);
+ throw new RuntimeException("Unable to analyze text: " + text, e);
}
}
@Override
public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
String f = field.getName();
- BytesRef low = part1 == null ? null : analyzeRangePart(f, part1);
- BytesRef high = part2 == null ? null : analyzeRangePart(f, part2);
- return new TermRangeQuery(field.getName(), low, high, minInclusive, maxInclusive);
+ BytesRef low = part1 == null ? null : getCollationKey(f, part1);
+ BytesRef high = part2 == null ? null : getCollationKey(f, part2);
+ if (!field.indexed() && field.hasDocValues()) {
+ if (field.multiValued()) {
+ return new ConstantScoreQuery(DocTermOrdsRangeFilter.newBytesRefRange(
+ field.getName(), low, high, minInclusive, maxInclusive));
+ } else {
+ return new ConstantScoreQuery(FieldCacheRangeFilter.newBytesRefRange(
+ field.getName(), low, high, minInclusive, maxInclusive));
+ }
+ } else {
+ return new TermRangeQuery(field.getName(), low, high, minInclusive, maxInclusive);
+ }
+ }
+
+ @Override
+ public void checkSchemaField(SchemaField field) {
+ // no-op
+ }
+
+ @Override
+ public List<StorableField> createFields(SchemaField field, Object value, float boost) {
+ if (field.hasDocValues()) {
+ List<StorableField> fields = new ArrayList<StorableField>();
+ fields.add(createField(field, value, boost));
+ final BytesRef bytes = getCollationKey(field.getName(), value.toString());
+ if (field.multiValued()) {
+ fields.add(new SortedSetDocValuesField(field.getName(), bytes));
+ } else {
+ fields.add(new SortedDocValuesField(field.getName(), bytes));
+ }
+ return fields;
+ } else {
+ return Collections.singletonList(createField(field, value, boost));
+ }
+ }
+
+ @Override
+ public Object marshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ final BytesRef val = (BytesRef)value;
+ return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ final String val = (String)value;
+ final byte[] bytes = Base64.base64ToByteArray(val);
+ return new BytesRef(bytes);
}
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/FieldType.java Thu Dec 19 17:48:47 2013
@@ -932,4 +932,20 @@ public abstract class FieldType extends
}
return analyzerProps;
}
+
+ /**
+ * Convert a value used by the FieldComparator for this FieldType's SortField
+ * into a marshalable value for distributed sorting.
+ */
+ public Object marshalSortValue(Object value) {
+ return value;
+ }
+
+ /**
+ * Convert a value marshaled via {@link #marshalSortValue} back
+ * into a value usable by the FieldComparator for this FieldType's SortField
+ */
+ public Object unmarshalSortValue(Object value) {
+ return value;
+ }
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java Thu Dec 19 17:48:47 2013
@@ -100,6 +100,27 @@ public class SortableDoubleField extends
String sval = f.stringValue();
writer.writeDouble(name, NumberUtils.SortableStr2double(sval));
}
+
+ @Override
+ public Object marshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ CharsRef chars = new CharsRef();
+ UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+ return NumberUtils.SortableStr2double(chars.toString());
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ String sortableString = NumberUtils.double2sortableStr(value.toString());
+ BytesRef bytes = new BytesRef();
+ UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+ return bytes;
+ }
}
class SortableDoubleFieldSource extends FieldCacheSource {
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java Thu Dec 19 17:48:47 2013
@@ -101,6 +101,27 @@ public class SortableFloatField extends
String sval = f.stringValue();
writer.writeFloat(name, NumberUtils.SortableStr2float(sval));
}
+
+ @Override
+ public Object marshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ CharsRef chars = new CharsRef();
+ UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+ return NumberUtils.SortableStr2float(chars.toString());
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ String sortableString = NumberUtils.float2sortableStr(value.toString());
+ BytesRef bytes = new BytesRef();
+ UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+ return bytes;
+ }
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableIntField.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableIntField.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableIntField.java Thu Dec 19 17:48:47 2013
@@ -104,6 +104,27 @@ public class SortableIntField extends Pr
String sval = f.stringValue();
writer.writeInt(name, NumberUtils.SortableStr2int(sval,0,sval.length()));
}
+
+ @Override
+ public Object marshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ CharsRef chars = new CharsRef();
+ UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+ return NumberUtils.SortableStr2int(chars.toString());
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ String sortableString = NumberUtils.int2sortableStr(value.toString());
+ BytesRef bytes = new BytesRef();
+ UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+ return bytes;
+ }
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableLongField.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableLongField.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/SortableLongField.java Thu Dec 19 17:48:47 2013
@@ -100,6 +100,27 @@ public class SortableLongField extends P
String sval = f.stringValue();
writer.writeLong(name, NumberUtils.SortableStr2long(sval,0,sval.length()));
}
+
+ @Override
+ public Object marshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ CharsRef chars = new CharsRef();
+ UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+ return NumberUtils.SortableStr2long(chars.toString());
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ String sortableString = NumberUtils.long2sortableStr(value.toString());
+ BytesRef bytes = new BytesRef();
+ UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+ return bytes;
+ }
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/StrField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/StrField.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/StrField.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/StrField.java Thu Dec 19 17:48:47 2013
@@ -29,6 +29,8 @@ import org.apache.lucene.index.StorableF
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
@@ -81,6 +83,27 @@ public class StrField extends PrimitiveF
@Override
public void checkSchemaField(SchemaField field) {
}
+
+ @Override
+ public Object marshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ CharsRef spare = new CharsRef();
+ UnicodeUtil.UTF8toUTF16((BytesRef)value, spare);
+ return spare.toString();
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ BytesRef spare = new BytesRef();
+ String stringVal = (String)value;
+ UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare);
+ return spare;
+ }
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/TextField.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/TextField.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/schema/TextField.java Thu Dec 19 17:48:47 2013
@@ -23,7 +23,9 @@ import org.apache.lucene.index.StorableF
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.QueryBuilder;
+import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
@@ -165,4 +167,25 @@ public class TextField extends FieldType
public boolean isExplicitMultiTermAnalyzer() {
return isExplicitMultiTermAnalyzer;
}
+
+ @Override
+ public Object marshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ CharsRef spare = new CharsRef();
+ UnicodeUtil.UTF8toUTF16((BytesRef)value, spare);
+ return spare.toString();
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ BytesRef spare = new BytesRef();
+ String stringVal = (String)value;
+ UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare);
+ return spare;
+ }
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/BitDocSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/BitDocSet.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/BitDocSet.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/BitDocSet.java Thu Dec 19 17:48:47 2013
@@ -296,11 +296,11 @@ public class BitDocSet extends DocSetBas
@Override
public long cost() {
// we don't want to actually compute cardinality, but
- // if its already been computed, we use it
+ // if its already been computed, we use it (pro-rated for the segment)
if (size != -1) {
- return size;
+ return (long)(size * ((OpenBitSet.bits2words(maxDoc)<<6) / (float)bs.capacity()));
} else {
- return bs.capacity();
+ return maxDoc;
}
}
};
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java Thu Dec 19 17:48:47 2013
@@ -118,21 +118,15 @@ public class CollapsingQParserPlugin ext
}
}
- private class CollapsingPostFilter extends ExtendedQueryBase implements PostFilter {
+ public class CollapsingPostFilter extends ExtendedQueryBase implements PostFilter, ScoreFilter {
private Object cacheId;
private String field;
- private int leafCount;
- private SortedDocValues docValues;
- private int maxDoc;
private String max;
private String min;
- private FieldType fieldType;
+ private boolean needsScores = true;
private int nullPolicy;
- private SolrIndexSearcher searcher;
- private SolrParams solrParams;
private Map context;
- private IndexSchema schema;
public static final int NULL_POLICY_IGNORE = 0;
public static final int NULL_POLICY_COLLAPSE = 1;
public static final int NULL_POLICY_EXPAND = 2;
@@ -180,7 +174,13 @@ public class CollapsingQParserPlugin ext
public CollapsingPostFilter(SolrParams localParams, SolrParams params, SolrQueryRequest request) throws IOException {
this.cacheId = new Object();
this.field = localParams.get("field");
- this.solrParams = params;
+ this.max = localParams.get("max");
+ this.min = localParams.get("min");
+ this.context = request.getContext();
+ if(this.min != null || this.max != null) {
+ this.needsScores = needsScores(params);
+ }
+
String nPolicy = localParams.get("nullPolicy", NULL_IGNORE);
if(nPolicy.equals(NULL_IGNORE)) {
this.nullPolicy = NULL_POLICY_IGNORE;
@@ -191,34 +191,12 @@ public class CollapsingQParserPlugin ext
} else {
throw new IOException("Invalid nullPolicy:"+nPolicy);
}
- this.searcher = request.getSearcher();
- this.leafCount = searcher.getTopReaderContext().leaves().size();
- this.maxDoc = searcher.maxDoc();
- this.schema = searcher.getSchema();
- SchemaField schemaField = schema.getField(this.field);
- if(schemaField.hasDocValues()) {
- this.docValues = searcher.getAtomicReader().getSortedDocValues(this.field);
- } else {
- this.docValues = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), this.field);
- }
-
- this.max = localParams.get("max");
- if(this.max != null) {
- this.fieldType = searcher.getSchema().getField(this.max).getType();
- }
-
- this.min = localParams.get("min");
- if(this.min != null) {
- this.fieldType = searcher.getSchema().getField(this.min).getType();
- }
-
- this.context = request.getContext();
}
- private IntOpenHashSet getBoostDocs(IndexSearcher indexSearcher, Set<String> boosted) throws IOException {
+ private IntOpenHashSet getBoostDocs(SolrIndexSearcher indexSearcher, Set<String> boosted) throws IOException {
IntOpenHashSet boostDocs = null;
if(boosted != null) {
- SchemaField idField = this.schema.getUniqueKeyField();
+ SchemaField idField = indexSearcher.getSchema().getUniqueKeyField();
String fieldName = idField.getName();
HashSet<BytesRef> localBoosts = new HashSet(boosted.size()*2);
Iterator<String> boostedIt = boosted.iterator();
@@ -258,22 +236,47 @@ public class CollapsingQParserPlugin ext
public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) {
try {
- IntOpenHashSet boostDocs = getBoostDocs(indexSearcher, (Set<String>) (this.context.get(QueryElevationComponent.BOOSTED)));
+
+ SolrIndexSearcher searcher = (SolrIndexSearcher)indexSearcher;
+ IndexSchema schema = searcher.getSchema();
+ SchemaField schemaField = schema.getField(this.field);
+
+ SortedDocValues docValues = null;
+
+ if(schemaField.hasDocValues()) {
+ docValues = searcher.getAtomicReader().getSortedDocValues(this.field);
+ } else {
+ docValues = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), this.field);
+ }
+
+ FieldType fieldType = null;
+
+ if(this.max != null) {
+ fieldType = searcher.getSchema().getField(this.max).getType();
+ }
+
+ if(this.min != null) {
+ fieldType = searcher.getSchema().getField(this.min).getType();
+ }
+
+ int maxDoc = searcher.maxDoc();
+ int leafCount = searcher.getTopReaderContext().leaves().size();
+
+ IntOpenHashSet boostDocs = getBoostDocs(searcher, (Set<String>) (this.context.get(QueryElevationComponent.BOOSTED)));
if(this.min != null || this.max != null) {
- return new CollapsingFieldValueCollector(this.maxDoc,
- this.leafCount,
- this.docValues,
- this.searcher,
- this.nullPolicy,
- max != null ? this.max : this.min,
- max != null,
- needsScores(this.solrParams),
- this.fieldType,
- boostDocs);
+ return new CollapsingFieldValueCollector(maxDoc,
+ leafCount,
+ docValues,
+ this.nullPolicy,
+ max != null ? this.max : this.min,
+ max != null,
+ this.needsScores,
+ fieldType,
+ boostDocs);
} else {
- return new CollapsingScoreCollector(this.maxDoc, this.leafCount, this.docValues, this.nullPolicy, boostDocs);
+ return new CollapsingScoreCollector(maxDoc, leafCount, docValues, this.nullPolicy, boostDocs);
}
} catch (Exception e) {
throw new RuntimeException(e);
@@ -283,7 +286,7 @@ public class CollapsingQParserPlugin ext
private boolean needsScores(SolrParams params) {
String sortSpec = params.get("sort");
- if(sortSpec != null) {
+ if(sortSpec != null && sortSpec.length()!=0) {
String[] sorts = sortSpec.split(",");
for(String s: sorts) {
String parts[] = s.split(" ");
@@ -500,7 +503,6 @@ public class CollapsingQParserPlugin ext
public CollapsingFieldValueCollector(int maxDoc,
int segments,
SortedDocValues values,
- SolrIndexSearcher searcher,
int nullPolicy,
String field,
boolean max,
@@ -516,11 +518,11 @@ public class CollapsingQParserPlugin ext
this.needsScores = needsScores;
this.boostDocs = boostDocs;
if(fieldType instanceof TrieIntField) {
- this.fieldValueCollapse = new IntValueCollapse(searcher, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ this.fieldValueCollapse = new IntValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
} else if(fieldType instanceof TrieLongField) {
- this.fieldValueCollapse = new LongValueCollapse(searcher, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ this.fieldValueCollapse = new LongValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
} else if(fieldType instanceof TrieFloatField) {
- this.fieldValueCollapse = new FloatValueCollapse(searcher, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ this.fieldValueCollapse = new FloatValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
} else {
throw new IOException("min/max must be either TrieInt, TrieLong or TrieFloat.");
}
@@ -616,7 +618,7 @@ public class CollapsingQParserPlugin ext
public abstract void collapse(int ord, int contextDoc, int globalDoc) throws IOException;
public abstract void setNextReader(AtomicReaderContext context) throws IOException;
- public FieldValueCollapse(SolrIndexSearcher searcher,
+ public FieldValueCollapse(int maxDoc,
String field,
int nullPolicy,
boolean max,
@@ -626,7 +628,7 @@ public class CollapsingQParserPlugin ext
this.nullPolicy = nullPolicy;
this.max = max;
this.needsScores = needsScores;
- this.collapsedSet = new OpenBitSet(searcher.maxDoc());
+ this.collapsedSet = new OpenBitSet(maxDoc);
this.boostDocs = boostDocs;
if(this.boostDocs != null) {
Iterator<IntCursor> it = boostDocs.iterator();
@@ -676,14 +678,14 @@ public class CollapsingQParserPlugin ext
private int nullVal;
private int[] ordVals;
- public IntValueCollapse(SolrIndexSearcher searcher,
+ public IntValueCollapse(int maxDoc,
String field,
int nullPolicy,
int[] ords,
boolean max,
boolean needsScores,
IntOpenHashSet boostDocs) throws IOException {
- super(searcher, field, nullPolicy, max, needsScores, boostDocs);
+ super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
this.ords = ords;
this.ordVals = new int[ords.length];
Arrays.fill(ords, -1);
@@ -745,14 +747,13 @@ public class CollapsingQParserPlugin ext
private long nullVal;
private long[] ordVals;
- public LongValueCollapse(SolrIndexSearcher searcher,
- String field,
+ public LongValueCollapse(int maxDoc, String field,
int nullPolicy,
int[] ords,
boolean max,
boolean needsScores,
IntOpenHashSet boostDocs) throws IOException {
- super(searcher, field, nullPolicy, max, needsScores, boostDocs);
+ super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
this.ords = ords;
this.ordVals = new long[ords.length];
Arrays.fill(ords, -1);
@@ -814,14 +815,14 @@ public class CollapsingQParserPlugin ext
private float nullVal;
private float[] ordVals;
- public FloatValueCollapse(SolrIndexSearcher searcher,
+ public FloatValueCollapse(int maxDoc,
String field,
int nullPolicy,
int[] ords,
boolean max,
boolean needsScores,
IntOpenHashSet boostDocs) throws IOException {
- super(searcher, field, nullPolicy, max, needsScores, boostDocs);
+ super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
this.ords = ords;
this.ordVals = new float[ords.length];
Arrays.fill(ords, -1);
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java Thu Dec 19 17:48:47 2013
@@ -86,9 +86,9 @@ class OldLuceneQParser extends LuceneQPa
public SortSpec getSort(boolean useGlobal) throws SyntaxError {
SortSpec sort = super.getSort(useGlobal);
if (sortStr != null && sortStr.length()>0 && sort.getSort()==null) {
- Sort oldSort = QueryParsing.parseSort(sortStr, getReq());
- if( oldSort != null ) {
- sort.sort = oldSort;
+ SortSpec oldSort = QueryParsing.parseSortSpec(sortStr, getReq());
+ if( oldSort.getSort() != null ) {
+ sort.setSortAndFields(oldSort.getSort(), oldSort.getSchemaFields());
}
}
return sort;
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QParser.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QParser.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QParser.java Thu Dec 19 17:48:47 2013
@@ -276,11 +276,11 @@ public abstract class QParser {
int start = startS != null ? Integer.parseInt(startS) : 0;
int rows = rowsS != null ? Integer.parseInt(rowsS) : 10;
- Sort sort = null;
- if( sortStr != null ) {
- sort = QueryParsing.parseSort(sortStr, req);
- }
- return new SortSpec( sort, start, rows );
+ SortSpec sort = QueryParsing.parseSortSpec(sortStr, req);
+
+ sort.setOffset(start);
+ sort.setCount(rows);
+ return sort;
}
public String[] getDefaultHighlightFields() {
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QParserPlugin.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QParserPlugin.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QParserPlugin.java Thu Dec 19 17:48:47 2013
@@ -52,7 +52,8 @@ public abstract class QParserPlugin impl
MaxScoreQParserPlugin.NAME, MaxScoreQParserPlugin.class,
BlockJoinParentQParserPlugin.NAME, BlockJoinParentQParserPlugin.class,
BlockJoinChildQParserPlugin.NAME, BlockJoinChildQParserPlugin.class,
- CollapsingQParserPlugin.NAME, CollapsingQParserPlugin.class
+ CollapsingQParserPlugin.NAME, CollapsingQParserPlugin.class,
+ SimpleQParserPlugin.NAME, SimpleQParserPlugin.class
};
/** return a {@link QParser} */
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QueryParsing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QueryParsing.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QueryParsing.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/QueryParsing.java Thu Dec 19 17:48:47 2013
@@ -43,6 +43,7 @@ import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import java.io.IOException;
+import java.util.Collections;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -219,16 +220,24 @@ public class QueryParsing {
return new MapSolrParams(localParams);
}
+ /**
+ * Returns the Sort object represented by the string, or null if default sort
+ * by score descending should be used.
+ * @see #parseSortSpec
+ * @deprecated use {@link #parseSortSpec}
+ */
+ @Deprecated
+ public static Sort parseSort(String sortSpec, SolrQueryRequest req) {
+ return parseSortSpec(sortSpec, req).getSort();
+ }
/**
- * Returns null if the sortSpec is the standard sort desc.
- * <p/>
* <p>
* The form of the sort specification string currently parsed is:
* </p>
* <pre>
* SortSpec ::= SingleSort [, SingleSort]*
- * SingleSort ::= <fieldname> SortDirection
+ * SingleSort ::= <fieldname|function> SortDirection
* SortDirection ::= top | desc | bottom | asc
* </pre>
* Examples:
@@ -239,10 +248,15 @@ public class QueryParsing {
* height desc,weight desc #sort by height descending, and use weight descending to break any ties
* height desc,weight asc #sort by height descending, using weight ascending as a tiebreaker
* </pre>
+ * @return a SortSpec object populated with the appropriate Sort (which may be null if
+ * default score sort is used) and SchemaFields (where applicable) using
+ * hardcoded default count & offset values.
*/
- public static Sort parseSort(String sortSpec, SolrQueryRequest req) {
- if (sortSpec == null || sortSpec.length() == 0) return null;
- List<SortField> lst = new ArrayList<SortField>(4);
+ public static SortSpec parseSortSpec(String sortSpec, SolrQueryRequest req) {
+ if (sortSpec == null || sortSpec.length() == 0) return newEmptySortSpec();
+
+ List<SortField> sorts = new ArrayList<SortField>(4);
+ List<SchemaField> fields = new ArrayList<SchemaField>(4);
try {
@@ -299,10 +313,11 @@ public class QueryParsing {
if (null != top) {
// we have a Query and a valid direction
if (q instanceof FunctionQuery) {
- lst.add(((FunctionQuery)q).getValueSource().getSortField(top));
+ sorts.add(((FunctionQuery)q).getValueSource().getSortField(top));
} else {
- lst.add((new QueryValueSource(q, 0.0f)).getSortField(top));
+ sorts.add((new QueryValueSource(q, 0.0f)).getSortField(top));
}
+ fields.add(null);
continue;
}
} catch (Exception e) {
@@ -327,12 +342,14 @@ public class QueryParsing {
if (SCORE.equals(field)) {
if (top) {
- lst.add(SortField.FIELD_SCORE);
+ sorts.add(SortField.FIELD_SCORE);
} else {
- lst.add(new SortField(null, SortField.Type.SCORE, true));
+ sorts.add(new SortField(null, SortField.Type.SCORE, true));
}
+ fields.add(null);
} else if (DOCID.equals(field)) {
- lst.add(new SortField(null, SortField.Type.DOC, top));
+ sorts.add(new SortField(null, SortField.Type.DOC, top));
+ fields.add(null);
} else {
// try to find the field
SchemaField sf = req.getSchema().getFieldOrNull(field);
@@ -348,7 +365,8 @@ public class QueryParsing {
(SolrException.ErrorCode.BAD_REQUEST,
"sort param field can't be found: " + field);
}
- lst.add(sf.getSortField(top));
+ sorts.add(sf.getSortField(top));
+ fields.add(sf);
}
}
@@ -358,13 +376,17 @@ public class QueryParsing {
// normalize a sort on score desc to null
- if (lst.size()==1 && lst.get(0) == SortField.FIELD_SCORE) {
- return null;
+ if (sorts.size()==1 && sorts.get(0) == SortField.FIELD_SCORE) {
+ return newEmptySortSpec();
}
- return new Sort(lst.toArray(new SortField[lst.size()]));
+ Sort s = new Sort(sorts.toArray(new SortField[sorts.size()]));
+ return new SortSpec(s, fields);
}
+ private static SortSpec newEmptySortSpec() {
+ return new SortSpec(null, Collections.<SchemaField>emptyList());
+ }
///////////////////////////
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Thu Dec 19 17:48:47 2013
@@ -863,6 +863,25 @@ public class SolrIndexSearcher extends I
}
};
+ private DocSet getDocSetScore(List<Query> queries) throws IOException {
+ Query main = queries.remove(0);
+ ProcessedFilter pf = getProcessedFilter(null, queries);
+ DocSetCollector setCollector = new DocSetCollector(maxDoc()>>6, maxDoc());
+ Collector collector = setCollector;
+ if (pf.postFilter != null) {
+ pf.postFilter.setLastDelegate(collector);
+ collector = pf.postFilter;
+ }
+
+ search(main, pf.filter, collector);
+
+ if(collector instanceof DelegatingCollector) {
+ ((DelegatingCollector) collector).finish();
+ }
+
+ DocSet docSet = setCollector.getDocSet();
+ return docSet;
+ }
/**
* Returns the set of document ids matching all queries.
@@ -873,6 +892,15 @@ public class SolrIndexSearcher extends I
* The DocSet returned should <b>not</b> be modified.
*/
public DocSet getDocSet(List<Query> queries) throws IOException {
+
+ if(queries != null) {
+ for(Query q : queries) {
+ if(q instanceof ScoreFilter) {
+ return getDocSetScore(queries);
+ }
+ }
+ }
+
ProcessedFilter pf = getProcessedFilter(null, queries);
if (pf.answer != null) return pf.answer;
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/SortSpec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/SortSpec.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/SortSpec.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/SortSpec.java Thu Dec 19 17:48:47 2013
@@ -20,29 +20,63 @@ package org.apache.solr.search;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
+import org.apache.solr.schema.SchemaField;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Collections;
/***
* SortSpec encapsulates a Lucene Sort and a count of the number of documents
* to return.
*/
public class SortSpec
{
- Sort sort;
- int num;
- int offset;
+ private Sort sort;
+ private List<SchemaField> fields;
+ private int num = 10;
+ private int offset = 0;
+
+ public SortSpec(Sort sort, List<SchemaField> fields) {
+ setSortAndFields(sort, fields);
+ }
+ public SortSpec(Sort sort, SchemaField[] fields) {
+ setSortAndFields(sort, Arrays.asList(fields));
+ }
+ /** @deprecated Specify both Sort and SchemaField[] when constructing */
+ @Deprecated
public SortSpec(Sort sort, int num) {
this(sort,0,num);
}
+ /** @deprecated Specify both Sort and SchemaField[] when constructing */
+ @Deprecated
public SortSpec(Sort sort, int offset, int num) {
- this.sort=sort;
+ setSort(sort);
this.offset=offset;
this.num=num;
}
+ /** @deprecated use {@link #setSortAndFields} */
+ @Deprecated
public void setSort( Sort s )
{
sort = s;
+ fields = Collections.unmodifiableList(Arrays.asList(new SchemaField[s.getSort().length]));
+ }
+
+ /**
+ * the specified SchemaFields must correspond one to one with the Sort's SortFields,
+ * using null where appropriate.
+ */
+ public void setSortAndFields( Sort s, List<SchemaField> fields )
+ {
+
+ assert null == s || s.getSort().length == fields.size()
+ : "SortFields and SchemaFields do not have equal lengths";
+ this.sort = s;
+ this.fields = Collections.unmodifiableList(fields);
}
public static boolean includesScore(Sort sort) {
@@ -64,9 +98,19 @@ public class SortSpec
public Sort getSort() { return sort; }
/**
+ * Gets the Solr SchemaFields that correspond to each of the SortFields used
+ * in this sort. The array may contain null if a SortField doesn't correspond directly
+ * to a SchemaField (ie: score, lucene docid, custom function sorting, etc...)
+ *
+ * @return an immutable list, may be empty if getSort is null
+ */
+ public List<SchemaField> getSchemaFields() { return fields; }
+
+ /**
* Offset into the list of results.
*/
public int getOffset() { return offset; }
+ public void setOffset(int offset) { this.offset = offset; }
/**
* Gets the number of documents to return after sorting.
@@ -74,6 +118,7 @@ public class SortSpec
* @return number of docs to return, or -1 for no cut off (just sort)
*/
public int getCount() { return num; }
+ public void setCount(int count) { this.num = count; }
@Override
public String toString() {