You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/06/20 16:15:11 UTC
svn commit: r1137636 - in /lucene/dev/branches/LUCENE2793: ./
dev-tools/idea/lucene/contrib/ lucene/
lucene/contrib/queries/src/test/org/apache/lucene/search/regex/
lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/
lucene/s...
Author: simonw
Date: Mon Jun 20 14:15:10 2011
New Revision: 1137636
URL: http://svn.apache.org/viewvc?rev=1137636&view=rev
Log:
catch up with trunk
Modified:
lucene/dev/branches/LUCENE2793/ (props changed)
lucene/dev/branches/LUCENE2793/dev-tools/idea/lucene/contrib/ (props changed)
lucene/dev/branches/LUCENE2793/lucene/ (props changed)
lucene/dev/branches/LUCENE2793/lucene/CHANGES.txt
lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/branches/LUCENE2793/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/Sort.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/SortField.java
lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java
lucene/dev/branches/LUCENE2793/solr/ (props changed)
lucene/dev/branches/LUCENE2793/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/LUCENE2793/solr/LICENSE.txt (props changed)
lucene/dev/branches/LUCENE2793/solr/NOTICE.txt (props changed)
lucene/dev/branches/LUCENE2793/solr/README.txt (props changed)
lucene/dev/branches/LUCENE2793/solr/build.xml (props changed)
lucene/dev/branches/LUCENE2793/solr/client/ (props changed)
lucene/dev/branches/LUCENE2793/solr/common-build.xml (props changed)
lucene/dev/branches/LUCENE2793/solr/contrib/ (props changed)
lucene/dev/branches/LUCENE2793/solr/example/ (props changed)
lucene/dev/branches/LUCENE2793/solr/lib/ (props changed)
lucene/dev/branches/LUCENE2793/solr/site/ (props changed)
lucene/dev/branches/LUCENE2793/solr/src/ (props changed)
lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/function/ValueSource.java
lucene/dev/branches/LUCENE2793/solr/src/test/org/apache/solr/search/QueryParsingTest.java
lucene/dev/branches/LUCENE2793/solr/testlogging.properties (props changed)
Modified: lucene/dev/branches/LUCENE2793/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/CHANGES.txt?rev=1137636&r1=1137635&r2=1137636&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/LUCENE2793/lucene/CHANGES.txt Mon Jun 20 14:15:10 2011
@@ -434,6 +434,10 @@ New features
need to lookup by that field or perform deletions against it, for
example in a near-real-time setting. (Mike McCandless)
+* SOLR-2533: Added support for rewriting Sort and SortFields using an
+ IndexSearcher. SortFields can have SortField.REWRITEABLE type which
+ requires they are rewritten before they are used. (Chris Male)
+
Optimizations
* LUCENE-2588: Don't store unnecessary suffixes when writing the terms
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/Sort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/Sort.java?rev=1137636&r1=1137635&r2=1137636&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/Sort.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/Sort.java Mon Jun 20 14:15:10 2011
@@ -17,6 +17,7 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import java.io.IOException;
import java.util.Arrays;
@@ -149,6 +150,30 @@ public class Sort {
return fields;
}
+ /**
+ * Rewrites the SortFields in this Sort, returning a new Sort if any of the fields
+ * changes during their rewriting.
+ *
+ * @param searcher IndexSearcher to use in the rewriting
+ * @return {@code this} if the Sort/Fields have not changed, or a new Sort if there
+ * is a change
+ * @throws IOException Can be thrown by the rewriting
+ * @lucene.experimental
+ */
+ public Sort rewrite(IndexSearcher searcher) throws IOException {
+ boolean changed = false;
+
+ SortField[] rewrittenSortFields = new SortField[fields.length];
+ for (int i = 0; i < fields.length; i++) {
+ rewrittenSortFields[i] = fields[i].rewrite(searcher);
+ if (fields[i] != rewrittenSortFields[i]) {
+ changed = true;
+ }
+ }
+
+ return (changed) ? new Sort(rewrittenSortFields) : this;
+ }
+
@Override
public String toString() {
StringBuilder buffer = new StringBuilder();
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/SortField.java?rev=1137636&r1=1137635&r2=1137636&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/SortField.java Mon Jun 20 14:15:10 2011
@@ -86,10 +86,14 @@ public class SortField {
* This is typically slower than {@link #STRING}, which
* uses ordinals to do the sorting. */
public static final int STRING_VAL = 11;
-
+
/** Sort use byte[] index values. */
public static final int BYTES = 12;
-
+
+ /** Force rewriting of SortField using {@link SortField#rewrite(IndexSearcher)}
+ * before it can be used for sorting */
+ public static final int REWRITEABLE = 13;
+
/** Represents sorting by document score (relevance). */
public static final SortField FIELD_SCORE = new SortField(null, SCORE);
@@ -475,9 +479,26 @@ public class SortField {
case SortField.STRING_VAL:
return new FieldComparator.TermValComparator(numHits, field);
+
+ case SortField.REWRITEABLE:
+ throw new IllegalStateException("SortField needs to be rewritten through Sort.rewrite(..) and SortField.rewrite(..)");
default:
throw new IllegalStateException("Illegal sort type: " + type);
}
}
+
+ /**
+ * Rewrites this SortField, returning a new SortField if a change is made.
+ * Subclasses should override this define their rewriting behavior when this
+ * SortField is of type {@link SortField#REWRITEABLE}
+ *
+ * @param searcher IndexSearcher to use during rewriting
+ * @return New rewritten SortField, or {@code this} if nothing has changed.
+ * @throws IOException Can be thrown by the rewriting
+ * @lucene.experimental
+ */
+ public SortField rewrite(IndexSearcher searcher) throws IOException {
+ return this;
+ }
}
Modified: lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java?rev=1137636&r1=1137635&r2=1137636&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java Mon Jun 20 14:15:10 2011
@@ -148,35 +148,23 @@ public class QueryUtils {
// we can't put deleted docs before the nested reader, because
// it will throw off the docIds
IndexReader[] readers = new IndexReader[] {
- edge < 0 ? r : emptyReaders[0],
- emptyReaders[0],
- new MultiReader(edge < 0 ? emptyReaders[4] : emptyReaders[0],
- emptyReaders[0],
- 0 == edge ? r : emptyReaders[0]),
- 0 < edge ? emptyReaders[0] : emptyReaders[7],
- emptyReaders[0],
- new MultiReader(0 < edge ? emptyReaders[0] : emptyReaders[5],
- emptyReaders[0],
- 0 < edge ? r : emptyReaders[0])
+ edge < 0 ? r : IndexReader.open(makeEmptyIndex(random, 0), true),
+ IndexReader.open(makeEmptyIndex(random, 0), true),
+ new MultiReader(IndexReader.open(makeEmptyIndex(random, edge < 0 ? 4 : 0), true),
+ IndexReader.open(makeEmptyIndex(random, 0), true),
+ 0 == edge ? r : IndexReader.open(makeEmptyIndex(random, 0), true)),
+ IndexReader.open(makeEmptyIndex(random, 0 < edge ? 0 : 7), true),
+ IndexReader.open(makeEmptyIndex(random, 0), true),
+ new MultiReader(IndexReader.open(makeEmptyIndex(random, 0 < edge ? 0 : 5), true),
+ IndexReader.open(makeEmptyIndex(random, 0), true),
+ 0 < edge ? r : IndexReader.open(makeEmptyIndex(random, 0), true))
};
IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers));
out.setSimilarityProvider(s.getSimilarityProvider());
return out;
}
-
- static final IndexReader[] emptyReaders = new IndexReader[8];
- static {
- try {
- emptyReaders[0] = makeEmptyIndex(new Random(0), 0);
- emptyReaders[4] = makeEmptyIndex(new Random(0), 4);
- emptyReaders[5] = makeEmptyIndex(new Random(0), 5);
- emptyReaders[7] = makeEmptyIndex(new Random(0), 7);
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- }
- }
- private static IndexReader makeEmptyIndex(Random random, final int numDeletedDocs)
+ private static Directory makeEmptyIndex(Random random, final int numDeletedDocs)
throws IOException {
Directory d = new MockDirectoryWrapper(random, new RAMDirectory());
IndexWriter w = new IndexWriter(d, new IndexWriterConfig(
@@ -200,7 +188,8 @@ public class QueryUtils {
IndexReader r = IndexReader.open(d, true);
Assert.assertEquals("reader has wrong number of deleted docs",
numDeletedDocs, r.numDeletedDocs());
- return r;
+ r.close();
+ return d;
}
/** alternate scorer skipTo(),skipTo(),next(),next(),skipTo(),skipTo(), etc
Modified: lucene/dev/branches/LUCENE2793/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/solr/CHANGES.txt?rev=1137636&r1=1137635&r2=1137636&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/solr/CHANGES.txt (original)
+++ lucene/dev/branches/LUCENE2793/solr/CHANGES.txt Mon Jun 20 14:15:10 2011
@@ -145,6 +145,9 @@ New Features
* SOLR-2417: Add explain info directly to return documents using
?fl=id,[explain] (ryan)
+* SOLR-2533: Converted ValueSource.ValueSourceSortField over to new rewriteable Lucene
+ SortFields. ValueSourceSortField instances must be rewritten before they can be used.
+ This is done by SolrIndexSearcher when necessary. (Chris Male).
Optimizations
Modified: lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1137636&r1=1137635&r2=1137636&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java Mon Jun 20 14:15:10 2011
@@ -475,26 +475,7 @@ public class SolrIndexSearcher extends I
/** Returns a weighted sort according to this searcher */
public Sort weightSort(Sort sort) throws IOException {
- if (sort == null) return null;
- SortField[] sorts = sort.getSort();
-
- boolean needsWeighting = false;
- for (SortField sf : sorts) {
- if (sf instanceof SolrSortField) {
- needsWeighting = true;
- break;
- }
- }
- if (!needsWeighting) return sort;
-
- SortField[] newSorts = Arrays.copyOf(sorts, sorts.length);
- for (int i=0; i<newSorts.length; i++) {
- if (newSorts[i] instanceof SolrSortField) {
- newSorts[i] = ((SolrSortField)newSorts[i]).weight(this);
- }
- }
-
- return new Sort(newSorts);
+ return (sort != null) ? sort.rewrite(this) : null;
}
Modified: lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/function/ValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/function/ValueSource.java?rev=1137636&r1=1137635&r2=1137636&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/function/ValueSource.java (original)
+++ lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/function/ValueSource.java Mon Jun 20 14:15:10 2011
@@ -26,8 +26,6 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.Bits;
import org.apache.lucene.index.MultiFields;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.search.SolrSortField;
import java.io.IOException;
import java.io.Serializable;
@@ -102,20 +100,13 @@ public abstract class ValueSource implem
return new ValueSourceSortField(reverse);
}
- private static FieldComparatorSource dummyComparator = new FieldComparatorSource() {
- @Override
- public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unweighted use of sort " + fieldname);
- }
- };
-
- class ValueSourceSortField extends SortField implements SolrSortField {
+ class ValueSourceSortField extends SortField {
public ValueSourceSortField(boolean reverse) {
- super(description(), dummyComparator, reverse);
+ super(description(), SortField.REWRITEABLE, reverse);
}
@Override
- public SortField weight(IndexSearcher searcher) throws IOException {
+ public SortField rewrite(IndexSearcher searcher) throws IOException {
Map context = newContext(searcher);
createWeight(context, searcher);
return new SortField(getField(), new ValueSourceComparatorSource(context), getReverse());
Modified: lucene/dev/branches/LUCENE2793/solr/src/test/org/apache/solr/search/QueryParsingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/solr/src/test/org/apache/solr/search/QueryParsingTest.java?rev=1137636&r1=1137635&r2=1137636&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/solr/src/test/org/apache/solr/search/QueryParsingTest.java (original)
+++ lucene/dev/branches/LUCENE2793/solr/src/test/org/apache/solr/search/QueryParsingTest.java Mon Jun 20 14:15:10 2011
@@ -96,7 +96,7 @@ public class QueryParsingTest extends So
//test functions
sort = QueryParsing.parseSort("pow(weight, 2) desc", req);
flds = sort.getSort();
- assertEquals(flds[0].getType(), SortField.CUSTOM);
+ assertEquals(flds[0].getType(), SortField.REWRITEABLE);
//Not thrilled about the fragility of string matching here, but...
//the value sources get wrapped, so the out field is different than the input
assertEquals(flds[0].getField(), "pow(float(weight),const(2))");
@@ -104,12 +104,12 @@ public class QueryParsingTest extends So
//test functions (more deep)
sort = QueryParsing.parseSort("sum(product(r_f1,sum(d_f1,t_f1,1.0)),a_f1) asc", req);
flds = sort.getSort();
- assertEquals(flds[0].getType(), SortField.CUSTOM);
+ assertEquals(flds[0].getType(), SortField.REWRITEABLE);
assertEquals(flds[0].getField(), "sum(product(float(r_f1),sum(float(d_f1),float(t_f1),const(1.0))),float(a_f1))");
sort = QueryParsing.parseSort("pow(weight, 2.0) desc", req);
flds = sort.getSort();
- assertEquals(flds[0].getType(), SortField.CUSTOM);
+ assertEquals(flds[0].getType(), SortField.REWRITEABLE);
//Not thrilled about the fragility of string matching here, but...
//the value sources get wrapped, so the out field is different than the input
assertEquals(flds[0].getField(), "pow(float(weight),const(2.0))");
@@ -117,7 +117,7 @@ public class QueryParsingTest extends So
sort = QueryParsing.parseSort("pow(weight, 2.0) desc, weight desc, bday asc", req);
flds = sort.getSort();
- assertEquals(flds[0].getType(), SortField.CUSTOM);
+ assertEquals(flds[0].getType(), SortField.REWRITEABLE);
//Not thrilled about the fragility of string matching here, but...
//the value sources get wrapped, so the out field is different than the input
@@ -137,7 +137,7 @@ public class QueryParsingTest extends So
//Test literals in functions
sort = QueryParsing.parseSort("strdist(foo_s1, \"junk\", jw) desc", req);
flds = sort.getSort();
- assertEquals(flds[0].getType(), SortField.CUSTOM);
+ assertEquals(flds[0].getType(), SortField.REWRITEABLE);
//the value sources get wrapped, so the out field is different than the input
assertEquals(flds[0].getField(), "strdist(str(foo_s1),literal(junk), dist=org.apache.lucene.search.spell.JaroWinklerDistance)");