You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2011/06/28 19:00:21 UTC

svn commit: r1140724 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/backwards/ lucene/backwards/src/test-framework/ lucene/backwards/src/test/ solr/ solr/src/common/org/apache/solr/common/params/ solr/src/java/org/apache/solr/handler/component/...

Author: yonik
Date: Tue Jun 28 17:00:20 2011
New Revision: 1140724

URL: http://svn.apache.org/viewvc?rev=1140724&view=rev
Log:
SOLR-2429: ability not not cache filters and post filtering

Added:
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/DelegatingCollector.java
      - copied, changed from r1140252, lucene/dev/trunk/solr/src/java/org/apache/solr/search/DelegatingCollector.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/ExtendedQuery.java
      - copied unchanged from r1140252, lucene/dev/trunk/solr/src/java/org/apache/solr/search/ExtendedQuery.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/ExtendedQueryBase.java
      - copied, changed from r1140252, lucene/dev/trunk/solr/src/java/org/apache/solr/search/ExtendedQueryBase.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/PostFilter.java
      - copied unchanged from r1140252, lucene/dev/trunk/solr/src/java/org/apache/solr/search/PostFilter.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/WrappedQuery.java
      - copied, changed from r1140252, lucene/dev/trunk/solr/src/java/org/apache/solr/search/WrappedQuery.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSourceScorer.java
      - copied, changed from r1140252, lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ValueSourceScorer.java
    lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestFiltering.java
      - copied unchanged from r1140252, lucene/dev/trunk/solr/src/test/org/apache/solr/search/TestFiltering.java
Modified:
    lucene/dev/branches/branch_3x/   (props changed)
    lucene/dev/branches/branch_3x/lucene/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/src/test/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/   (props changed)
    lucene/dev/branches/branch_3x/solr/   (props changed)
    lucene/dev/branches/branch_3x/solr/CHANGES.txt
    lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common/params/CommonParams.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/component/QueryComponent.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/BoostQParserPlugin.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/DisMaxQParser.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/Grouping.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/QParser.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/QueryUtils.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/DocValues.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSource.java
    lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java
    lucene/dev/branches/branch_3x/solr/src/test-framework/org/apache/solr/SolrTestCaseJ4.java
    lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestQueryTypes.java
    lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestSearchPerf.java

Modified: lucene/dev/branches/branch_3x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/CHANGES.txt?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/solr/CHANGES.txt Tue Jun 28 17:00:20 2011
@@ -31,6 +31,15 @@ New Features
 * LUCENE-3234: add a new parameter hl.phraseLimit for FastVectorHighlighter speed up.
   (Mike Sokolov via koji)
 
+* SOLR-2429: Ability to add cache=false to queries and query filters to avoid
+  using the filterCache or queryCache.  A cost may also be specified and is used
+  to order the evaluation of non-cached filters from least to greatest cost .
+  For very expensive query filters (cost >= 100) if the query implements
+  the PostFilter interface, it will be used to obtain a Collector that is
+  checked only for documents that match the main query and all other filters.
+  The "frange" query now implements the PostFilter interface.  (yonik)
+
+
 Optimizations
 ----------------------
 

Modified: lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common/params/CommonParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common/params/CommonParams.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common/params/CommonParams.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common/params/CommonParams.java Tue Jun 28 17:00:20 2011
@@ -132,5 +132,16 @@ public interface CommonParams {
   public static final String VALUE = "v";
   public static final String TRUE = Boolean.TRUE.toString();
   public static final String FALSE = Boolean.FALSE.toString();
+
+  /** Used as a local parameter on queries.  cache=false means don't check any query or filter caches.
+   * cache=true is the default.
+   */
+  public static final String CACHE = "cache";
+
+  /** Used as a local param on filter queries in conjunction with cache=false.  Filters are checked in order, from
+   * smallest cost to largest. If cost>=100 and the query implements PostFilter, then that interface will be used to do post query filtering.
+   */
+  public static final String COST = "cost";
+
 }
 

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/component/QueryComponent.java Tue Jun 28 17:00:20 2011
@@ -162,6 +162,7 @@ public class QueryComponent extends Sear
       DocListAndSet res = new DocListAndSet();
       res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
       if (rb.isNeedDocSet()) {
+        // TODO: create a cache for this!
         List<Query> queries = new ArrayList<Query>();
         queries.add(rb.getQuery());
         List<Query> filters = rb.getFilters();

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/BoostQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/BoostQParserPlugin.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/BoostQParserPlugin.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/BoostQParserPlugin.java Tue Jun 28 17:00:20 2011
@@ -54,10 +54,10 @@ public class BoostQParserPlugin extends 
       public Query parse() throws ParseException {
         b = localParams.get(BOOSTFUNC);
         baseParser = subQuery(localParams.get(QueryParsing.V), null);
-        Query q = baseParser.parse();
+        Query q = baseParser.getQuery();
 
         if (b == null) return q;
-        Query bq = subQuery(b, FunctionQParserPlugin.NAME).parse();
+        Query bq = subQuery(b, FunctionQParserPlugin.NAME).getQuery();
         if (bq instanceof FunctionQuery) {
           vs = ((FunctionQuery)bq).getValueSource();
         } else {

Copied: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/DelegatingCollector.java (from r1140252, lucene/dev/trunk/solr/src/java/org/apache/solr/search/DelegatingCollector.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/DelegatingCollector.java?p2=lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/DelegatingCollector.java&p1=lucene/dev/trunk/solr/src/java/org/apache/solr/search/DelegatingCollector.java&r1=1140252&r2=1140724&rev=1140724&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/DelegatingCollector.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/DelegatingCollector.java Tue Jun 28 17:00:20 2011
@@ -31,7 +31,7 @@ public class DelegatingCollector extends
 
   protected Collector delegate;
   protected Scorer scorer;
-  protected IndexReader.AtomicReaderContext context;
+  protected IndexReader reader;
   protected int docBase;
 
   public Collector getDelegate() {
@@ -62,10 +62,10 @@ public class DelegatingCollector extends
   }
 
   @Override
-  public void setNextReader(IndexReader.AtomicReaderContext context) throws IOException {
-    this.context = context;
-    this.docBase = context.docBase;
-    delegate.setNextReader(context);
+  public void setNextReader(IndexReader reader, int docBase) throws IOException {
+    this.reader = reader;
+    this.docBase = docBase;
+    delegate.setNextReader(reader, docBase);
   }
 
   @Override

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/DisMaxQParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/DisMaxQParser.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/DisMaxQParser.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/DisMaxQParser.java Tue Jun 28 17:00:20 2011
@@ -89,7 +89,7 @@ public class DisMaxQParser extends QPars
         if (null == boostFunc || "".equals(boostFunc)) continue;
         Map<String, Float> ff = SolrPluginUtils.parseFieldBoosts(boostFunc);
         for (String f : ff.keySet()) {
-          Query fq = subQuery(f, FunctionQParserPlugin.NAME).parse();
+          Query fq = subQuery(f, FunctionQParserPlugin.NAME).getQuery();
           Float b = ff.get(f);
           if (null != b) {
             fq.setBoost(b);
@@ -108,7 +108,7 @@ public class DisMaxQParser extends QPars
       boostQueries = new ArrayList<Query>();
       for (String qs : boostParams) {
         if (qs.trim().length() == 0) continue;
-        Query q = subQuery(qs, null).parse();
+        Query q = subQuery(qs, null).getQuery();
         boostQueries.add(q);
       }
     }
@@ -173,7 +173,7 @@ public class DisMaxQParser extends QPars
     String altQ = solrParams.get(DisMaxParams.ALTQ);
     if (altQ != null) {
       QParser altQParser = subQuery(altQ, null);
-      return altQParser.parse();
+      return altQParser.getQuery();
     } else {
       throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing query string");
     }

Copied: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/ExtendedQueryBase.java (from r1140252, lucene/dev/trunk/solr/src/java/org/apache/solr/search/ExtendedQueryBase.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/ExtendedQueryBase.java?p2=lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/ExtendedQueryBase.java&p1=lucene/dev/trunk/solr/src/java/org/apache/solr/search/ExtendedQueryBase.java&r1=1140252&r2=1140724&rev=1140724&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/ExtendedQueryBase.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/ExtendedQueryBase.java Tue Jun 28 17:00:20 2011
@@ -24,27 +24,22 @@ public class ExtendedQueryBase extends Q
   private boolean cache = true;
   private boolean cacheSep;
 
-  @Override
   public void setCache(boolean cache) {
     this.cache = cache;
   }
 
-  @Override
   public boolean getCache() {
     return cache;
   }
 
-  @Override
   public void setCacheSep(boolean cacheSep) {
     this.cacheSep = cacheSep;
   }
 
-  @Override
   public boolean getCacheSep() {
     return cacheSep;
   }
 
-  @Override
   public void setCost(int cost) {
     this.cost = cost;
   }

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java Tue Jun 28 17:00:20 2011
@@ -16,13 +16,18 @@
  */
 package org.apache.solr.search;
 
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.search.Query;
+import org.apache.lucene.search.*;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrConfig;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.search.function.*;
 
+import java.io.IOException;
+import java.util.Map;
+
 /**
  * Create a range query over a function.
  * <br>Other parameters:
@@ -48,7 +53,7 @@ public class FunctionRangeQParserPlugin 
       @Override
       public Query parse() throws ParseException {
         funcStr = localParams.get(QueryParsing.V, null);
-        Query funcQ = subQuery(funcStr, FunctionQParserPlugin.NAME).parse();
+        Query funcQ = subQuery(funcStr, FunctionQParserPlugin.NAME).getQuery();
         if (funcQ instanceof FunctionQuery) {
           vs = ((FunctionQuery)funcQ).getValueSource();
         } else {
@@ -62,10 +67,50 @@ public class FunctionRangeQParserPlugin 
 
         // TODO: add a score=val option to allow score to be the value
         ValueSourceRangeFilter rf = new ValueSourceRangeFilter(vs, l, u, includeLower, includeUpper);
-        SolrConstantScoreQuery csq = new SolrConstantScoreQuery(rf);
-        return csq;
+        FunctionRangeQuery frq = new FunctionRangeQuery(rf);
+        return frq;
       }
     };
   }
 
 }
+
+// This class works as either a normal constant score query, or as a PostFilter using a collector
+class FunctionRangeQuery extends SolrConstantScoreQuery implements PostFilter {
+  final ValueSourceRangeFilter rangeFilt;
+
+  public FunctionRangeQuery(ValueSourceRangeFilter filter) {
+    super(filter);
+    this.rangeFilt = filter;
+  }
+
+  public DelegatingCollector getFilterCollector(IndexSearcher searcher) {
+    Map fcontext = ValueSource.newContext();
+    return new FunctionRangeCollector(fcontext);
+  }
+
+  class FunctionRangeCollector extends DelegatingCollector {
+    final Map fcontext;
+    ValueSourceScorer scorer;
+    int maxdoc;
+
+    public FunctionRangeCollector(Map fcontext) {
+      this.fcontext = fcontext;
+    }
+
+    @Override
+    public void collect(int doc) throws IOException {
+      if (doc<maxdoc && scorer.matches(doc)) {
+        delegate.collect(doc);
+      }
+    }
+
+    @Override
+    public void setNextReader(IndexReader reader, int docBase) throws IOException {
+      maxdoc = reader.maxDoc();
+      DocValues dv = rangeFilt.getValueSource().getValues(fcontext, reader);
+      scorer = dv.getRangeScorer(reader, rangeFilt.getLowerVal(), rangeFilt.getUpperVal(), rangeFilt.isIncludeLower(), rangeFilt.isIncludeUpper());
+      super.setNextReader(reader, docBase);
+    }
+  }
+}

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/Grouping.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/Grouping.java Tue Jun 28 17:00:20 2011
@@ -222,8 +222,8 @@ public class Grouping {
     DocListAndSet out = new DocListAndSet();
     qr.setDocListAndSet(out);
 
-    filter = cmd.getFilter() != null ? cmd.getFilter() : searcher.getDocSet(cmd.getFilterList());
-    luceneFilter = filter == null ? null : filter.getTopFilter();
+    SolrIndexSearcher.ProcessedFilter pf = searcher.getProcessedFilter(cmd.getFilter(), cmd.getFilterList());
+    final Filter luceneFilter = pf.filter;
     maxDoc = searcher.maxDoc();
 
     needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
@@ -275,6 +275,11 @@ public class Grouping {
       }
     }
 
+    if (pf.postFilter != null) {
+      pf.postFilter.setLastDelegate(allCollectors);
+      allCollectors = pf.postFilter;
+    }
+
     if (allCollectors != null) {
       searcher.search(query, luceneFilter, allCollectors);
     }
@@ -303,6 +308,10 @@ public class Grouping {
             searcher.search(query, luceneFilter, secondPhaseCollectors);
           }
         } else {
+          if (pf.postFilter != null) {
+            pf.postFilter.setLastDelegate(secondPhaseCollectors);
+            secondPhaseCollectors = pf.postFilter;
+          }
           searcher.search(query, luceneFilter, secondPhaseCollectors);
         }
       }

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/QParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/QParser.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/QParser.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/QParser.java Tue Jun 28 17:00:20 2011
@@ -140,10 +140,39 @@ public abstract class QParser {
   public Query getQuery() throws ParseException {
     if (query==null) {
       query=parse();
+
+      if (localParams != null) {
+        String cacheStr = localParams.get(CommonParams.CACHE);
+        if (cacheStr != null) {
+          if (CommonParams.FALSE.equals(cacheStr)) {
+            extendedQuery().setCache(false);
+          } else if (CommonParams.TRUE.equals(cacheStr)) {
+            extendedQuery().setCache(true);
+          } else if ("sep".equals(cacheStr)) {
+            extendedQuery().setCacheSep(true);
+          }
+        }
+
+        int cost = localParams.getInt(CommonParams.COST, Integer.MIN_VALUE);
+        if (cost != Integer.MIN_VALUE) {
+          extendedQuery().setCost(cost);
+        }
+      }
     }
     return query;
   }
 
+  // returns an extended query (and sets "query" to a new wrapped query if necessary)
+  private ExtendedQuery extendedQuery() {
+    if (query instanceof ExtendedQuery) {
+      return (ExtendedQuery)query;
+    } else {
+      WrappedQuery wq = new WrappedQuery(query);
+      query = wq;
+      return wq;
+    }
+  }
+
   private void checkRecurse() throws ParseException {
     if (recurseCount++ >= 100) {
       throw new ParseException("Infinite Recursion detected parsing query '" + qstr + "'");

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/QueryUtils.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/QueryUtils.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/QueryUtils.java Tue Jun 28 17:00:20 2011
@@ -52,6 +52,15 @@ public class QueryUtils {
    * @return
    */
   static Query getAbs(Query q) {
+    if (q instanceof WrappedQuery) {
+      Query subQ = ((WrappedQuery)q).getWrappedQuery();
+      Query absSubQ = getAbs(subQ);
+      if (absSubQ == subQ) return q;
+      WrappedQuery newQ = (WrappedQuery)q.clone();
+      newQ.setWrappedQuery(absSubQ);
+      return newQ;
+    }
+
     if (!(q instanceof BooleanQuery)) return q;
     BooleanQuery bq = (BooleanQuery)q;
 
@@ -87,6 +96,9 @@ public class QueryUtils {
    * lucene.
    */
   static Query makeQueryable(Query q) {
+    if (q instanceof WrappedQuery) {
+      return makeQueryable(((WrappedQuery)q).getWrappedQuery());
+    }
     return isNegative(q) ? fixNegativeQuery(q) : q;
   }
 

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java Tue Jun 28 17:00:20 2011
@@ -33,7 +33,9 @@ import java.util.Map;
  *
  * Experimental and subject to change.
  */
-public class SolrConstantScoreQuery extends ConstantScoreQuery {
+public class SolrConstantScoreQuery extends ConstantScoreQuery implements ExtendedQuery {
+  boolean cache = true;  // cache by default
+  int cost;
 
   public SolrConstantScoreQuery(Filter filter) {
     super(filter);
@@ -45,6 +47,30 @@ public class SolrConstantScoreQuery exte
     return filter;
   }
 
+  public void setCache(boolean cache) {
+    this.cache = cache;
+  }
+
+  public boolean getCache() {
+    return cache;
+  }
+
+  public void setCacheSep(boolean cacheSep) {
+  }
+
+  public boolean getCacheSep() {
+    return false;
+  }
+
+  public void setCost(int cost) {
+    this.cost = cost;
+  }
+
+  public int getCost() {
+    return cost;
+  }
+
+
   @Override
   public Query rewrite(IndexReader reader) throws IOException {
     return this;

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java Tue Jun 28 17:00:20 2011
@@ -572,6 +572,17 @@ public class SolrIndexSearcher extends I
    * The DocSet returned should <b>not</b> be modified.
    */
   public DocSet getDocSet(Query query) throws IOException {
+    if (query instanceof ExtendedQuery) {
+      ExtendedQuery eq = (ExtendedQuery)query;
+      if (!eq.getCache()) {
+        if (query instanceof WrappedQuery) {
+          query = ((WrappedQuery)query).getWrappedQuery();
+        }
+        query = QueryUtils.makeQueryable(query);
+        return getDocSetNC(query, null);
+      }
+    }
+
     // Get the absolute value (positive version) of this query.  If we
     // get back the same reference, we know it's positive.
     Query absQ = QueryUtils.getAbs(query);
@@ -604,7 +615,8 @@ public class SolrIndexSearcher extends I
       if (answer!=null) return answer;
     }
     answer = getDocSetNC(q,null);
-    if (filterCache != null) filterCache.put(q,answer);
+    if (filterCache != null) filterCache.put(
+        q,answer);
     return answer;
   }
 
@@ -623,6 +635,21 @@ public class SolrIndexSearcher extends I
 
   private static Query matchAllDocsQuery = new MatchAllDocsQuery();
 
+
+  static class ProcessedFilter {
+    DocSet answer;  // the answer, if non-null
+    Filter filter;
+    DelegatingCollector postFilter;
+  }
+
+
+  private static Comparator<Query> sortByCost = new Comparator<Query>() {
+    public int compare(Query q1, Query q2) {
+      return ((ExtendedQuery)q1).getCost() - ((ExtendedQuery)q2).getCost();
+    }
+  };
+
+
   /**
    * Returns the set of document ids matching all queries.
    * This method is cache-aware and attempts to retrieve the answer from the cache if possible.
@@ -632,123 +659,161 @@ public class SolrIndexSearcher extends I
    * The DocSet returned should <b>not</b> be modified.
    */
   public DocSet getDocSet(List<Query> queries) throws IOException {
-    if (queries==null) return null;
-    if (queries.size()==1) return getDocSet(queries.get(0));
-    DocSet answer=null;
+    ProcessedFilter pf = getProcessedFilter(null, queries);
+    if (pf.answer != null) return pf.answer;
 
-    boolean[] neg = new boolean[queries.size()];
-    DocSet[] sets = new DocSet[queries.size()];
 
-    int smallestIndex = -1;
-    int smallestCount = Integer.MAX_VALUE;
-    for (int i=0; i<sets.length; i++) {
-      Query q = queries.get(i);
-      Query posQuery = QueryUtils.getAbs(q);
-      sets[i] = getPositiveDocSet(posQuery);
-      // Negative query if absolute value different from original
-      if (q==posQuery) {
-        neg[i] = false;
-        // keep track of the smallest positive set.
-        // This optimization is only worth it if size() is cached, which it would
-        // be if we don't do any set operations.
-        int sz = sets[i].size();
-        if (sz<smallestCount) {
-          smallestCount=sz;
-          smallestIndex=i;
-          answer = sets[i];
+    DocSetCollector setCollector = new DocSetCollector(maxDoc()>>6, maxDoc());
+    Collector collector = setCollector;
+    if (pf.postFilter != null) {
+      pf.postFilter.setLastDelegate(collector);
+      collector = pf.postFilter;
+    }
+
+    final SolrIndexReader[] leaves = reader.getLeafReaders();
+    final int offsets[] = reader.getLeafOffsets();
+
+    for (int i=0; i<leaves.length; i++) {
+      SolrIndexReader subReader = leaves[i];
+      int baseDoc = offsets[i];
+      DocIdSet idSet = null;
+      if (pf.filter != null) {
+        idSet = pf.filter.getDocIdSet(subReader);
+        if (idSet == null) continue;
+      }
+      DocIdSetIterator idIter = null;
+      if (idSet != null) {
+        idIter = idSet.iterator();
+        if (idIter == null) continue;
+      }
+
+      collector.setNextReader(subReader, baseDoc);
+      int max = subReader.maxDoc();
+
+      if (idIter == null) {
+        TermDocs tdocs = subReader.termDocs(null);
+        while (tdocs.next()) {
+          int docid = tdocs.doc();
+          collector.collect(docid);
         }
       } else {
-        neg[i] = true;
+        for (int docid = -1; (docid = idIter.advance(docid+1)) < max; ) {
+          if (subReader.isDeleted(docid)) continue;
+          collector.collect(docid);
+        }
       }
     }
 
-    // if no positive queries, start off with all docs
-    if (answer==null) answer = getPositiveDocSet(matchAllDocsQuery);
-
-    // do negative queries first to shrink set size
-    for (int i=0; i<sets.length; i++) {
-      if (neg[i]) answer = answer.andNot(sets[i]);
-    }
-
-    for (int i=0; i<sets.length; i++) {
-      if (!neg[i] && i!=smallestIndex) answer = answer.intersection(sets[i]);
-    }
-
-    return answer;
+    return setCollector.getDocSet();
   }
 
-  Filter getFilter(Query q) throws IOException {
-    if (q == null) return null;
-    // TODO: support pure negative queries?
 
-    // if (q instanceof) {
-    // }
-
-    return getDocSet(q).getTopFilter();
-  }
-
-
-  Filter getFilter(DocSet setFilter, List<Query> queries) throws IOException {
-    Filter answer = setFilter == null ? null : setFilter.getTopFilter();
-
-    if (queries == null || queries.size() == 0) {
-      return answer;
+  public ProcessedFilter getProcessedFilter(DocSet setFilter, List<Query> queries) throws IOException {
+    ProcessedFilter pf = new ProcessedFilter();
+    if (queries==null || queries.size()==0) {
+      if (setFilter != null)
+        pf.filter = setFilter.getTopFilter();
+      return pf;
     }
 
-    if (answer == null && queries.size() == 1) {
-      return getFilter(queries.get(0));  
-    }
+    DocSet answer=null;
 
+    boolean[] neg = new boolean[queries.size()+1];
+    DocSet[] sets = new DocSet[queries.size()+1];
+    List<Query> notCached = null;
+    List<Query> postFilters = null;
 
-    DocSet finalSet=null;
+    int end = 0;
+    int smallestIndex = -1;
 
-    int nDocSets =0;
-    boolean[] neg = new boolean[queries.size()];
-    DocSet[] sets = new DocSet[queries.size()];
-    Query[] nocache = new Query[queries.size()];
+    if (setFilter != null) {
+      answer = sets[end++] = setFilter;
+      smallestIndex = end;
+    }
 
-    int smallestIndex = -1;
     int smallestCount = Integer.MAX_VALUE;
     for (Query q : queries) {
-      // if (q instanceof)
-
+      if (q instanceof ExtendedQuery) {
+        ExtendedQuery eq = (ExtendedQuery)q;
+        if (!eq.getCache()) {
+          if (eq.getCost() >= 100 && eq instanceof PostFilter) {
+            if (postFilters == null) postFilters = new ArrayList<Query>(sets.length-end);
+            postFilters.add(q);
+          } else {
+            if (notCached == null) notCached = new ArrayList<Query>(sets.length-end);
+            notCached.add(q);
+          }
+          continue;
+        }
+      }
 
       Query posQuery = QueryUtils.getAbs(q);
-      sets[nDocSets] = getPositiveDocSet(posQuery);
+      sets[end] = getPositiveDocSet(posQuery);
       // Negative query if absolute value different from original
       if (q==posQuery) {
-        neg[nDocSets] = false;
+        neg[end] = false;
         // keep track of the smallest positive set.
         // This optimization is only worth it if size() is cached, which it would
         // be if we don't do any set operations.
-        int sz = sets[nDocSets].size();
+        int sz = sets[end].size();
         if (sz<smallestCount) {
           smallestCount=sz;
-          smallestIndex=nDocSets;
-          finalSet = sets[nDocSets];
+          smallestIndex=end;
+          answer = sets[end];
         }
       } else {
-        neg[nDocSets] = true;
+        neg[end] = true;
       }
 
-      nDocSets++;
+      end++;
     }
 
-    // if no positive queries, start off with all docs
-    if (finalSet==null) finalSet = getPositiveDocSet(matchAllDocsQuery);
+    // Are all of our normal cached filters negative?
+    if (end > 0 && answer==null) {
+      answer = getPositiveDocSet(matchAllDocsQuery);
+    }
 
     // do negative queries first to shrink set size
-    for (int i=0; i<sets.length; i++) {
-      if (neg[i]) finalSet = finalSet.andNot(sets[i]);
+    for (int i=0; i<end; i++) {
+      if (neg[i]) answer = answer.andNot(sets[i]);
     }
 
-    for (int i=0; i<sets.length; i++) {
-      if (!neg[i] && i!=smallestIndex) finalSet = finalSet.intersection(sets[i]);
+    for (int i=0; i<end; i++) {
+      if (!neg[i] && i!=smallestIndex) answer = answer.intersection(sets[i]);
     }
 
-    return finalSet.getTopFilter();
+    if (notCached != null) {
+      Collections.sort(notCached, sortByCost);
+      List<Weight> weights = new ArrayList<Weight>(notCached.size());
+      for (Query q : notCached) {
+        Query qq = QueryUtils.makeQueryable(q);
+        weights.add(createNormalizedWeight(qq));
+      }
+      pf.filter = new FilterImpl(answer, weights);
+    } else {
+      if (postFilters == null) {
+        if (answer == null) {
+          answer = getPositiveDocSet(matchAllDocsQuery);
+        }
+        // "answer" is the only part of the filter, so set it.
+        pf.answer = answer;
+      }
 
+      if (answer != null) {
+        pf.filter = answer.getTopFilter();
+      }
+    }
+
+    if (postFilters != null) {
+      Collections.sort(postFilters, sortByCost);
+      for (int i=postFilters.size()-1; i>=0; i--) {
+        DelegatingCollector prev = pf.postFilter;
+        pf.postFilter = ((PostFilter)postFilters.get(i)).getFilterCollector(this);
+        if (prev != null) pf.postFilter.setDelegate(prev);
+      }
+    }
 
+    return pf;
   }
 
   // query must be positive
@@ -852,6 +917,17 @@ public class SolrIndexSearcher extends I
   public DocSet getDocSet(Query query, DocSet filter) throws IOException {
     if (filter==null) return getDocSet(query);
 
+    if (query instanceof ExtendedQuery) {
+      ExtendedQuery eq = (ExtendedQuery)query;
+      if (!eq.getCache()) {
+        if (query instanceof WrappedQuery) {
+          query = ((WrappedQuery)query).getWrappedQuery();
+        }
+        query = QueryUtils.makeQueryable(query);
+        return getDocSetNC(query, filter);
+      }
+    }
+
     // Negative query if absolute value different from original
     Query absQ = QueryUtils.getAbs(query);
     boolean positive = absQ==query;
@@ -947,8 +1023,9 @@ public class SolrIndexSearcher extends I
   }
 
   private static final int NO_CHECK_QCACHE       = 0x80000000;
-  public static final int GET_DOCSET            = 0x40000000;
+  public static final int GET_DOCSET             = 0x40000000;
   private static final int NO_CHECK_FILTERCACHE  = 0x20000000;
+  private static final int NO_SET_QCACHE         = 0x10000000;
 
   public static final int GET_DOCLIST           =        0x02;
   public static final int GET_SCORES             =       0x01;
@@ -966,21 +1043,33 @@ public class SolrIndexSearcher extends I
     // check for overflow, and check for # docs in index
     if (maxDocRequested < 0 || maxDocRequested > maxDoc()) maxDocRequested = maxDoc();
     int supersetMaxDoc= maxDocRequested;
-    DocList superset;
+    DocList superset = null;
+
+    int flags = cmd.getFlags();
+    Query q = cmd.getQuery();
+    if (q instanceof ExtendedQuery) {
+      ExtendedQuery eq = (ExtendedQuery)q;
+      if (!eq.getCache()) {
+        flags |= (NO_CHECK_QCACHE | NO_SET_QCACHE | NO_CHECK_FILTERCACHE);
+      }
+    }
+
 
     // we can try and look up the complete query in the cache.
     // we can't do that if filter!=null though (we don't want to
     // do hashCode() and equals() for a big DocSet).
-    if (queryResultCache != null && cmd.getFilter()==null) {
+    if (queryResultCache != null && cmd.getFilter()==null
+        && (flags & (NO_CHECK_QCACHE|NO_SET_QCACHE)) != ((NO_CHECK_QCACHE|NO_SET_QCACHE)))
+    {
         // all of the current flags can be reused during warming,
         // so set all of them on the cache key.
-        key = new QueryResultKey(cmd.getQuery(), cmd.getFilterList(), cmd.getSort(), cmd.getFlags());
-        if ((cmd.getFlags() & NO_CHECK_QCACHE)==0) {
+        key = new QueryResultKey(q, cmd.getFilterList(), cmd.getSort(), flags);
+        if ((flags & NO_CHECK_QCACHE)==0) {
           superset = queryResultCache.get(key);
 
           if (superset != null) {
             // check that the cache entry has scores recorded if we need them
-            if ((cmd.getFlags() & GET_SCORES)==0 || superset.hasScores()) {
+            if ((flags & GET_SCORES)==0 || superset.hasScores()) {
               // NOTE: subset() returns null if the DocList has fewer docs than
               // requested
               out.docList = superset.subset(cmd.getOffset(),cmd.getLen());
@@ -990,12 +1079,11 @@ public class SolrIndexSearcher extends I
             // found the docList in the cache... now check if we need the docset too.
             // OPT: possible future optimization - if the doclist contains all the matches,
             // use it to make the docset instead of rerunning the query.
-            if (out.docSet==null && ((cmd.getFlags() & GET_DOCSET)!=0) ) {
+            if (out.docSet==null && ((flags & GET_DOCSET)!=0) ) {
               if (cmd.getFilterList()==null) {
                 out.docSet = getDocSet(cmd.getQuery());
               } else {
-                List<Query> newList = new ArrayList<Query>(cmd.getFilterList()
-.size()+1);
+                List<Query> newList = new ArrayList<Query>(cmd.getFilterList().size()+1);
                 newList.add(cmd.getQuery());
                 newList.addAll(cmd.getFilterList());
                 out.docSet = getDocSet(newList);
@@ -1005,9 +1093,10 @@ public class SolrIndexSearcher extends I
           }
         }
 
-        // If we are going to generate the result, bump up to the
-        // next resultWindowSize for better caching.
+      // If we are going to generate the result, bump up to the
+      // next resultWindowSize for better caching.
 
+      if ((flags & NO_SET_QCACHE) == 0) {
         // handle 0 special case as well as avoid idiv in the common case.
         if (maxDocRequested < queryResultWindowSize) {
           supersetMaxDoc=queryResultWindowSize;
@@ -1015,6 +1104,9 @@ public class SolrIndexSearcher extends I
           supersetMaxDoc = ((maxDocRequested -1)/queryResultWindowSize + 1)*queryResultWindowSize;
           if (supersetMaxDoc < 0) supersetMaxDoc=maxDocRequested;
         }
+      } else {
+        key = null;  // we won't be caching the result
+      }
     }
 
 
@@ -1027,7 +1119,7 @@ public class SolrIndexSearcher extends I
 
     // check if we should try and use the filter cache
     boolean useFilterCache=false;
-    if ((cmd.getFlags() & (GET_SCORES|NO_CHECK_FILTERCACHE))==0 && useFilterForSortedQuery && cmd.getSort() != null && filterCache != null) {
+    if ((flags & (GET_SCORES|NO_CHECK_FILTERCACHE))==0 && useFilterForSortedQuery && cmd.getSort() != null && filterCache != null) {
       useFilterCache=true;
       SortField[] sfields = cmd.getSort().getSort();
       for (SortField sf : sfields) {
@@ -1056,7 +1148,7 @@ public class SolrIndexSearcher extends I
     } else {
       // do it the normal way...
       cmd.setSupersetMaxDoc(supersetMaxDoc);
-      if ((cmd.getFlags() & GET_DOCSET)!=0) {
+      if ((flags & GET_DOCSET)!=0) {
         // this currently conflates returning the docset for the base query vs
         // the base query and all filters.
         DocSet qDocSet = getDocListAndSetNC(qr,cmd);
@@ -1066,8 +1158,10 @@ public class SolrIndexSearcher extends I
         getDocListNC(qr,cmd);
         //Parameters: cmd.getQuery(),theFilt,cmd.getSort(),0,supersetMaxDoc,cmd.getFlags(),cmd.getTimeAllowed(),responseHeader);
       }
-      superset = out.docList;
-      out.docList = superset.subset(cmd.getOffset(),cmd.getLen());
+      if (key != null) {
+        superset = out.docList;
+        out.docList = superset.subset(cmd.getOffset(),cmd.getLen());
+      }
     }
 
     // lastly, put the superset in the cache if the size is less than or equal
@@ -1080,9 +1174,6 @@ public class SolrIndexSearcher extends I
 
 
   private void getDocListNC(QueryResult qr,QueryCommand cmd) throws IOException {
-    //Parameters: cmd.getQuery(),theFilt,cmd.getSort(),0,supersetMaxDoc,cmd.getFlags(),cmd.getTimeAllowed(),responseHeader);
-    //Query query, DocSet filter, Sort lsort, int offset, int len, int flags, long timeAllowed, NamedList<Object> responseHeader
-    DocSet filter = cmd.getFilter()!=null ? cmd.getFilter() : getDocSet(cmd.getFilterList());
     final long timeAllowed = cmd.getTimeAllowed();
     int len = cmd.getSupersetMaxDoc();
     int last = len;
@@ -1098,7 +1189,8 @@ public class SolrIndexSearcher extends I
 
     Query query = QueryUtils.makeQueryable(cmd.getQuery());
 
-    final Filter luceneFilter = filter==null ? null : filter.getTopFilter();
+    ProcessedFilter pf = getProcessedFilter(cmd.getFilter(), cmd.getFilterList());
+    final Filter luceneFilter = pf.filter;
 
     // handle zero case...
     if (lastDocRequested<=0) {
@@ -1150,6 +1242,11 @@ public class SolrIndexSearcher extends I
       if( timeAllowed > 0 ) {
         collector = new TimeLimitingCollector(collector, timeAllowed);
       }
+      if (pf.postFilter != null) {
+        pf.postFilter.setLastDelegate(collector);
+        collector = pf.postFilter;
+      }
+
       try {
         super.search(query, luceneFilter, collector);
       }
@@ -1174,6 +1271,10 @@ public class SolrIndexSearcher extends I
       if( timeAllowed > 0 ) {
         collector = new TimeLimitingCollector(collector, timeAllowed);
       }
+      if (pf.postFilter != null) {
+        pf.postFilter.setLastDelegate(collector);
+        collector = pf.postFilter;
+      }
       try {
         super.search(query, luceneFilter, collector);
       }
@@ -1206,7 +1307,6 @@ public class SolrIndexSearcher extends I
   // be cached if desired.
   private DocSet getDocListAndSetNC(QueryResult qr,QueryCommand cmd) throws IOException {
     int len = cmd.getSupersetMaxDoc();
-    DocSet filter = cmd.getFilter()!=null ? cmd.getFilter() : getDocSet(cmd.getFilterList());
     int last = len;
     if (last < 0 || last > maxDoc()) last=maxDoc();
     final int lastDocRequested = last;
@@ -1221,11 +1321,12 @@ public class SolrIndexSearcher extends I
     int maxDoc = maxDoc();
     int smallSetSize = maxDoc>>6;
 
+    ProcessedFilter pf = getProcessedFilter(cmd.getFilter(), cmd.getFilterList());
+    final Filter luceneFilter = pf.filter;
+
     Query query = QueryUtils.makeQueryable(cmd.getQuery());
     final long timeAllowed = cmd.getTimeAllowed();
 
-    final Filter luceneFilter = filter==null ? null : filter.getTopFilter();
-
     // handle zero case...
     if (lastDocRequested<=0) {
       final float[] topscore = new float[] { Float.NEGATIVE_INFINITY };
@@ -1260,6 +1361,11 @@ public class SolrIndexSearcher extends I
        if( timeAllowed > 0 ) {
          collector = new TimeLimitingCollector(collector, timeAllowed);
        }
+      if (pf.postFilter != null) {
+        pf.postFilter.setLastDelegate(collector);
+        collector = pf.postFilter;
+      }
+
        try {
          super.search(query, luceneFilter, collector);
        }
@@ -1291,6 +1397,10 @@ public class SolrIndexSearcher extends I
       if( timeAllowed > 0 ) {
         collector = new TimeLimitingCollector(collector, timeAllowed );
       }
+      if (pf.postFilter != null) {
+        pf.postFilter.setLastDelegate(collector);
+        collector = pf.postFilter;
+      }
       try {
         super.search(query, luceneFilter, collector);
       }
@@ -1327,7 +1437,7 @@ public class SolrIndexSearcher extends I
 
     // TODO: currently we don't generate the DocSet for the base query,
     // but the QueryDocSet == CompleteDocSet if filter==null.
-    return filter==null ? qr.getDocSet() : null;
+    return pf.filter==null && pf.postFilter==null ? qr.getDocSet() : null;
   }
 
 
@@ -1902,4 +2012,134 @@ public class SolrIndexSearcher extends I
 
 
 
+class FilterImpl extends Filter {
+  final DocSet filter;
+  final Filter topFilter;
+  final List<Weight> weights;
+
+  public FilterImpl(DocSet filter, List<Weight> weights) {
+    this.filter = filter;
+    this.weights = weights;
+    this.topFilter = filter == null ? null : filter.getTopFilter();
+  }
+
+  @Override
+  public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
+    DocIdSet sub = topFilter == null ? null : topFilter.getDocIdSet(reader);
+    if (weights.size() == 0) return sub;
+    return new FilterSet(sub, reader);
+  }
+
+  private class FilterSet extends DocIdSet {
+    DocIdSet docIdSet;
+    IndexReader reader;
+
+    public FilterSet(DocIdSet docIdSet, IndexReader reader) {
+      this.docIdSet = docIdSet;
+      this.reader = reader;
+    }
+
+    @Override
+    public DocIdSetIterator iterator() throws IOException {
+      List<DocIdSetIterator> iterators = new ArrayList<DocIdSetIterator>(weights.size()+1);
+      if (docIdSet != null) {
+        DocIdSetIterator iter = docIdSet.iterator();
+        if (iter == null) return null;
+        iterators.add(iter);
+      }
+      for (Weight w : weights) {
+        Scorer scorer = w.scorer(reader, true, false);
+        if (scorer == null) return null;
+        iterators.add(scorer);
+      }
+      if (iterators.size()==0) return null;
+      if (iterators.size()==1) return iterators.get(0);
+      if (iterators.size()==2) return new DualFilterIterator(iterators.get(0), iterators.get(1));
+      return new FilterIterator(iterators.toArray(new DocIdSetIterator[iterators.size()]));
+    }
+  }
+
+  private static class FilterIterator extends DocIdSetIterator {
+    final DocIdSetIterator[] iterators;
+    final DocIdSetIterator first;
+
+    public FilterIterator(DocIdSetIterator[] iterators) {
+      this.iterators = iterators;
+      this.first = iterators[0];
+    }
+
+    @Override
+    public int docID() {
+      return first.docID();
+    }
+
+    private int doNext(int doc) throws IOException {
+      int which=0;  // index of the iterator with the highest id
+      int i=1;
+      outer: for(;;) {
+        for (; i<iterators.length; i++) {
+          if (i == which) continue;
+          DocIdSetIterator iter = iterators[i];
+          int next = iter.advance(doc);
+          if (next != doc) {
+            doc = next;
+            which = i;
+            i = 0;
+            continue outer;
+          }
+        }
+        return doc;
+      }
+    }
+
+
+    @Override
+    public int nextDoc() throws IOException {
+      return doNext(first.nextDoc());
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+      return doNext(first.advance(target));
+    }
+  }
+
+  private static class DualFilterIterator extends DocIdSetIterator {
+    final DocIdSetIterator a;
+    final DocIdSetIterator b;
+
+    public DualFilterIterator(DocIdSetIterator a, DocIdSetIterator b) {
+      this.a = a;
+      this.b = b;
+    }
+
+    @Override
+    public int docID() {
+      return a.docID();
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+      int doc = a.nextDoc();
+      for(;;) {
+        int other = b.advance(doc);
+        if (other == doc) return doc;
+        doc = a.advance(other);
+        if (other == doc) return doc;
+      }
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+      int doc = a.advance(target);
+      for(;;) {
+        int other = b.advance(doc);
+        if (other == doc) return doc;
+        doc = a.advance(other);
+        if (other == doc) return doc;
+      }
+    }
+  }
+
+}
 

Copied: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/WrappedQuery.java (from r1140252, lucene/dev/trunk/solr/src/java/org/apache/solr/search/WrappedQuery.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/WrappedQuery.java?p2=lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/WrappedQuery.java&p1=lucene/dev/trunk/solr/src/java/org/apache/solr/search/WrappedQuery.java&r1=1140252&r2=1140724&rev=1140724&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/WrappedQuery.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/WrappedQuery.java Tue Jun 28 17:00:20 2011
@@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexRead
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Searcher;
 import org.apache.lucene.search.Weight;
 
 import java.io.IOException;
@@ -53,7 +54,7 @@ public class WrappedQuery extends Extend
   }
 
   @Override
-  public Weight createWeight(IndexSearcher searcher) throws IOException {
+  public Weight createWeight(Searcher searcher) throws IOException {
     return q.createWeight(searcher);
   }
 

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/DocValues.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/DocValues.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/DocValues.java Tue Jun 28 17:00:20 2011
@@ -66,7 +66,7 @@ public abstract class DocValues {
   // A RangeValueSource can't easily be a ValueSource that takes another ValueSource
   // because it needs different behavior depending on the type of fields.  There is also
   // a setup cost - parsing and normalizing params, and doing a binary search on the StringIndex.
-  
+  // TODO: change "reader" to AtomicReaderContext
   public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
     float lower;
     float upper;

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSource.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSource.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSource.java Tue Jun 28 17:00:20 2011
@@ -198,84 +198,4 @@ public abstract class ValueSource implem
 }
 
 
-class ValueSourceScorer extends Scorer {
-  protected IndexReader reader;
-  private int doc = -1;
-  protected final int maxDoc;
-  protected final DocValues values;
-  protected boolean checkDeletes;
-
-  protected ValueSourceScorer(IndexReader reader, DocValues values) {
-    super(null, null);
-    this.reader = reader;
-    this.maxDoc = reader.maxDoc();
-    this.values = values;
-    setCheckDeletes(true);
-  }
-
-  public IndexReader getReader() {
-    return reader;
-  }
-
-  public void setCheckDeletes(boolean checkDeletes) {
-    this.checkDeletes = checkDeletes && reader.hasDeletions();
-  }
-
-  public boolean matches(int doc) {
-    return (!checkDeletes || !reader.isDeleted(doc)) && matchesValue(doc);
-  }
-
-  public boolean matchesValue(int doc) {
-    return true;
-  }
-
-  @Override
-  public int docID() {
-    return doc;
-  }
-
-  @Override
-  public int nextDoc() throws IOException {
-    for (; ;) {
-      doc++;
-      if (doc >= maxDoc) return doc = NO_MORE_DOCS;
-      if (matches(doc)) return doc;
-    }
-  }
-
-  @Override
-  public int advance(int target) throws IOException {
-    // also works fine when target==NO_MORE_DOCS
-    doc = target - 1;
-    return nextDoc();
-  }
-
-  public int doc() {
-    return doc;
-  }
-
-  public boolean next() {
-    for (; ;) {
-      doc++;
-      if (doc >= maxDoc) return false;
-      if (matches(doc)) return true;
-    }
-  }
-
-  public boolean skipTo(int target) {
-    doc = target - 1;
-    return next();
-  }
-
-
-  @Override
-  public float score() throws IOException {
-    return values.floatVal(doc);
-  }
-
-  public Explanation explain(int doc) throws IOException {
-    return values.explain(doc);
-  }
-}
-
 

Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java Tue Jun 28 17:00:20 2011
@@ -49,6 +49,27 @@ public class ValueSourceRangeFilter exte
     this.includeUpper = upperVal != null && includeUpper;
   }
 
+  public ValueSource getValueSource() {
+    return valueSource;
+  }
+
+  public String getLowerVal() {
+    return lowerVal;
+  }
+
+  public String getUpperVal() {
+    return upperVal;
+  }
+
+  public boolean isIncludeLower() {
+    return includeLower;
+  }
+
+  public boolean isIncludeUpper() {
+    return includeUpper;
+  }
+
+
   @Override
   public DocIdSet getDocIdSet(final Map context, final IndexReader reader) throws IOException {
      return new DocIdSet() {

Copied: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSourceScorer.java (from r1140252, lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ValueSourceScorer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSourceScorer.java?p2=lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSourceScorer.java&p1=lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ValueSourceScorer.java&r1=1140252&r2=1140724&rev=1140724&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ValueSourceScorer.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/function/ValueSourceScorer.java Tue Jun 28 17:00:20 2011
@@ -18,27 +18,25 @@
 package org.apache.solr.search.function;
 
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.util.Bits;
 
 import java.io.IOException;
 
+/** @lucene.internal */
 public class ValueSourceScorer extends Scorer {
   protected IndexReader reader;
   private int doc = -1;
   protected final int maxDoc;
   protected final DocValues values;
   protected boolean checkDeletes;
-  private final Bits delDocs;
 
   protected ValueSourceScorer(IndexReader reader, DocValues values) {
-    super(null);
+    super(null, null);
     this.reader = reader;
     this.maxDoc = reader.maxDoc();
     this.values = values;
     setCheckDeletes(true);
-    this.delDocs = MultiFields.getDeletedDocs(reader);
   }
 
   public IndexReader getReader() {
@@ -50,7 +48,7 @@ public class ValueSourceScorer extends S
   }
 
   public boolean matches(int doc) {
-    return (!checkDeletes || !delDocs.get(doc)) && matchesValue(doc);
+    return (!checkDeletes || !reader.isDeleted(doc)) && matchesValue(doc);
   }
 
   public boolean matchesValue(int doc) {
@@ -78,8 +76,31 @@ public class ValueSourceScorer extends S
     return nextDoc();
   }
 
+  public int doc() {
+    return doc;
+  }
+
+  public boolean next() {
+    for (; ;) {
+      doc++;
+      if (doc >= maxDoc) return false;
+      if (matches(doc)) return true;
+    }
+  }
+
+  public boolean skipTo(int target) {
+    doc = target - 1;
+    return next();
+  }
+
+
   @Override
   public float score() throws IOException {
     return values.floatVal(doc);
   }
+
+  public Explanation explain(int doc) throws IOException {
+    return values.explain(doc);
+  }
 }
+

Modified: lucene/dev/branches/branch_3x/solr/src/test-framework/org/apache/solr/SolrTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/test-framework/org/apache/solr/SolrTestCaseJ4.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/test-framework/org/apache/solr/SolrTestCaseJ4.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/test-framework/org/apache/solr/SolrTestCaseJ4.java Tue Jun 28 17:00:20 2011
@@ -430,6 +430,7 @@ public abstract class SolrTestCaseJ4 ext
       }
 
       for (String test : tests) {
+        if (test == null || test.length()==0) continue;
         String testJSON = test.replace('\'', '"');
 
         try {

Modified: lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestQueryTypes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestQueryTypes.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestQueryTypes.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestQueryTypes.java Tue Jun 28 17:00:20 2011
@@ -117,6 +117,16 @@ public class TestQueryTypes extends Abst
       assertQ(req( "q", "{!frange v="+f+" l='"+v+"' u='"+v+"'}" )
               ,"//result[@numFound='1']"
               );
+
+       // frange as filter not cached
+      assertQ(req( "q","*:*", "fq", "{!frange cache=false v="+f+" l='"+v+"' u='"+v+"'}" )
+              ,"//result[@numFound='1']"
+              );
+
+       // frange as filter run after the main query
+      assertQ(req( "q","*:*", "fq", "{!frange cache=false cost=100 v="+f+" l='"+v+"' u='"+v+"'}" )
+              ,"//result[@numFound='1']"
+              );
       
       // function query... just make sure it doesn't throw an exception
       assertQ(req( "q", "+id:999 _val_:\"" + f + "\"")

Modified: lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestSearchPerf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestSearchPerf.java?rev=1140724&r1=1140723&r2=1140724&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestSearchPerf.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/search/TestSearchPerf.java Tue Jun 28 17:00:20 2011
@@ -202,10 +202,10 @@ public class TestSearchPerf extends Abst
 
     SolrQueryRequest req = lrf.makeRequest();
     QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", null, req);
-    Query range = parser.parse();
+    Query range = parser.getQuery();
                                      
     QParser parser2 = QParser.getParser("{!frange l="+l+" u="+u+"}foomany_s", null, req);
-    Query frange = parser2.parse();
+    Query frange = parser2.getQuery();
     req.close();
 
     createIndex2(indexSize,"foomany_s");
@@ -228,13 +228,13 @@ public class TestSearchPerf extends Abst
     SolrQueryRequest req = lrf.makeRequest();
 
     QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", null, req);
-    Query rangeQ = parser.parse();
+    Query rangeQ = parser.getQuery();
     List<Query> filters = new ArrayList<Query>();
     filters.add(rangeQ);
     req.close();
 
     parser = QParser.getParser("{!dismax qf=t10_100_ws pf=t10_100_ws ps=20}"+ t(0) + ' ' + t(1) + ' ' + t(2), null, req);
-    Query q= parser.parse();
+    Query q= parser.getQuery();
 
     // SolrIndexSearcher searcher = req.getSearcher();
     // DocSet range = searcher.getDocSet(rangeQ, null);