You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by yo...@apache.org on 2006/01/26 06:40:05 UTC

svn commit: r372455 [7/11] - in /incubator/solr/trunk: ./ src/ src/apps/ src/apps/SolarTest/ src/apps/SolarTest/src/ src/java/ src/java/org/ src/java/org/apache/ src/java/org/apache/solr/ src/java/org/apache/solr/analysis/ src/java/org/apache/solr/core...

Added: incubator/solr/trunk/src/java/org/apache/solr/search/QueryResultKey.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/search/QueryResultKey.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/search/QueryResultKey.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/search/QueryResultKey.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,107 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+
+import java.util.List;
+
+/** A hash key encapsulating a query, a list of filters, and a sort
+ * @author yonik
+ * @version $Id$
+ */
+public final class QueryResultKey {
+  final Query query;
+  final Sort sort; // change to normal Sort after Lucene 1.4.3
+  final SortField[] sfields;
+  final List<Query> filters;
+  final int nc_flags;  // non-comparable flags... ignored by hashCode and equals
+
+  private final int hc;  // cached hashCode
+
+  private static SortField[] defaultSort = new SortField[0];
+
+
+  public QueryResultKey(Query query, List<Query> filters, Sort sort, int nc_flags) {
+    this.query = query;
+    this.sort = sort;
+    this.filters = filters;
+    this.nc_flags = nc_flags;
+
+    int h = query.hashCode();
+
+    if (filters != null) h ^= filters.hashCode();
+
+    sfields = (this.sort !=null) ? this.sort.getSort() : defaultSort;
+    for (SortField sf : sfields) {
+      // mix the bits so that sortFields are position dependent
+      // so that a,b won't hash to the same value as b,a
+      h ^= (h << 8) | (h >>> 25);   // reversible hash
+
+      if (sf.getField() != null) h += sf.getField().hashCode();
+      h += sf.getType();
+      if (sf.getReverse()) h=~h;
+      if (sf.getLocale()!=null) h+=sf.getLocale().hashCode();
+      if (sf.getFactory()!=null) h+=sf.getFactory().hashCode();
+    }
+
+    hc = h;
+  }
+
+  public int hashCode() {
+    return hc;
+  }
+
+  public boolean equals(Object o) {
+    if (o==this) return true;
+    if (!(o instanceof QueryResultKey)) return false;
+    QueryResultKey other = (QueryResultKey)o;
+
+    // fast check of the whole hash code... most hash tables will only use
+    // some of the bits, so if this is a hash collision, it's still likely
+    // that the full cached hash code will be different.
+    if (this.hc != other.hc) return false;
+
+    // check for the thing most likely to be different (and the fastest things)
+    // first.
+    if (this.sfields.length != other.sfields.length) return false;
+    if (!this.query.equals(other.query)) return false;
+    if (!isEqual(this.filters, other.filters)) return false;
+
+    for (int i=0; i<sfields.length; i++) {
+      SortField sf1 = this.sfields[i];
+      SortField sf2 = other.sfields[i];
+      if (sf1.getType() != sf2.getType()) return false;
+      if (sf1.getReverse() != sf2.getReverse()) return false;
+      if (!isEqual(sf1.getField(),sf2.getField())) return false;
+      if (!isEqual(sf1.getLocale(), sf2.getLocale())) return false;
+      if (!isEqual(sf1.getFactory(), sf2.getFactory())) return false;
+      // NOTE: the factory must be identical!!! use singletons!
+    }
+
+    return true;
+  }
+
+
+  private static boolean isEqual(Object o1, Object o2) {
+    if (o1==o2) return true;  // takes care of identity and null cases
+    if (o1==null || o2==null) return false;
+    return o1.equals(o2);
+  }
+}

Propchange: incubator/solr/trunk/src/java/org/apache/solr/search/QueryResultKey.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/search/SolrCache.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/search/SolrCache.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/search/SolrCache.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/search/SolrCache.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,102 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.solr.core.SolrInfoMBean;
+
+import java.util.Map;
+import java.util.logging.Logger;
+import java.io.IOException;
+
+
+/**
+ *
+ * @author yonik
+ * @version $Id: SolrCache.java,v 1.6 2005/06/21 05:26:43 yonik Exp $
+ */
+public interface SolrCache extends SolrInfoMBean {
+  public final static Logger log = Logger.getLogger(SolrCache.class.getName());
+
+
+  /**
+   * The initialization routine.  Instance specific arguments are passed in
+   * the <code>args</code> map.
+   * <p>
+   * The persistence object will exist across different lifetimes of similar caches.
+   * For example, all filter caches will share the same persistence object, sometimes
+   * at the same time (it must be threadsafe).  If null is passed, then the cache
+   * implementation should create and return a new persistence object.  If not null,
+   * the passed in object should be returned again.
+   * <p>
+   * Since it will exist across the lifetime of many caches, care should be taken to
+   * not reference any particular cache instance and prevent it from being
+   * garbage collected (no using inner classes unless they are static).
+   * <p>
+   * Since the persistence object is designed to be used as a way for statistics
+   * to accumulate across all instances of the same type of cache, however the
+   * object may be of any type desired by the cache implementation.
+   * <p>
+   * The {@link CacheRegenerator} is what the cache uses during auto-warming to
+   * renenerate an item in the new cache from an entry in the old cache.
+   *
+   */
+  public Object init(Map args, Object persistence, CacheRegenerator regenerator);
+  // I don't think we need a factory for faster creation given that these
+  // will be associated with slow-to-create SolrIndexSearchers.
+  // change to NamedList when other plugins do?
+
+  // symbolic name for this cache
+  public String name();
+
+
+  // Should SolrCache just extend the java.util.Map interface?
+  // Following the conventions of the java.util.Map interface in any case.
+
+  public int size();
+
+  public Object put(Object key, Object value);
+
+  public Object get(Object key);
+
+  public void clear();
+
+
+  /**
+   * Set different cache states.
+   * The state a cache is in can have an effect on how statistics are kept.
+   * The cache user (SolrIndexSearcher) will take care of switching
+   * cache states.
+   */
+  public enum State { CREATED, STATICWARMING, AUTOWARMING, LIVE }
+  public void setState(State state);
+  public State getState();
+
+
+  /**
+   * Warm this cache associated with <code>searcher</code> using the <code>old</code>
+   * cache object.  <code>this</code> and <code>old</code> will have the same concrete type.
+   */
+  void warm(SolrIndexSearcher searcher, SolrCache old) throws IOException;
+  // Q: an alternative to passing the searcher here would be to pass it in
+  // init and have the cache implementation save it.
+
+
+  /** Frees any non-memory resources */
+  public void close();
+
+}
+

Propchange: incubator/solr/trunk/src/java/org/apache/solr/search/SolrCache.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,1163 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.PriorityQueue;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrInfoMBean;
+import org.apache.solr.core.SolrInfoRegistry;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.util.NamedList;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.*;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+
+/**
+ * SolrIndexSearcher adds schema awareness and caching functionality
+ * over the lucene IndexSearcher.
+
+ * @author yonik
+ * @version $Id: SolrIndexSearcher.java,v 1.49 2005/12/20 16:05:46 yonik Exp $
+ * @since solr 0.9
+ */
+
+// Since the internal reader in IndexSearcher is
+// package protected, I can't get to it by inheritance.
+// For now, I am using delgation and creating the
+// IndexReader to pass to the searcher myself.
+// NOTE: as of Lucene 1.9, this has changed!
+
+public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
+  private static Logger log = Logger.getLogger(SolrIndexSearcher.class.getName());
+
+  private final IndexSchema schema;
+
+  private final String name;
+  private final IndexSearcher searcher;
+  private final IndexReader reader;
+  private final boolean closeReader;
+
+  private final boolean cachingEnabled;
+  private final SolrCache filterCache;
+  private final SolrCache queryResultCache;
+  private final SolrCache documentCache;
+
+  // map of generic caches - not synchronized since it's read-only after the constructor.
+  private final HashMap<String, SolrCache> cacheMap;
+  private static final HashMap<String, SolrCache> noGenericCaches=new HashMap<String,SolrCache>(0);
+
+  // list of all caches associated with this searcher.
+  private final SolrCache[] cacheList;
+  private static final SolrCache[] noCaches = new SolrCache[0];
+
+    /** Creates a searcher searching the index in the named directory. */
+    /** Creates a searcher searching the index in the named directory. */
+  public SolrIndexSearcher(IndexSchema schema, String name, String path, boolean enableCache) throws IOException {
+    this(schema,name,IndexReader.open(path), true, enableCache);
+  }
+
+  /** Creates a searcher searching the index in the provided directory. */
+  public SolrIndexSearcher(IndexSchema schema, String name, Directory directory, boolean enableCache) throws IOException {
+    this(schema,name,IndexReader.open(directory), true, enableCache);
+  }
+
+  /** Creates a searcher searching the provided index. */
+  public SolrIndexSearcher(IndexSchema schema, String name, IndexReader r, boolean enableCache) {
+    this(schema,name,r, false, enableCache);
+  }
+
+  private SolrIndexSearcher(IndexSchema schema, String name, IndexReader r, boolean closeReader, boolean enableCache) {
+    this.schema = schema;
+    this.name = "Searcher@" + Integer.toHexString(hashCode()) + (name!=null ? " "+name : "");
+
+    log.info("Opening " + this.name);
+
+    reader = r;
+    searcher = new IndexSearcher(r);
+    this.closeReader = closeReader;
+    searcher.setSimilarity(schema.getSimilarity());
+
+    cachingEnabled=enableCache;
+    if (cachingEnabled) {
+      ArrayList<SolrCache> clist = new ArrayList<SolrCache>();
+      filterCache= filterCacheConfig==null ? null : filterCacheConfig.newInstance();
+      if (filterCache!=null) clist.add(filterCache);
+      queryResultCache = queryResultCacheConfig==null ? null : queryResultCacheConfig.newInstance();
+      if (queryResultCache!=null) clist.add(queryResultCache);
+      documentCache = documentCacheConfig==null ? null : documentCacheConfig.newInstance();
+      if (documentCache!=null) clist.add(documentCache);
+
+      if (userCacheConfigs == null) {
+        cacheMap = noGenericCaches;
+      } else {
+        cacheMap = new HashMap<String,SolrCache>(userCacheConfigs.length);
+        for (CacheConfig userCacheConfig : userCacheConfigs) {
+          SolrCache cache = null;
+          if (userCacheConfig != null) cache = userCacheConfig.newInstance();
+          if (cache != null) {
+            cacheMap.put(cache.name(), cache);
+            clist.add(cache);
+          }
+        }
+      }
+
+      cacheList = clist.toArray(new SolrCache[clist.size()]);
+    } else {
+      filterCache=null;
+      queryResultCache=null;
+      documentCache=null;
+      cacheMap = noGenericCaches;
+      cacheList= noCaches;
+    }
+  }
+
+
+  public String toString() {
+    return name;
+  }
+
+
+  /*** Register sub-objects such as caches
+   */
+  public void register() {
+    for (SolrCache cache : cacheList) {
+      cache.setState(SolrCache.State.LIVE);
+      SolrInfoRegistry.getRegistry().put(cache.name(), cache);
+    }
+  }
+
+
+  public void close() throws IOException {
+    if (cachingEnabled) {
+      StringBuilder sb = new StringBuilder();
+      sb.append("Closing ").append(name);
+      for (SolrCache cache : cacheList) {
+        sb.append("\n\t");
+        sb.append(cache);
+      }
+      log.info(sb.toString());
+    } else {
+      log.fine("Closing " + name);
+    }
+    try {
+      searcher.close();
+    }
+    finally {
+      if(closeReader) reader.close();
+      for (SolrCache cache : cacheList) {
+        cache.close();
+      }
+    }
+  }
+
+  public IndexReader getReader() { return reader; }
+  public IndexSchema getSchema() { return schema; }
+
+
+  // params for the "nutch" query optimizer
+  private static boolean filtOptEnabled=SolrConfig.config.getBool("query/boolTofilterOptimizer/@enabled",false);
+  private static int filtOptCacheSize=SolrConfig.config.getInt("query/boolTofilterOptimizer/@cacheSize",32);
+  private static float filtOptThreshold= SolrConfig.config.getFloat("query/boolTofilterOptimizer/@threshold",.05f);
+  private LuceneQueryOptimizer optimizer = filtOptEnabled ? new LuceneQueryOptimizer(filtOptCacheSize,filtOptThreshold) : null;
+
+
+  private static final CacheConfig filterCacheConfig = CacheConfig.getConfig("query/filterCache");
+  private static final CacheConfig queryResultCacheConfig = CacheConfig.getConfig("query/queryResultCache");
+  private static final CacheConfig documentCacheConfig = CacheConfig.getConfig("query/documentCache");
+  private static final CacheConfig[] userCacheConfigs = CacheConfig.getMultipleConfigs("query/cache");
+
+
+  //
+  // Set default regenerators on filter and query caches if they don't have any
+  //
+  static {
+    if (filterCacheConfig != null && filterCacheConfig.getRegenerator() == null) {
+      filterCacheConfig.setRegenerator(
+              new CacheRegenerator() {
+                public boolean regenerateItem(SolrIndexSearcher newSearcher, SolrCache newCache, SolrCache oldCache, Object oldKey, Object oldVal) throws IOException {
+                  newSearcher.cacheDocSet((Query)oldKey, null, false);
+                  return true;
+                }
+              }
+      );
+    }
+
+    if (queryResultCacheConfig != null && queryResultCacheConfig.getRegenerator() == null) {
+      queryResultCacheConfig.setRegenerator(
+              new CacheRegenerator() {
+                public boolean regenerateItem(SolrIndexSearcher newSearcher, SolrCache newCache, SolrCache oldCache, Object oldKey, Object oldVal) throws IOException {
+                  QueryResultKey key = (QueryResultKey)oldKey;
+                  int nDocs=1;
+                  // request 1 doc and let caching round up to the next window size...
+                  // unless the window size is <=1, in which case we will pick
+                  // the minimum of the number of documents requested last time and
+                  // a reasonable number such as 40.
+                  // TODO: make more configurable later...
+
+                  if (queryResultWindowSize<=1) {
+                    DocList oldList = (DocList)oldVal;
+                    int oldnDocs = oldList.offset() + oldList.size();
+                    // 40 has factors of 2,4,5,10,20
+                    nDocs = Math.min(oldnDocs,40);
+                  }
+
+                  DocListAndSet ret = new DocListAndSet();
+                  int flags=NO_CHECK_QCACHE | key.nc_flags;
+
+                  newSearcher.getDocListC(ret, key.query, key.filters, null, key.sort, 0, nDocs, flags);
+                  return true;
+                }
+              }
+      );
+    }
+  }
+
+
+  private static boolean useFilterForSortedQuery=SolrConfig.config.getBool("query/useFilterForSortedQuery", false);
+  private static int queryResultWindowSize=SolrConfig.config.getInt("query/queryResultWindowSize", 1);
+
+
+  public Hits search(Query query, Filter filter, Sort sort) throws IOException {
+    // todo - when SOLAR starts accepting filters, need to
+    // change this conditional check (filter!=null) and create a new filter
+    // that ANDs them together if it already exists.
+
+    if (optimizer==null || filter!=null || !(query instanceof BooleanQuery)
+    ) {
+      return searcher.search(query,filter,sort);
+    } else {
+      Query[] newQuery = new Query[1];
+      Filter[] newFilter = new Filter[1];
+      optimizer.optimize((BooleanQuery)query, searcher, 0, newQuery, newFilter);
+
+      // TODO REMOVE
+      if (newFilter[0]!=null) {
+        // System.out.println("OPTIMIZED QUERY: FILTER=" + newFilter[0]);
+      }
+
+      return searcher.search(newQuery[0], newFilter[0], sort);
+    }
+  }
+
+  /******
+   * Shouldn't be needed since IndexReader has it's own finalize method
+   * and there is nothing else to clean up here (for now at least)
+   *
+  protected void finalize() {
+    try {
+      close();
+      super.finalize();
+    } catch (Throwable e) {
+      SolrException.log(log,e);
+    }
+  }
+  ******/
+
+
+  public Hits search(Query query, Filter filter) throws IOException {
+    return searcher.search(query, filter);
+  }
+
+  public Hits search(Query query, Sort sort) throws IOException {
+    return searcher.search(query, sort);
+  }
+
+  /***  Replaced this one with one that does filter optimization
+  public Hits search(Query query, Filter filter, Sort sort) throws IOException {
+    return searcher.search(query, filter, sort);
+  }
+  ***/
+
+  public void search(Query query, HitCollector results) throws IOException {
+    searcher.search(query, results);
+  }
+
+  public void setSimilarity(Similarity similarity) {
+    searcher.setSimilarity(similarity);
+  }
+
+  public Similarity getSimilarity() {
+    return searcher.getSimilarity();
+  }
+
+  public int docFreq(Term term) throws IOException {
+    return searcher.docFreq(term);
+  }
+
+  public Document doc(int i) throws IOException {
+    Document d;
+    if (documentCache != null) {
+      d = (Document)documentCache.get(i);
+      if (d!=null) return d;
+    }
+
+    d = searcher.doc(i);
+
+    if (documentCache != null) {
+      documentCache.put(i,d);
+    }
+
+    return d;
+  }
+
+  public int maxDoc() throws IOException {
+    return searcher.maxDoc();
+  }
+
+  public TopDocs search(Weight weight, Filter filter, int i) throws IOException {
+    return searcher.search(weight, filter, i);
+  }
+
+  public TopDocs search(Query query, Filter filter, int nDocs) throws IOException {
+    return searcher.search(query, filter, nDocs);
+  }
+
+  public TopFieldDocs search(Query query, Filter filter, int nDocs, Sort sort) throws IOException {
+    return searcher.search(query, filter, nDocs, sort);
+  }
+
+  public void search(Weight weight, Filter filter, HitCollector hitCollector) throws IOException {
+    searcher.search(weight, filter, hitCollector);
+  }
+
+  public void search(Query query, Filter filter, HitCollector results) throws IOException {
+    searcher.search(query, filter, results);
+  }
+
+  public Query rewrite(Query original) throws IOException {
+    return searcher.rewrite(original);
+  }
+
+  public Explanation explain(Weight weight, int i) throws IOException {
+    return searcher.explain(weight, i);
+  }
+
+  public Explanation explain(Query query, int doc) throws IOException {
+    return searcher.explain(query, doc);
+  }
+
+  public TopFieldDocs search(Weight weight, Filter filter, int i, Sort sort) throws IOException {
+    return searcher.search(weight, filter, i, sort);
+  }
+
+  ////////////////////////////////////////////////////////////////////////////////
+  ////////////////////////////////////////////////////////////////////////////////
+  ////////////////////////////////////////////////////////////////////////////////
+
+  /**
+   * Returns the first document number containing the term <code>t</code>
+   * Returns -1 if no document was found.
+   * This method is primarily intended for clients that want to fetch
+   * documents using a unique identifier."
+   * @param t
+   * @return the first document number containing the term
+   */
+  public int getFirstMatch(Term t) throws IOException {
+    TermDocs tdocs = null;
+    try {
+      tdocs = reader.termDocs(t);
+      if (!tdocs.next()) return -1;
+      return tdocs.doc();
+    } finally {
+      if (tdocs!=null) tdocs.close();
+    }
+  }
+
+
+  /**
+   * Compute and cache the DocSet that matches a query.
+   * The normal usage is expected to be cacheDocSet(myQuery, null,false)
+   * meaning that Solr will determine if the Query warrants caching, and
+   * if so, will compute the DocSet that matches the Query and cache it.
+   * If the answer to the query is already cached, nothing further will be done.
+   * <p>
+   * If the optionalAnswer DocSet is provided, it should *not* be modified
+   * after this call.
+   *
+   * @param query           the lucene query that will act as the key
+   * @param optionalAnswer   the DocSet to be cached - if null, it will be computed.
+   * @param mustCache        if true, a best effort will be made to cache this entry.
+   *                         if false, heuristics may be used to determine if it should be cached.
+   */
+  public void cacheDocSet(Query query, DocSet optionalAnswer, boolean mustCache) throws IOException {
+    // Even if the cache is null, still compute the DocSet as it may serve to warm the Lucene
+    // or OS disk cache.
+    if (optionalAnswer != null) {
+      if (filterCache!=null) {
+        filterCache.put(query,optionalAnswer);
+      }
+      return;
+    }
+
+    // Throw away the result, relying on the fact that getDocSet
+    // will currently always cache what it found.  If getDocSet() starts
+    // using heuristics about what to cache, and mustCache==true, (or if we
+    // want this method to start using heuristics too) then
+    // this needs to change.
+    getDocSet(query);
+  }
+
+  /**
+   * Returns the set of document ids matching a query.
+   * This method is cache-aware and attempts to retrieve the answer from the cache if possible.
+   * If the answer was not cached, it may have been inserted into the cache as a result of this call.
+   * <p>
+   * The DocSet returned should <b>not</b> be modified.
+   */
+  public DocSet getDocSet(Query query) throws IOException {
+    DocSet answer;
+    if (filterCache != null) {
+      answer = (DocSet)filterCache.get(query);
+      if (answer!=null) return answer;
+    }
+
+    answer = getDocSetNC(query, null);
+
+    if (filterCache != null) {
+      filterCache.put(query, answer);
+    }
+
+    return answer;
+  }
+
+
+  // TODO: do a more efficient version that starts with the
+  // smallest DocSet and drives the intersection off that
+  // or implement an intersection() function that takes multiple
+  // DocSets (prob the better way)
+  protected DocSet getDocSet(List<Query> queries) throws IOException {
+    DocSet answer=null;
+    if (queries==null) return null;
+    for (Query q : queries) {
+      if (answer==null) {
+        answer = getDocSet(q);
+      } else {
+        answer = answer.intersection(getDocSet(q));
+      }
+    }
+    return answer;
+  }
+
+
+
+  protected DocSet getDocSetNC(Query query, DocSet filter) throws IOException {
+    SetHitCollector hc = new SetHitCollector(filter, maxDoc());
+    searcher.search(query, null, hc);
+    return hc.getDocSet();
+  }
+
+
+  /**
+   * Returns the set of document ids matching both the query and the filter.
+   * This method is cache-aware and attempts to retrieve the answer from the cache if possible.
+   * If the answer was not cached, it may have been inserted into the cache as a result of this call.
+   * <p>
+   * The DocSet returned should <b>not</b> be modified.
+   *
+   * @param query
+   * @param filter may be null
+   */
+  public DocSet getDocSet(Query query, DocSet filter) throws IOException {
+    if (filter==null) return getDocSet(query);
+
+    DocSet first;
+    if (filterCache != null) {
+      first = (DocSet)filterCache.get(query);
+      if (first==null) {
+        first = getDocSetNC(query,null);
+        filterCache.put(query,first);
+      }
+      return first.intersection(filter);
+    }
+
+
+    // If there isn't a cache, then do a single filtered query.
+    return getDocSetNC(query,filter);
+
+
+    /******* OLD VERSION that did a filtered query instead of
+     * an intersection if the query docset wasn't found in the cache.
+     * It made misses != inserts (even if no evictions)
+    DocSet first=null;
+    if (filterCache != null) {
+      first = (DocSet)filterCache.get(query);
+      if (first != null) {
+        return first.intersection(filter);
+      }
+    }
+
+    DocSet answer = getDocSetNC(query, filter);
+    // nothing is inserted into the cache, because we don't cache materialized filters.
+    // Hmmm, we *could* make a hitcollector that made a DocSet out of the query at the
+    // same time it was running the filter though...
+
+    // Q: we could call getDocSet(query) and then take the intersection instead of running
+    // the query as a filter.  Then it could be cached.
+    return answer;
+    ****************/
+  }
+
+
+  /**
+  * Converts a filter into a DocSet.
+  * This method is not cache-aware and no caches are checked.
+  */
+  public DocSet convertFilter(Filter lfilter) throws IOException {
+    return new BitDocSet(lfilter.bits(this.reader));
+  }
+
+  /**
+   * Returns documents matching both <code>query</code> and <code>filter</code>
+   * and sorted by <code>sort</code>.
+   * <p>
+   * This method is cache aware and may retrieve <code>filter</code> from
+   * the cache or make an insertion into the cache as a result of this call.
+   * <p>
+   * FUTURE: The returned DocList may be retrieved from a cache.
+   *
+   * The DocList returned should <b>not</b> be modified.
+   *
+   * @param query
+   * @param filter   may be null
+   * @param lsort    criteria by which to sort (if null, query relevance is used)
+   * @param offset   offset into the list of documents to return
+   * @param len      maximum number of documents to return
+   * @return
+   * @throws IOException
+   */
+  public DocList getDocList(Query query, Query filter, Sort lsort, int offset, int len) throws IOException {
+    List<Query> filterList = null;
+    if (filter != null) {
+      filterList = new ArrayList<Query>(1);
+      filterList.add(filter);
+    }
+    return getDocList(query, filterList, lsort, offset, len, 0);
+  }
+
+
+  public DocList getDocList(Query query, List<Query> filterList, Sort lsort, int offset, int len, int flags) throws IOException {
+    DocListAndSet answer = new DocListAndSet();
+    getDocListC(answer,query,filterList,null,lsort,offset,len,flags);
+    return answer.docList;
+  }
+
+
+  static final int NO_CHECK_QCACHE=0x80;
+  
+  public static final int GET_SCORES=0x01;
+  public static final int NO_CHECK_FILTERCACHE=0x02;
+
+  protected void getDocListC(DocListAndSet out, Query query, List<Query> filterList, DocSet filter, Sort lsort, int offset, int len, int flags) throws IOException {
+    QueryResultKey key=null;
+    int maxDoc = offset + len;
+    int supersetMaxDoc=maxDoc;
+    DocList superset;
+
+
+    // we can try and look up the complete query in the cache.
+    // we can't do that if filter!=null though (we don't want to
+    // do hashCode() and equals() for a big DocSet).
+    if (queryResultCache != null && filter==null) {
+        // all of the current flags can be reused during warming,
+        // so set all of them on the cache key.
+        key = new QueryResultKey(query, filterList, lsort, flags);
+        if ((flags & NO_CHECK_QCACHE)==0) {
+          superset = (DocList)queryResultCache.get(key);
+
+          if (superset != null) {
+            // check that the cache entry has scores recorded if we need them
+            if ((flags & GET_SCORES)==0 || superset.hasScores()) {
+              out.docList = superset.subset(offset,len);
+            }
+          }
+          if (out.docList != null) return;
+        }
+
+        // If we are going to generate the result, bump up to the
+        // next resultWindowSize for better caching.
+
+        // handle 0 special case as well as avoid idiv in the common case.
+        if (maxDoc < queryResultWindowSize) {
+          supersetMaxDoc=queryResultWindowSize;
+        } else {
+          supersetMaxDoc = ((maxDoc-1)/queryResultWindowSize + 1)*queryResultWindowSize;
+        }
+    }
+
+
+    // OK, so now we need to generate an answer.
+    // One way to do that would be to check if we have an unordered list
+    // of results for the base query.  If so, we can apply the filters and then
+    // sort by the resulting set.  This can only be used if:
+    // - the sort doesn't contain score
+    // - we don't want score returned.
+
+    // check if we should try and use the filter cache
+    boolean useFilterCache=false;
+    if ((flags & (GET_SCORES|NO_CHECK_FILTERCACHE))==0 && useFilterForSortedQuery && lsort != null && filterCache != null) {
+      useFilterCache=true;
+      SortField[] sfields = lsort.getSort();
+      for (SortField sf : sfields) {
+        if (sf.getType() == SortField.SCORE) {
+          useFilterCache=false;
+          break;
+        }
+      }
+    }
+
+    if (useFilterCache) {
+      // now actually use the filter cache.
+      // for large filters that match few documents, this may be
+      // slower than simply re-executing the query.
+      if (out.docSet == null) {
+        out.docSet = getDocSet(query,filter);
+        DocSet bigFilt = getDocSet(filterList);
+        if (bigFilt != null) out.docSet = out.docSet.intersection(bigFilt);
+      }
+      // todo: there could be a sortDocSet that could take a list of
+      // the filters instead of anding them first...
+      // perhaps there should be a multi-docset-iterator
+      superset = sortDocSet(out.docSet,lsort,supersetMaxDoc);
+      out.docList = superset.subset(offset,len);
+    } else {
+      // do it the normal way...
+      DocSet theFilt = filter!=null ? filter : getDocSet(filterList);
+      superset = getDocListNC(query,theFilt,lsort,0,supersetMaxDoc,flags);
+      // OPT... if getDocListNC can get the set at the same time (later version)
+      // then set it as out.docSet.
+      out.docList = superset.subset(offset,len);
+    }
+
+    // lastly, put the superset in the cache
+    if (key != null) {
+      queryResultCache.put(key, superset);
+    }
+  }
+
+
+
+  private DocList getDocListNC(Query query, DocSet filter, Sort lsort, int offset, int len, int flags) throws IOException {
+    final int lastDocRequested = offset+len;
+    int nDocsReturned;
+    int totalHits;
+    float maxScore;
+    int[] ids;
+    float[] scores;
+
+
+    // handle zero case...
+    if (lastDocRequested<=0) {
+      final DocSet filt = filter;
+      final float[] topscore = new float[] { Float.NEGATIVE_INFINITY };
+      final int[] numHits = new int[1];
+
+      searcher.search(query, new HitCollector() {
+        public void collect(int doc, float score) {
+          if (filt!=null && !filt.exists(doc)) return;
+          numHits[0]++;
+          if (score > topscore[0]) topscore[0]=score;
+        }
+      }
+      );
+
+      nDocsReturned=0;
+      ids = new int[nDocsReturned];
+      scores = new float[nDocsReturned];
+      totalHits = numHits[0];
+      maxScore = totalHits>0 ? topscore[0] : 0.0f;
+    } else if (lsort != null) {
+      // can't use TopDocs if there is a sort since it
+      // will do automatic score normalization.
+      // NOTE: this changed late in Lucene 1.9
+
+      final DocSet filt = filter;
+      final PublicFieldSortedHitQueue hq = new PublicFieldSortedHitQueue(reader, lsort.getSort(), offset+len);
+
+      searcher.search(query, new HitCollector() {
+        public void collect(int doc, float score) {
+          if (filt!=null && !filt.exists(doc)) return;
+          hq.insert(new FieldDoc(doc, score));
+        }
+      }
+      );
+
+      totalHits = hq.getTotalHits();
+      maxScore = totalHits>0 ? hq.getMaxScore() : 0.0f;
+
+      nDocsReturned = hq.size();
+      ids = new int[nDocsReturned];
+      scores = (flags&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
+      for (int i = nDocsReturned -1; i >= 0; i--) {
+        FieldDoc fieldDoc = (FieldDoc)hq.pop();
+        // fillFields is the point where score normalization happens
+        // hq.fillFields(fieldDoc)
+        ids[i] = fieldDoc.doc;
+        if (scores != null) scores[i] = fieldDoc.score;
+      }
+    } else {
+      // No Sort specified (sort by score descending)
+      // This case could be done with TopDocs, but would currently require
+      // getting a BitSet filter from a DocSet which may be inefficient.
+
+      final DocSet filt = filter;
+      final ScorePriorityQueue hq = new ScorePriorityQueue(lastDocRequested);
+      final int[] numHits = new int[1];
+      searcher.search(query, new HitCollector() {
+        float minScore=Float.NEGATIVE_INFINITY;  // minimum score in the priority queue
+        public void collect(int doc, float score) {
+          if (filt!=null && !filt.exists(doc)) return;
+          if (numHits[0]++ < lastDocRequested || score >= minScore) {
+            // if docs are always delivered in order, we could use "score>minScore"
+            // but might BooleanScorer14 might still be used and deliver docs out-of-order?
+            hq.insert(new ScoreDoc(doc, score));
+            minScore = ((ScoreDoc)hq.top()).score;
+          }
+        }
+      }
+      );
+
+      totalHits = numHits[0];
+      nDocsReturned = hq.size();
+      ids = new int[nDocsReturned];
+      scores = (flags&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
+      ScoreDoc sdoc =null;
+      for (int i = nDocsReturned -1; i >= 0; i--) {
+        sdoc = (ScoreDoc)hq.pop();
+        ids[i] = sdoc.doc;
+        if (scores != null) scores[i] = sdoc.score;
+      }
+      maxScore = sdoc ==null ? 0.0f : sdoc.score;
+    }
+
+
+    int sliceLen = Math.min(lastDocRequested,nDocsReturned) - offset;
+    if (sliceLen < 0) sliceLen=0;
+    return new DocSlice(offset,sliceLen,ids,scores,totalHits,maxScore);
+
+
+
+    /**************** older implementation using TopDocs *******************
+
+
+      Filter lfilter=null;
+      if (filter != null) {
+        final BitSet bits = filter.getBits();   // avoid if possible
+        lfilter = new Filter() {
+          public BitSet bits(IndexReader reader)  {
+            return bits;
+          }
+        };
+      }
+
+      int lastDocRequested=offset+len;
+
+      // lucene doesn't allow 0 to be passed for nDocs
+      if (lastDocRequested==0) lastDocRequested=1;
+
+      // TopFieldDocs sortedDocs;  // use TopDocs so both versions can use it
+      TopDocs sortedDocs;
+      if (lsort!=null) {
+         sortedDocs = searcher.search(query, lfilter, lastDocRequested, lsort);
+      } else {
+         sortedDocs = searcher.search(query, lfilter, lastDocRequested);
+      }
+
+      int nDocsReturned = sortedDocs.scoreDocs.length;
+      int[] docs = new int[nDocsReturned];
+      for (int i=0; i<nDocsReturned; i++) {
+        docs[i] = sortedDocs.scoreDocs[i].doc;
+      }
+      float[] scores=null;
+      float maxScore=0.0f;
+      if ((flags & GET_SCORES) != 0) {
+        scores = new float[nDocsReturned];
+        for (int i=0; i<nDocsReturned; i++) {
+          scores[i] = sortedDocs.scoreDocs[i].score;
+        }
+        if (nDocsReturned>0) {
+          maxScore=sortedDocs.scoreDocs[0].score;
+        }
+      }
+      int sliceLen = Math.min(offset+len,nDocsReturned) - offset;
+      if (sliceLen < 0) sliceLen=0;
+      return new DocSlice(offset,sliceLen,docs,scores,sortedDocs.totalHits, maxScore);
+
+    **********************************************************************************/
+
+  }
+
+
+
+
+  /**
+   * Returns documents matching both <code>query</code> and <code>filter</code>
+   * and sorted by <code>sort</code>.
+   * FUTURE: The returned DocList may be retrieved from a cache.
+   * <p>
+   * The DocList returned should <b>not</b> be modified.
+   *
+   * @param query
+   * @param filter   may be null
+   * @param lsort    criteria by which to sort (if null, query relevance is used)
+   * @param offset   offset into the list of documents to return
+   * @param len      maximum number of documents to return
+   * @return
+   * @throws IOException
+   */
+  public DocList getDocList(Query query, DocSet filter, Sort lsort, int offset, int len) throws IOException {
+    DocListAndSet answer = new DocListAndSet();
+    getDocListC(answer,query,null,filter,lsort,offset,len,0);
+    return answer.docList;
+  }
+
+  /**
+   * Returns documents matching both <code>query</code> and <code>filter</code>
+   * and sorted by <code>sort</code>.  Also returns the compete set of documents
+   * matching <code>query</code> and <code>filter</code> (regardless of <code>offset</code> and <code>len</code>).
+   * <p>
+   * This method is cache aware and may retrieve <code>filter</code> from
+   * the cache or make an insertion into the cache as a result of this call.
+   * <p>
+   * FUTURE: The returned DocList may be retrieved from a cache.
+   * <p>
+   * The DocList and DocSet returned should <b>not</b> be modified.
+   *
+   * @param query
+   * @param filter   may be null
+   * @param lsort    criteria by which to sort (if null, query relevance is used)
+   * @param offset   offset into the list of documents to return
+   * @param len      maximum number of documents to return
+   * @return
+   * @throws IOException
+   */
+  public DocListAndSet getDocListAndSet(Query query, Query filter, Sort lsort, int offset, int len) throws IOException {
+    List<Query> filterList = null;
+    if (filter != null) {
+      filterList = new ArrayList<Query>(2);
+      filterList.add(filter);
+    }
+    return getDocListAndSet(query, filterList, lsort, offset, len);
+
+  }
+
+
+  public DocListAndSet getDocListAndSet(Query query, List<Query> filterList, Sort lsort, int offset, int len) throws IOException {
+    DocListAndSet ret = new DocListAndSet();
+    getDocListC(ret,query,filterList,null,lsort,offset,len,0);
+    if (ret.docSet == null) {
+      List<Query> newList = new ArrayList<Query>(filterList.size()+1);
+      newList.add(query);
+      newList.addAll(filterList);
+      ret.docSet = getDocSet(newList);
+    }
+    return ret;
+  }
+
+
+
+  /**
+   * Returns documents matching both <code>query</code> and <code>filter</code>
+   * and sorted by <code>sort</code>. Also returns the compete set of documents
+   * matching <code>query</code> and <code>filter</code> (regardless of <code>offset</code> and <code>len</code>).
+   * <p>
+   * FUTURE: The returned DocList may be retrieved from a cache.
+   * <p>
+   * The DocList and DocSet returned should <b>not</b> be modified.
+   *
+   * @param query
+   * @param filter   may be null
+   * @param lsort    criteria by which to sort (if null, query relevance is used)
+   * @param offset   offset into the list of documents to return
+   * @param len      maximum number of documents to return
+   * @return
+   * @throws IOException
+   */
+  public DocListAndSet getDocListAndSet(Query query, DocSet filter, Sort lsort, int offset, int len) throws IOException {
+    DocListAndSet ret = new DocListAndSet();
+    getDocListC(ret,query,null,filter,lsort,offset,len,0);
+    if (ret.docSet == null) {
+      ret.docSet = getDocSet(query,filter);
+    }
+
+    // TODO: OPT: Hmmm, but if docList.size() == docList.matches() then
+    // we actually already have all the ids (the set)!  We could simply
+    // return the docList as the docSet also, or hash the ids into
+    // a HashDocSet, etc... no need to run the query again!
+    //
+
+    assert(ret.docList.matches() == ret.docSet.size());
+    return ret;
+  }
+
+
+  protected DocList sortDocSet(DocSet set, Sort sort, int nDocs) throws IOException {
+    final PublicFieldSortedHitQueue hq =
+            new PublicFieldSortedHitQueue(reader, sort.getSort(), nDocs);
+    DocIterator iter = set.iterator();
+    int hits=0;
+    while(iter.hasNext()) {
+      int doc = iter.nextDoc();
+      hits++;   // could just use set.size(), but that would be slower for a bitset
+      hq.insert(new FieldDoc(doc,1.0f));
+    }
+
+    int numCollected = hq.size();
+    int[] ids = new int[numCollected];
+    for (int i = numCollected-1; i >= 0; i--) {
+      FieldDoc fieldDoc = (FieldDoc)hq.pop();
+      // hq.fillFields(fieldDoc)  // optional, if we need that info
+      ids[i] = fieldDoc.doc;
+    }
+
+    return new DocSlice(0,numCollected,ids,null,hits,0.0f);
+  }
+
+
+
+  /**
+   * Returns the number of documents that match both <code>a</code> and <code>b</code>.
+   * <p>
+   * This method is cache-aware and may check as well as modify the cache.
+   *
+   * @param a
+   * @param b
+   * @return the numer of documents in the intersection between <code>a</code> and <code>b</code>.
+   * @throws IOException
+   */
+  public int numDocs(Query a, DocSet b) throws IOException {
+    // reverse: do the query on filter and filter using docs...
+    // or if filter is a term query, can get the freq and
+    // drive things off that.
+    //  this higher level API leaves open more optimization possibilities.
+    // prob only worth it if cacheHitRatio is bad...
+
+    return b.intersectionSize(getDocSet(a));
+  }
+
+   /**
+   * Returns the number of documents that match both <code>a</code> and <code>b</code>.
+   * <p>
+   * This method is cache-aware and may check as well as modify the cache.
+   *
+   * @param a
+   * @param b
+   * @return the numer of documents in the intersection between <code>a</code> and <code>b</code>.
+   * @throws IOException
+   */
+  public int numDocs(Query a, Query b) throws IOException {
+    return getDocSet(b).intersectionSize(getDocSet(a));
+  }
+
+
+  // Takes a list of docs (the doc ids actually), and returns all of
+  // the stored fields.
+  public Document[] readDocs(DocList ids) throws IOException {
+     Document[] docs = new Document[ids.size()];
+     readDocs(docs,ids);
+     return docs;
+  }
+
+  public void readDocs(Document[] docs, DocList ids) throws IOException {
+    DocIterator iter = ids.iterator();
+    for (int i=0; i<docs.length; i++) {
+      docs[i] = doc(iter.nextDoc());
+    }
+  }
+
+
+
+  /**
+   * Warm this searcher based on an old one (primarily for auto-cache warming).
+   */
+  public void warm(SolrIndexSearcher old) throws IOException {
+    // Make sure this is first!  filters can help queryResults execute!
+    boolean logme = log.isLoggable(Level.INFO);
+
+    // warm the caches in order...
+    for (int i=0; i<cacheList.length; i++) {
+      if (logme) log.info("autowarming " + this + " from " + old + "\n\t" + old.cacheList[i]);
+      this.cacheList[i].warm(this, old.cacheList[i]);
+      if (logme) log.info("autowarming result for " + this + "\n\t" + this.cacheList[i]);
+    }
+  }
+
+
+  /**
+   * return the named generic cache
+   */
+  public SolrCache getCache(String cacheName) {
+    return cacheMap.get(cacheName);
+  }
+
+  /**
+   * lookup an entry in a generic cache
+   */
+  public Object cacheLookup(String cacheName, Object key) {
+    SolrCache cache = cacheMap.get(cacheName);
+    return cache==null ? null : cache.get(key);
+  }
+
+  /**
+   * insert an entry in a generic cache
+   */
+  public Object cacheInsert(String cacheName, Object key, Object val) {
+    SolrCache cache = cacheMap.get(cacheName);
+    return cache==null ? null : cache.put(key,val);
+  }
+
+
+  /////////////////////////////////////////////////////////////////////
+  // SolrInfoMBean stuff: Statistics and Module Info
+  /////////////////////////////////////////////////////////////////////
+
+  public String getName() {
+    return SolrIndexSearcher.class.getName();
+  }
+
+  public String getVersion() {
+    return SolrCore.version;
+  }
+
+  public String getDescription() {
+    return "the searcher that handles all index queries";
+  }
+
+  public Category getCategory() {
+    return Category.CORE;
+  }
+
+  public String getCvsId() {
+    return "$Id: SolrIndexSearcher.java,v 1.49 2005/12/20 16:05:46 yonik Exp $";
+  }
+
+  public String getCvsName() {
+    return "$Name:  $";
+  }
+
+  public String getCvsSource() {
+    return "$Source: /cvs/main/searching/solr/solarcore/src/solr/search/SolrIndexSearcher.java,v $";
+  }
+
+  public URL[] getDocs() {
+    return null;
+  }
+
+  public NamedList getStatistics() {
+    /***
+    NamedList lst = new NamedList();
+    lst.add("requests", numRequests);
+    lst.add("errors", numErrors);
+    return lst;
+    ***/
+    return new NamedList();
+  }
+
+
+
+
+}
+
+
+
+
+
+// Todo: counting only hit collector (for speed comparison w/ caching filters)
+// todo: fast term query
+// todo: do a both hit collector that can get a DocList and DocSet at the same time
+
+final class SetHitCollector extends HitCollector {
+  int pos=0;
+  final DocSet filter;
+    // should we bother with filters at this point?
+    // how much faster would it be to take the check for
+    // filter!=null out of the loop??? depends on HotSpot... it may
+    // optimize it anyway.
+
+  final BitSet bits;
+
+  // in case there aren't that many hits, we may not want a very sparse
+  // bit array.  Optimistically collect the first few docs in an array
+  // in case there are only a few.
+  static final int ARRAY_COLLECT_SZ=HashDocSet.MAX_SIZE;
+  final int[] scratch = ARRAY_COLLECT_SZ>0 ? new int[ARRAY_COLLECT_SZ] : null;
+
+  public SetHitCollector(DocSet filter, int maxDoc) {
+    bits = new BitSet(maxDoc);
+    this.filter = filter;
+  }
+
+  public void collect(int doc, float score) {
+    if (filter!=null && !filter.exists(doc)) return;
+
+    // OPTIMIZATION: should I only set bits *after* I have run out of
+    // room in scratch?  (then at the end I could add all of the docs
+    // in scratch to the bitset.
+    bits.set(doc);
+
+    // optimistically collect the first docs in an array
+    // in case the total number will be small enough to represent
+    // as a HashDocSet() instead...
+    // It is assumed that storing in this array will be quicker to convert
+    // than scanning through a potentially huge bit vector.
+    // FUTURE: when search methods all start returning docs in order, maybe
+    // we could have a SortedListDocSet() and use the collected array directly.
+    if (pos < ARRAY_COLLECT_SZ) {
+      scratch[pos]=doc;
+    }
+
+    pos++;
+  }
+
+  public DocSet getDocSet() {
+    if (pos<=ARRAY_COLLECT_SZ) {
+      return new HashDocSet(scratch,0,pos);
+    }
+    return new BitDocSet(bits,pos);
+  }
+
+}
+
+
+
+// Lucene's HitQueue isn't public, so here is our own.
+final class ScorePriorityQueue extends PriorityQueue {
+  ScorePriorityQueue(int size) {
+    initialize(size);
+  }
+
+  protected final boolean lessThan(Object o1, Object o2) {
+    ScoreDoc sd1 = (ScoreDoc)o1;
+    ScoreDoc sd2 = (ScoreDoc)o2;
+    // use index order as a tiebreaker to make sorts stable
+    return sd1.score < sd2.score || (sd1.score==sd2.score && sd1.doc > sd2.doc);
+  }
+}
+
+
+
+

Propchange: incubator/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/search/SolrQueryParser.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/search/SolrQueryParser.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/search/SolrQueryParser.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/search/SolrQueryParser.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,81 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.search.*;
+import org.apache.lucene.index.Term;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.FieldType;
+
+// TODO: implement the analysis of simple fields with
+// FieldType.toInternal() instead of going through the
+// analyzer.  Should lead to faster query parsing.
+
+/**
+ * @author yonik
+ */
+public class SolrQueryParser extends QueryParser {
+  protected final IndexSchema schema;
+
+  public SolrQueryParser(IndexSchema schema) {
+    super(schema.getDefaultSearchFieldName(), schema.getQueryAnalyzer());
+    this.schema = schema;
+    setLowercaseExpandedTerms(false);
+  }
+
+  protected Query getFieldQuery(String field, String queryText) throws ParseException {
+    // intercept magic field name of "_" to use as a hook for our
+    // own functions.
+    if (field.equals("_val_")) {
+      return QueryParsing.parseFunction(queryText, schema);
+    }
+
+    // default to a normal field query
+    return super.getFieldQuery(field, queryText);
+  }
+
+  protected Query getRangeQuery(String field, String part1, String part2, boolean inclusive) throws ParseException {
+    FieldType ft = schema.getFieldType(field);
+    return new ConstantScoreRangeQuery(
+      field,
+      "*".equals(part1) ? null : ft.toInternal(part1),
+      "*".equals(part2) ? null : ft.toInternal(part2),
+      inclusive, inclusive);
+  }
+
+  protected Query getPrefixQuery(String field, String termStr) throws ParseException {
+    if (getLowercaseExpandedTerms()) {
+      termStr = termStr.toLowerCase();
+    }
+
+    // TODO: toInternal() won't necessarily work on partial
+    // values, so it looks like i need a getPrefix() function
+    // on fieldtype?  Or at the minimum, a method on fieldType
+    // that can tell me if I should lowercase or not...
+    // Schema could tell if lowercase filter is in the chain,
+    // but a more sure way would be to run something through
+    // the first time and check if it got lowercased.
+
+    // TODO: throw exception of field type doesn't support prefixes?
+    // (sortable numeric types don't do prefixes, but can do range queries)
+    Term t = new Term(field, termStr);
+    return new ConstantScorePrefixQuery(t);
+  }
+
+}

Propchange: incubator/solr/trunk/src/java/org/apache/solr/search/SolrQueryParser.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/search/SolrSimilarity.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/search/SolrSimilarity.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/search/SolrSimilarity.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/search/SolrSimilarity.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,37 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.search.DefaultSimilarity;
+
+import java.util.HashMap;
+
+/**
+ * @author yonik
+ */
+// don't make it public for now... easier to change later.
+
+// This class is currently unused.
+class SolrSimilarity extends DefaultSimilarity {
+  private final HashMap<String,Float> lengthNormConfig = new HashMap<String,Float>();
+
+  public float lengthNorm(String fieldName, int numTerms) {
+    // Float f = lengthNormConfig.
+    // if (lengthNormDisabled.)
+    return super.lengthNorm(fieldName, numTerms);
+  }
+}

Propchange: incubator/solr/trunk/src/java/org/apache/solr/search/SolrSimilarity.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/search/Sorting.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/search/Sorting.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/search/Sorting.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/search/Sorting.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,57 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.search.*;
+
+/**
+ * Extra lucene sorting utilities & convenience methods
+ *
+ * @author yonik
+ * @version $Id: Sorting.java,v 1.1 2005/06/02 04:43:06 yonik Exp $
+ *
+ */
+
+public class Sorting {
+
+
+  /** Returns a {@link SortField} for a string field.
+   *  If nullLast and nullFirst are both false, then default lucene string sorting is used where
+   *  null strings sort first in an ascending sort, and last in a descending sort.
+   *
+   * @param fieldName   the name of the field to sort on
+   * @param reverse     true for a reverse (desc) sort
+   * @param nullLast    true if null should come last, regardless of sort order
+   * @param nullFirst   true if null should come first, regardless of sort order
+   * @return SortField
+   */
+  public static SortField getStringSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
+    if (nullLast) {
+      if (!reverse) return new SortField(fieldName, nullStringLastComparatorSource);
+      else return new SortField(fieldName, SortField.STRING, true);
+    } else if (nullFirst) {
+      if (reverse) return new SortField(fieldName, nullStringLastComparatorSource);
+      else return new SortField(fieldName, SortField.STRING, false);
+    } else {
+      return new SortField(fieldName, SortField.STRING, reverse);
+    }
+  }
+
+
+  static final SortComparatorSource nullStringLastComparatorSource = new MissingStringLastComparatorSource();
+}
+

Propchange: incubator/solr/trunk/src/java/org/apache/solr/search/Sorting.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/search/test/TestDocSet.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/search/test/TestDocSet.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/search/test/TestDocSet.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/search/test/TestDocSet.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,180 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.test;
+
+import org.apache.solr.search.BitDocSet;
+import org.apache.solr.search.HashDocSet;
+import org.apache.solr.search.DocSet;
+
+import java.util.Random;
+import java.util.BitSet;
+
+/**
+ * @author yonik
+ */
+public class TestDocSet {
+
+  // use test instead of assert since asserts may be turned off
+  public static void test(boolean condition) {
+      if (!condition) {
+        throw new RuntimeException("test requestHandler: assertion failed!");
+      }
+  }
+
+  static Random rand = new Random();
+
+
+  static BitSet bs;
+  static BitDocSet bds;
+  static HashDocSet hds;
+  static int[] ids; // not unique
+
+  static void generate(int maxSize, int bitsToSet) {
+    bs = new BitSet(maxSize);
+    ids = new int[bitsToSet];
+    int count=0;
+    if (maxSize>0) {
+      for (int i=0; i<bitsToSet; i++) {
+        int id=rand.nextInt(maxSize);
+        if (!bs.get(id)) {
+          bs.set(id);
+          ids[count++]=id;
+        }
+      }
+    }
+    bds = new BitDocSet(bs,bitsToSet);
+    hds = new HashDocSet(ids,0,count);
+  }
+
+
+
+  public static void main(String[] args) {
+    String bsSize=args[0];
+    boolean randSize=false;
+
+    if (bsSize.endsWith("-")) {
+      bsSize=bsSize.substring(0,bsSize.length()-1);
+      randSize=true;
+    }
+
+    int bitSetSize = Integer.parseInt(bsSize);
+    int numSets = Integer.parseInt(args[1]);
+    int numBitsSet = Integer.parseInt(args[2]);
+    String test = args[3].intern();
+    int iter = Integer.parseInt(args[4]);
+
+    int ret=0;
+
+    BitSet[] sets = new BitSet[numSets];
+    DocSet[] bset = new DocSet[numSets];
+    DocSet[] hset = new DocSet[numSets];
+    BitSet scratch=new BitSet();
+
+    for (int i=0; i<numSets; i++) {
+      generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
+      sets[i] = bs;
+      bset[i] = bds;
+      hset[i] = hds;
+    }
+
+    long start = System.currentTimeMillis();
+
+    if ("test".equals(test)) {
+      for (int it=0; it<iter; it++) {
+        generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
+        BitSet bs1=bs;
+        BitDocSet bds1=bds;
+        HashDocSet hds1=hds;
+        generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
+
+        BitSet res = ((BitSet)bs1.clone());
+        res.and(bs);
+        int icount = res.cardinality();
+
+        test(bds1.intersection(bds).size() == icount);
+        test(bds1.intersectionSize(bds) == icount);
+        if (bds1.intersection(hds).size() != icount) {
+          DocSet ds = bds1.intersection(hds);
+          System.out.println("STOP");
+        }
+
+        test(bds1.intersection(hds).size() == icount);
+        test(bds1.intersectionSize(hds) == icount);
+        test(hds1.intersection(bds).size() == icount);
+        test(hds1.intersectionSize(bds) == icount);
+        test(hds1.intersection(hds).size() == icount);
+        test(hds1.intersectionSize(hds) == icount);
+
+        ret += icount;
+      }
+    }
+
+    String type=null;
+    String oper=null;
+
+    if (test.endsWith("B")) { type="B"; }
+    if (test.endsWith("H")) { type="H"; }
+    if (test.endsWith("M")) { type="M"; }
+    if (test.startsWith("intersect")) oper="intersect";
+    if (test.startsWith("intersectSize")) oper="intersectSize";
+    if (test.startsWith("intersectAndSize")) oper="intersectSize";
+
+
+    if (oper!=null) {
+      for (int it=0; it<iter; it++) {
+        int idx1 = rand.nextInt(numSets);
+        int idx2 = rand.nextInt(numSets);
+        DocSet a=null,b=null;
+
+        if (type=="B") {
+          a=bset[idx1]; b=bset[idx2];
+        } else if (type=="H") {
+          a=hset[idx1]; b=bset[idx2];
+        } else if (type=="M") {
+          if (idx1 < idx2) {
+            a=bset[idx1];
+            b=hset[idx2];
+          } else {
+            a=hset[idx1];
+            b=bset[idx2];
+          }
+        }
+
+        if (oper=="intersect") {
+          DocSet res = a.intersection(b);
+          ret += res.memSize();
+        } else if (oper=="intersectSize") {
+          ret += a.intersectionSize(b);
+        } else if (oper=="intersectAndSize") {
+          DocSet res = a.intersection(b);
+          ret += res.size();
+        }
+      }
+    }
+
+
+
+    long end = System.currentTimeMillis();
+    System.out.println("TIME="+(end-start));
+
+    // System.out.println("ret="+ret + " scratchsize="+scratch.size());
+    System.out.println("ret="+ret);
+  }
+
+
+
+}

Propchange: incubator/solr/trunk/src/java/org/apache/solr/search/test/TestDocSet.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/tst/OldRequestHandler.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/tst/OldRequestHandler.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/tst/OldRequestHandler.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/tst/OldRequestHandler.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,149 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.tst;
+
+import org.apache.lucene.search.*;
+import org.apache.lucene.document.Document;
+
+import java.util.List;
+import java.util.logging.Level;
+import java.io.IOException;
+import java.net.URL;
+
+import org.apache.solr.util.StrUtils;
+import org.apache.solr.util.NamedList;
+import org.apache.solr.search.DocSlice;
+import org.apache.solr.search.QueryParsing;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.request.SolrQueryResponse;
+import org.apache.solr.request.SolrQueryRequest;
+
+/**
+ * @author yonik
+ * @version $Id: OldRequestHandler.java,v 1.7 2005/12/02 04:31:05 yonik Exp $
+ */
+
+
+public class OldRequestHandler implements SolrRequestHandler {
+
+  long numRequests;
+  long numErrors;
+
+  public void init(NamedList args) {
+    SolrCore.log.log(Level.INFO, "Unused request handler arguments:" + args);
+  }
+
+
+  public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
+    numRequests++;
+
+    Query query = null;
+    Filter filter = null;
+
+    List<String> commands = StrUtils.splitSmart(req.getQueryString(),';');
+
+    String qs = commands.size() >= 1 ? commands.get(0) : "";
+    query = QueryParsing.parseQuery(qs, req.getSchema());
+
+    // If the first non-query, non-filter command is a simple sort on an indexed field, then
+    // we can use the Lucene sort ability.
+    Sort sort = null;
+    if (commands.size() >= 2) {
+      QueryParsing.SortSpec sortSpec = QueryParsing.parseSort(commands.get(1), req.getSchema());
+      if (sortSpec != null) {
+        sort = sortSpec.getSort();
+        // ignore the count for now... it's currently only controlled by start & limit on req
+        // count = sortSpec.getCount();
+      }
+    }
+
+    Hits hits=null;
+
+    try {
+      hits = req.getSearcher().search(query,filter,sort);
+
+      int numHits = hits.length();
+      int startRow = Math.min(numHits, req.getStart());
+      int endRow = Math.min(numHits,req.getStart()+req.getLimit());
+      int numRows = endRow-startRow;
+
+      int[] ids = new int[numRows];
+      Document[] data = new Document[numRows];
+      for (int i=startRow; i<endRow; i++) {
+        ids[i] = hits.id(i);
+        data[i] = hits.doc(i);
+      }
+
+      rsp.add(null, new DocSlice(0,numRows,ids,null,numHits,0.0f));
+
+      /***********************
+      rsp.setResults(new DocSlice(0,numRows,ids,null,numHits));
+
+      // Setting the actual document objects is optional
+      rsp.setResults(data);
+      ************************/
+    } catch (IOException e) {
+      rsp.setException(e);
+      numErrors++;
+      return;
+    }
+
+  }
+
+
+  public String getName() {
+    return OldRequestHandler.class.getName();
+  }
+
+  public String getVersion() {
+    return SolrCore.version;
+  }
+
+  public String getDescription() {
+    return "The original Hits based request handler";
+  }
+
+  public Category getCategory() {
+    return Category.QUERYHANDLER;
+  }
+
+  public String getCvsId() {
+    return "$Id: OldRequestHandler.java,v 1.7 2005/12/02 04:31:05 yonik Exp $";
+  }
+
+  public String getCvsName() {
+    return "$Name:  $";
+  }
+
+  public String getCvsSource() {
+    return "$Source: /cvs/main/searching/solr/solarcore/src/solr/tst/OldRequestHandler.java,v $";
+  }
+
+  public URL[] getDocs() {
+    return null;
+  }
+
+  public NamedList getStatistics() {
+    NamedList lst = new NamedList();
+    lst.add("requests", numRequests);
+    lst.add("errors", numErrors);
+    return lst;
+  }
+
+
+}

Propchange: incubator/solr/trunk/src/java/org/apache/solr/tst/OldRequestHandler.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/tst/TestRequestHandler.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/tst/TestRequestHandler.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/tst/TestRequestHandler.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/tst/TestRequestHandler.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,300 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.tst;
+
+import org.apache.lucene.search.*;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
+
+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.logging.Logger;
+import java.util.logging.Level;
+import java.net.URL;
+
+import org.apache.solr.util.StrUtils;
+import org.apache.solr.util.NamedList;
+import org.apache.solr.search.*;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrException;
+import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryResponse;
+
+/**
+ * @author yonik
+ * @version $Id: TestRequestHandler.java,v 1.19 2005/12/02 04:31:05 yonik Exp $
+ */
+
+public class TestRequestHandler implements SolrRequestHandler {
+  private static Logger log = Logger.getLogger(SolrIndexSearcher.class.getName());
+
+  public void init(NamedList args) {
+    SolrCore.log.log(Level.INFO, "Unused request handler arguments:" + args);
+  }
+
+
+
+  // use test instead of assert since asserts may be turned off
+  public void test(boolean condition) {
+    try {
+      if (!condition) {
+        throw new RuntimeException("test requestHandler: assertion failed!");
+      }
+    } catch (RuntimeException e) {
+      SolrException.log(log,e);
+      throw(e);
+    }
+  }
+
+
+  private long numRequests;
+  private long numErrors;
+
+  private final Pattern splitList=Pattern.compile(",| ");
+
+
+  public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
+    numRequests++;
+
+    // TODO: test if lucene will accept an escaped ';', otherwise
+    // we need to un-escape them before we pass to QueryParser
+    try {
+      String sreq = req.getQueryString();
+      if (sreq==null) throw new SolrException(400,"Missing queryString");
+      List<String> commands = StrUtils.splitSmart(sreq,';');
+
+      String qs = commands.size() >= 1 ? commands.get(0) : "";
+      Query query = QueryParsing.parseQuery(qs, req.getSchema());
+
+      // find fieldnames to return (fieldlist)
+      String fl = req.getParam("fl");
+      int flags=0;
+      if (fl != null) {
+        // TODO - this could become more efficient if widely used.
+        // TODO - should field order be maintained?
+        String[] flst = splitList.split(fl,0);
+        if (flst.length > 0 && !(flst.length==1 && flst[0].length()==0)) {
+          Set<String> set = new HashSet<String>();
+          for (String fname : flst) {
+            if ("score".equals(fname)) flags |= SolrIndexSearcher.GET_SCORES;
+            set.add(fname);
+          }
+          rsp.setReturnFields(set);
+        }
+      }
+
+
+      // If the first non-query, non-filter command is a simple sort on an indexed field, then
+      // we can use the Lucene sort ability.
+      Sort sort = null;
+      if (commands.size() >= 2) {
+        QueryParsing.SortSpec sortSpec = QueryParsing.parseSort(commands.get(1), req.getSchema());
+        if (sortSpec != null) {
+          sort = sortSpec.getSort();
+          // ignore the count for now... it's currently only controlled by start & limit on req
+          // count = sortSpec.getCount();
+        }
+      }
+
+      SolrIndexSearcher searcher = req.getSearcher();
+
+      /***
+      Object o = searcher.cacheLookup("dfllNode", query);
+      if (o == null) {
+        searcher.cacheInsert("dfllNode",query,"Hello Bob");
+      } else {
+        System.out.println("User Cache Hit On " + o);
+      }
+      ***/
+
+      int start=req.getStart();
+      int limit=req.getLimit();
+
+      Query filterQuery=null;
+      DocSet filter=null;
+      Filter lfilter=null;
+
+      DocList results = req.getSearcher().getDocList(query, null, sort, req.getStart(), req.getLimit(), flags);
+      rsp.add(null, results);
+
+
+      if (qs.startsWith("values")) {
+        rsp.add("testname1","testval1");
+
+        rsp.add("testarr1",new String[]{"my val 1","my val 2"});
+
+        NamedList nl = new NamedList();
+        nl.add("myInt", 333);
+        nl.add("myNullVal", null);
+        nl.add("myFloat",1.414213562f);
+        nl.add("myDouble", 1e100d);
+        nl.add("myBool", false);
+        nl.add("myLong",999999999999L);
+
+        Document doc = new Document();
+        doc.add(new Field("id","55",true,true,false));
+        nl.add("myDoc",doc);
+
+        nl.add("myResult",results);
+        nl.add("myStr","&wow! test escaping: a&b<c&");
+        nl.add(null, "this value had a null name...");
+        nl.add("myIntArray", new Integer[] { 100, 5, -10, 42 });
+        nl.add("epoch", new Date(0));
+        nl.add("currDate", new Date(System.currentTimeMillis()));
+        rsp.add("myNamedList", nl);
+      } else if (qs.startsWith("fields")) {
+        NamedList nl = new NamedList();
+        Collection flst;
+        flst = searcher.getReader().getFieldNames(IndexReader.FieldOption.INDEXED);
+        nl.add("indexed",flst);
+        flst = searcher.getReader().getFieldNames(IndexReader.FieldOption.UNINDEXED);
+        nl.add("unindexed",flst);
+        rsp.add("fields", nl);
+      }
+
+      test(results.size() <= limit);
+      test(results.size() <= results.matches());
+      // System.out.println("limit="+limit+" results.size()="+results.size()+" matches="+results.matches());
+      test((start==0 && limit>=results.matches()) ? results.size()==results.matches() : true );
+
+      //
+      // test against hits
+      //
+      Hits hits = searcher.search(query, lfilter, sort);
+      test(hits.length() == results.matches());
+
+
+      DocList rrr2 = results.subset(start,limit);
+      test(rrr2 == results);
+
+      DocIterator iter=results.iterator();
+
+
+      /***
+      for (int i=0; i<hits.length(); i++) {
+        System.out.println("doc="+hits.id(i) + " score="+hits.score(i));
+      }
+      ***/
+
+      for (int i=0; i<results.size(); i++) {
+        test( iter.nextDoc() == hits.id(i+results.offset()) );
+
+        // Document doesn't implement equals()
+        // test( searcher.document(i).equals(hits.doc(i)));
+      }
+
+
+      DocList results2 = req.getSearcher().getDocList(query,query,sort,start,limit);
+      test(results2.size()==results.size() && results2.matches()==results.matches());
+      DocList results3 = req.getSearcher().getDocList(query,query,null,start,limit);
+      test(results3.size()==results.size() && results3.matches()==results.matches());
+
+      //
+      // getting both the list and set
+      //
+      DocListAndSet both = searcher.getDocListAndSet(query,filter,sort,start, limit);
+      test( both.docList.equals(results) );
+      test( both.docList.matches() == both.docSet.size() );
+      test( (start==0 && both.docSet.size() <= limit) ? both.docSet.equals(both.docList) : true);
+
+      // use the result set as a filter itself...
+      DocListAndSet both2 = searcher.getDocListAndSet(query,both.docSet,sort,start, limit);
+      test( both2.docList.equals(both.docList) );
+      test( both2.docSet.equals(both.docSet) );
+
+      BitSet bits = both.docSet.getBits();
+      BitSet neg = ((BitSet)bits.clone());
+      neg.flip(0, bits.length());
+
+      // use the negative as a filter (should result in 0 matches)
+      // todo - fix if filter is not null
+      both2 = searcher.getDocListAndSet(query,new BitDocSet(neg),sort, start, limit);
+      test( both2.docList.size() == 0 );
+      test( both2.docList.matches() == 0 );
+      test( both2.docSet.size() == 0 );
+
+      DocSet allResults=searcher.getDocSet(query,filter);
+      test ( allResults.equals(both.docSet) );
+
+      if (filter != null) {
+        DocSet res=searcher.getDocSet(query);
+        test( res.size() >= results.size() );
+        test( res.intersection(filter).equals(both.docSet));
+
+        test( res.intersectionSize(filter) == both.docSet.size() );
+        if (filterQuery != null) {
+          test( searcher.numDocs(filterQuery,res) == both.docSet.size() );
+        }
+      }
+
+
+    } catch (Exception e) {
+      rsp.setException(e);
+      numErrors++;
+      return;
+    }
+  }
+
+
+    //////////////////////// SolrInfoMBeans methods //////////////////////
+
+
+  public String getName() {
+    return TestRequestHandler.class.getName();
+  }
+
+  public String getVersion() {
+    return SolrCore.version;
+  }
+
+  public String getDescription() {
+    return "A test handler that runs some sanity checks on results";
+  }
+
+  public Category getCategory() {
+    return Category.QUERYHANDLER;
+  }
+
+  public String getCvsId() {
+    return "$Id: TestRequestHandler.java,v 1.19 2005/12/02 04:31:05 yonik Exp $";
+  }
+
+  public String getCvsName() {
+    return "$Name:  $";
+  }
+
+  public String getCvsSource() {
+    return "$Source: /cvs/main/searching/solr/solarcore/src/solr/tst/TestRequestHandler.java,v $";
+  }
+
+  public URL[] getDocs() {
+    return null;
+  }
+
+  public NamedList getStatistics() {
+    NamedList lst = new NamedList();
+    lst.add("requests", numRequests);
+    lst.add("errors", numErrors);
+    return lst;
+  }
+
+
+
+}
+

Propchange: incubator/solr/trunk/src/java/org/apache/solr/tst/TestRequestHandler.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/update/AddUpdateCommand.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/update/AddUpdateCommand.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/update/AddUpdateCommand.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/update/AddUpdateCommand.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,44 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update;
+
+import org.apache.lucene.document.Document;
+/**
+ * @author yonik
+ * @version $Id$
+ */
+public class AddUpdateCommand extends UpdateCommand {
+   public String id;
+   public Document doc;
+   public boolean allowDups;
+   public boolean overwritePending;
+   public boolean overwriteCommitted;
+
+   public AddUpdateCommand() {
+     super("add");
+   }
+
+   public String toString() {
+     StringBuilder sb = new StringBuilder(commandName);
+     sb.append(':');
+     if (id!=null) sb.append("id=").append(id);
+     sb.append(",allowDups=").append(allowDups);
+     sb.append(",overwritePending=").append(overwritePending);
+     sb.append(",overwriteCommitted=").append(overwriteCommitted);
+     return sb.toString();
+   }
+ }

Propchange: incubator/solr/trunk/src/java/org/apache/solr/update/AddUpdateCommand.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,37 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update;
+/**
+ * @author yonik
+ * @version $Id$
+ */
+public class CommitUpdateCommand extends UpdateCommand {
+  public boolean optimize;
+  public boolean waitFlush;
+  public boolean waitSearcher=true;
+
+  public CommitUpdateCommand(boolean optimize) {
+    super("commit");
+    this.optimize=optimize;
+  }
+  public String toString() {
+    return "commit(optimize="+optimize
+            +",waitFlush="+waitFlush
+            +",waitSearcher="+waitSearcher
+            +')';
+  }
+}

Propchange: incubator/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/solr/trunk/src/java/org/apache/solr/update/DeleteUpdateCommand.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/update/DeleteUpdateCommand.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/update/DeleteUpdateCommand.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/update/DeleteUpdateCommand.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,41 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update;
+/**
+ * @author yonik
+ * @version $Id$
+ */
+public class DeleteUpdateCommand extends UpdateCommand {
+  public String id;
+  public String query;
+  public boolean fromPending;
+  public boolean fromCommitted;
+
+  public DeleteUpdateCommand() {
+    super("delete");
+  }
+
+  public String toString() {
+    StringBuilder sb = new StringBuilder(commandName);
+    sb.append(':');
+    if (id!=null) sb.append("id=").append(id);
+    else sb.append("query=`").append(query).append('`');
+    sb.append(",fromPending=").append(fromPending);
+    sb.append(",fromCommitted=").append(fromCommitted);
+    return sb.toString();
+  }
+}

Propchange: incubator/solr/trunk/src/java/org/apache/solr/update/DeleteUpdateCommand.java
------------------------------------------------------------------------------
    svn:eol-style = native