You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2017/11/01 16:59:03 UTC

lucene-solr:master: LUCENE-7994: use int/int scatter map to count facets when number of hits is small relative to number of unique facet labels

Repository: lucene-solr
Updated Branches:
  refs/heads/master 1ff6084d8 -> 77e6e291b


LUCENE-7994: use int/int scatter map to count facets when number of hits is small relative to number of unique facet labels


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/77e6e291
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/77e6e291
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/77e6e291

Branch: refs/heads/master
Commit: 77e6e291bf34ffaa6f1afc2d9c64779f4b250b65
Parents: 1ff6084
Author: Mike McCandless <mi...@apache.org>
Authored: Wed Nov 1 12:58:53 2017 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Nov 1 12:58:53 2017 -0400

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |   4 +
 .../org/apache/lucene/search/QueryRescorer.java |   2 +-
 lucene/facet/build.xml                          |   1 +
 lucene/facet/ivy.xml                            |   6 +
 .../lucene/facet/LongValueFacetCounts.java      | 128 +++--------
 .../facet/taxonomy/FastTaxonomyFacetCounts.java |   8 +-
 .../facet/taxonomy/FloatTaxonomyFacets.java     |  12 +-
 .../facet/taxonomy/IntTaxonomyFacets.java       | 149 ++++++++++---
 .../facet/taxonomy/TaxonomyFacetCounts.java     |   4 +-
 .../TaxonomyFacetSumIntAssociations.java        |   4 +-
 .../lucene/facet/taxonomy/TaxonomyFacets.java   |  47 +++-
 .../writercache/LruTaxonomyWriterCache.java     |   4 +
 .../writercache/TaxonomyWriterCache.java        |   4 +-
 .../writercache/UTF8TaxonomyWriterCache.java    | 215 ++++++++++---------
 .../facet/taxonomy/TestTaxonomyFacetCounts.java |   8 +
 .../TestConcurrentFacetedIndexing.java          |   2 +
 .../directory/TestDirectoryTaxonomyWriter.java  |   2 +
 .../TestUTF8TaxonomyWriterCache.java            |  22 +-
 lucene/ivy-versions.properties                  |   2 +-
 lucene/licenses/hppc-0.7.3.jar.sha1             |   1 +
 lucene/licenses/hppc-LICENSE-ASL.txt            | 177 +++++++++++++++
 lucene/licenses/hppc-NOTICE.txt                 |   0
 solr/licenses/hppc-0.7.1.jar.sha1               |   1 -
 solr/licenses/hppc-0.7.3.jar.sha1               |   1 +
 24 files changed, 547 insertions(+), 257 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f214620..d0ed234 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -41,6 +41,10 @@ Optimizations
 * LUCENE-8018: Smaller FieldInfos memory footprint by not retaining unnecessary
   references to TreeMap entries. (Julian Vassev via Adrien Grand)
 
+* LUCENE-7994: Use int/int scatter map to gather facet counts when the
+  number of hits is small relative to the number of unique facet labels
+  (Dawid Weiss, Robert Muir, Mike McCandless)
+
 ======================= Lucene 7.1.0 =======================
 
 Changes in Runtime Behavior

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java b/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java
index a8477e9..73c37d2 100644
--- a/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java
@@ -85,7 +85,7 @@ public abstract class QueryRescorer extends Rescorer {
         scorer = weight.scorer(readerContext);
       }
 
-      if(scorer != null) {
+      if (scorer != null) {
         int targetDoc = docID - docBase;
         int actualDoc = scorer.docID();
         if (actualDoc < targetDoc) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/build.xml
----------------------------------------------------------------------
diff --git a/lucene/facet/build.xml b/lucene/facet/build.xml
index 1a568ea..e91f162 100644
--- a/lucene/facet/build.xml
+++ b/lucene/facet/build.xml
@@ -28,6 +28,7 @@
   <path id="classpath">
     <path refid="base.classpath"/>
     <pathelement path="${queries.jar}"/>
+    <fileset dir="lib"/>
   </path>
 
   <target name="compile-core" depends="jar-queries,common.compile-core" />

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/ivy.xml
----------------------------------------------------------------------
diff --git a/lucene/facet/ivy.xml b/lucene/facet/ivy.xml
index 326f71f..249c78e 100644
--- a/lucene/facet/ivy.xml
+++ b/lucene/facet/ivy.xml
@@ -18,4 +18,10 @@
 -->
 <ivy-module version="2.0">
   <info organisation="org.apache.lucene" module="facet"/>
+  <configurations defaultconfmapping="compile->master">
+    <conf name="compile" transitive="false"/>
+  </configurations>
+  <dependencies>
+    <dependency org="com.carrotsearch" name="hppc" rev="${/com.carrotsearch/hppc}" conf="compile"/>
+  </dependencies>
 </ivy-module>

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/java/org/apache/lucene/facet/LongValueFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/LongValueFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/LongValueFacetCounts.java
index 7d7a32b..ac3f5b3 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/LongValueFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/LongValueFacetCounts.java
@@ -40,6 +40,10 @@ import org.apache.lucene.search.LongValuesSource;
 import org.apache.lucene.util.InPlaceMergeSorter;
 import org.apache.lucene.util.PriorityQueue;
 
+import com.carrotsearch.hppc.LongIntScatterMap;
+import com.carrotsearch.hppc.cursors.LongIntCursor;
+
+
 /** {@link Facets} implementation that computes counts for
  *  all uniqute long values, more efficiently counting small values (0-1023) using an int array,
  *  and switching to a <code>HashMap</code> for values above 1023.
@@ -53,7 +57,7 @@ public class LongValueFacetCounts extends Facets {
   private final int[] counts = new int[1024];
 
   /** Used for all values that are >= 1K. */
-  private final HashTable hashCounts = new HashTable();
+  private final LongIntScatterMap hashCounts = new LongIntScatterMap();
 
   private final String field;
 
@@ -253,7 +257,7 @@ public class LongValueFacetCounts extends Facets {
     if (value >= 0 && value < counts.length) {
       counts[(int) value]++;
     } else {
-      hashCounts.add(value, 1);
+      hashCounts.addTo(value, 1);
     }
   }
 
@@ -276,7 +280,7 @@ public class LongValueFacetCounts extends Facets {
 
   /** Returns the specified top number of facets, sorted by count. */
   public FacetResult getTopChildrenSortByCount(int topN) {
-    PriorityQueue<Entry> pq = new PriorityQueue<Entry>(Math.min(topN, counts.length + hashCounts.size)) {
+    PriorityQueue<Entry> pq = new PriorityQueue<Entry>(Math.min(topN, counts.length + hashCounts.size())) {
                         @Override
         protected boolean lessThan(Entry a, Entry b) {
           // sort by count descending, breaking ties by value ascending:
@@ -298,15 +302,15 @@ public class LongValueFacetCounts extends Facets {
       }
     }
 
-    if (hashCounts.size != 0) {
-      childCount += hashCounts.size;
-      for (int i = 0; i < hashCounts.values.length; i++) {
-        int count = hashCounts.counts[i];
+    if (hashCounts.size() != 0) {
+      childCount += hashCounts.size();
+      for (LongIntCursor c : hashCounts) {
+        int count = c.value;
         if (count != 0) {
           if (e == null) {
             e = new Entry();
           }
-          e.value = hashCounts.values[i];
+          e.value = c.key;
           e.count = count;
           e = pq.insertWithOverflow(e);
         }
@@ -328,47 +332,47 @@ public class LongValueFacetCounts extends Facets {
     List<LabelAndValue> labelValues = new ArrayList<>();
 
     // compact & sort hash table's arrays by value
+    int[] hashCounts = new int[this.hashCounts.size()];
+    long[] hashValues = new long[this.hashCounts.size()];
+    
     int upto = 0;
-    for (int i = 0; i < hashCounts.values.length; i++) {
-      if (hashCounts.counts[i] != 0) {
-        hashCounts.counts[upto] = hashCounts.counts[i];
-        hashCounts.values[upto] = hashCounts.values[i];
+    for (LongIntCursor c : this.hashCounts) {
+      if (c.value != 0) {
+        hashCounts[upto] = c.value;
+        hashValues[upto] = c.key;
         upto++;
       }
     }
 
-    // zero fill all remaining counts so if we are called again we don't mistake these as real values
-    Arrays.fill(hashCounts.counts, upto, hashCounts.counts.length, 0);
-
-    assert upto == hashCounts.size : "upto=" + upto + " hashCounts.size=" + hashCounts.size;
+    assert upto == this.hashCounts.size() : "upto=" + upto + " hashCounts.size=" + this.hashCounts.size();
 
     new InPlaceMergeSorter() {
       @Override
       public int compare(int i, int j) {
-        return Long.compare(hashCounts.values[i], hashCounts.values[j]);
+        return Long.compare(hashValues[i], hashValues[j]);
       }
 
       @Override
       public void swap(int i, int j) {
-        int x = hashCounts.counts[i];
-        hashCounts.counts[i] = hashCounts.counts[j];
-        hashCounts.counts[j] = x;
+        int x = hashCounts[i];
+        hashCounts[i] = hashCounts[j];
+        hashCounts[j] = x;
 
-        long y = hashCounts.values[j];
-        hashCounts.values[j] = hashCounts.values[i];
-        hashCounts.values[i] = y;
+        long y = hashValues[j];
+        hashValues[j] = hashValues[i];
+        hashValues[i] = y;
       }
     }.sort(0, upto);
 
     boolean countsAdded = false;
     for (int i = 0; i < upto; i++) {
-      if (countsAdded == false && hashCounts.values[i] >= counts.length) {
+      if (countsAdded == false && hashValues[i] >= counts.length) {
         countsAdded = true;
         appendCounts(labelValues);
       }
 
-      labelValues.add(new LabelAndValue(Long.toString(hashCounts.values[i]),
-                                        hashCounts.counts[i]));
+      labelValues.add(new LabelAndValue(Long.toString(hashValues[i]),
+                                        hashCounts[i]));
     }
 
     if (countsAdded == false) {
@@ -413,13 +417,13 @@ public class LongValueFacetCounts extends Facets {
       }
     }
 
-    if (hashCounts.size != 0) {
-      for (int i = 0; i < hashCounts.values.length; i++) {
-        if (hashCounts.counts[i] != 0) {
+    if (hashCounts.size() != 0) {
+      for (LongIntCursor c : hashCounts) {
+        if (c.value != 0) {
           b.append("  ");
-          b.append(hashCounts.values[i]);
+          b.append(c.key);
           b.append(" -> count=");
-          b.append(hashCounts.counts[i]);
+          b.append(c.value);
           b.append('\n');
         }
       }
@@ -427,66 +431,4 @@ public class LongValueFacetCounts extends Facets {
 
     return b.toString();
   }
-
-  /** Native typed hash table. */
-  static class HashTable {
-
-    static final float LOAD_FACTOR = 0.7f;
-
-    long[] values; // values identifying a value
-    int[] counts;
-    int mask;
-    int size;
-    int threshold;
-
-    HashTable() {
-      int capacity = 64; // must be a power of 2
-      values = new long[capacity];
-      counts = new int[capacity];
-      mask = capacity - 1;
-      size = 0;
-      threshold = (int) (capacity * LOAD_FACTOR);
-    }
-
-    private int hash(long v) {
-      int h = (int) (v ^ (v >>> 32));
-      h = (31 * h) & mask; // * 31 to try to use the whole table, even if values are dense
-      return h;
-    }
-
-    void add(long value, int inc) {
-      if (size >= threshold) {
-        rehash();
-      }
-      final int h = hash(value);
-      for (int slot = h;; slot = (slot + 1) & mask) {
-        if (counts[slot] == 0) {
-          values[slot] = value;
-          ++size;
-        } else if (values[slot] != value) {
-          continue;
-        }
-        counts[slot] += inc;
-        break;
-      }
-    }
-
-    private void rehash() {
-      final long[] oldValues = values;
-      final int[] oldCounts = counts;
-
-      final int newCapacity = values.length * 2;
-      values = new long[newCapacity];
-      counts = new int[newCapacity];
-      mask = newCapacity - 1;
-      threshold = (int) (LOAD_FACTOR * newCapacity);
-      size = 0;
-
-      for (int i = 0; i < oldValues.length; ++i) {
-        if (oldCounts[i] > 0) {
-          add(oldValues[i], oldCounts[i]);
-        }
-      }
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
index d560d40..4767336 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
@@ -50,7 +50,7 @@ public class FastTaxonomyFacetCounts extends IntTaxonomyFacets {
    *  FacetsConfig#setIndexFieldName} to change the index
    *  field name for certain dimensions. */
   public FastTaxonomyFacetCounts(String indexFieldName, TaxonomyReader taxoReader, FacetsConfig config, FacetsCollector fc) throws IOException {
-    super(indexFieldName, taxoReader, config);
+    super(indexFieldName, taxoReader, config, fc);
     count(fc.getMatchingDocs());
   }
 
@@ -60,7 +60,7 @@ public class FastTaxonomyFacetCounts extends IntTaxonomyFacets {
    *  the same result as searching on {@link MatchAllDocsQuery},
    *  but faster */
   public FastTaxonomyFacetCounts(String indexFieldName, IndexReader reader, TaxonomyReader taxoReader, FacetsConfig config) throws IOException {
-    super(indexFieldName, taxoReader, config);
+    super(indexFieldName, taxoReader, config, null);
     countAll(reader);
   }
 
@@ -85,7 +85,7 @@ public class FastTaxonomyFacetCounts extends IntTaxonomyFacets {
           byte b = bytes[offset++];
           if (b >= 0) {
             prev = ord = ((ord << 7) | b) + prev;
-            ++values[ord];
+            increment(ord);
             ord = 0;
           } else {
             ord = (ord << 7) | (b & 0x7F);
@@ -120,7 +120,7 @@ public class FastTaxonomyFacetCounts extends IntTaxonomyFacets {
           byte b = bytes[offset++];
           if (b >= 0) {
             prev = ord = ((ord << 7) | b) + prev;
-            ++values[ord];
+            increment(ord);
             ord = 0;
           } else {
             ord = (ord << 7) | (b & 0x7F);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FloatTaxonomyFacets.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FloatTaxonomyFacets.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FloatTaxonomyFacets.java
index 2e8231d..371e327 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FloatTaxonomyFacets.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FloatTaxonomyFacets.java
@@ -29,6 +29,8 @@ import org.apache.lucene.facet.TopOrdAndFloatQueue;
  *  to a per-ords float[]. */
 public abstract class FloatTaxonomyFacets extends TaxonomyFacets {
 
+  // TODO: also use native hash map for sparse collection, like IntTaxonomyFacets
+
   /** Per-ordinal value. */
   protected final float[] values;
 
@@ -41,6 +43,7 @@ public abstract class FloatTaxonomyFacets extends TaxonomyFacets {
   /** Rolls up any single-valued hierarchical dimensions. */
   protected void rollup() throws IOException {
     // Rollup any necessary dims:
+    int[] children = getChildren();
     for(Map.Entry<String,DimConfig> ent : config.getDimConfigs().entrySet()) {
       String dim = ent.getKey();
       DimConfig ft = ent.getValue();
@@ -52,7 +55,9 @@ public abstract class FloatTaxonomyFacets extends TaxonomyFacets {
     }
   }
 
-  private float rollup(int ord) {
+  private float rollup(int ord) throws IOException {
+    int[] children = getChildren();
+    int[] siblings = getSiblings();
     float sum = 0;
     while (ord != TaxonomyReader.INVALID_ORDINAL) {
       float childValue = values[ord] + rollup(children[ord]);
@@ -97,6 +102,9 @@ public abstract class FloatTaxonomyFacets extends TaxonomyFacets {
     TopOrdAndFloatQueue q = new TopOrdAndFloatQueue(Math.min(taxoReader.getSize(), topN));
     float bottomValue = 0;
 
+    int[] children = getChildren();
+    int[] siblings = getSiblings();
+
     int ord = children[dimOrd];
     float sumValues = 0;
     int childCount = 0;
@@ -146,4 +154,4 @@ public abstract class FloatTaxonomyFacets extends TaxonomyFacets {
 
     return new FacetResult(dim, path, sumValues, labelValues, childCount);
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/IntTaxonomyFacets.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/IntTaxonomyFacets.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/IntTaxonomyFacets.java
index 1b0a96d..71b628c 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/IntTaxonomyFacets.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/IntTaxonomyFacets.java
@@ -20,28 +20,87 @@ import java.io.IOException;
 import java.util.Map;
 
 import org.apache.lucene.facet.FacetResult;
-import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
 import org.apache.lucene.facet.FacetsConfig.DimConfig;
+import org.apache.lucene.facet.FacetsConfig;
 import org.apache.lucene.facet.LabelAndValue;
 import org.apache.lucene.facet.TopOrdAndIntQueue;
 
+import com.carrotsearch.hppc.IntIntScatterMap;
+import com.carrotsearch.hppc.cursors.IntIntCursor;
+
 /** Base class for all taxonomy-based facets that aggregate
  *  to a per-ords int[]. */
 
 public abstract class IntTaxonomyFacets extends TaxonomyFacets {
 
   /** Per-ordinal value. */
-  protected final int[] values;
+  private final int[] values;
+  private final IntIntScatterMap sparseValues;
 
   /** Sole constructor. */
-  protected IntTaxonomyFacets(String indexFieldName, TaxonomyReader taxoReader, FacetsConfig config) throws IOException {
+  protected IntTaxonomyFacets(String indexFieldName, TaxonomyReader taxoReader, FacetsConfig config, FacetsCollector fc) throws IOException {
     super(indexFieldName, taxoReader, config);
-    values = new int[taxoReader.getSize()];
+
+    if (useHashTable(fc, taxoReader)) {
+      sparseValues = new IntIntScatterMap();
+      values = null;
+    } else {
+      sparseValues = null;
+      values = new int[taxoReader.getSize()];
+    }
   }
-  
+
+  /** Return true if a sparse hash table should be used for counting, instead of a dense int[]. */
+  protected boolean useHashTable(FacetsCollector fc, TaxonomyReader taxoReader) {
+    if (taxoReader.getSize() < 1024) {
+      // small number of unique values: use an array
+      return false;
+    }
+
+    if (fc == null) {
+      // counting all docs: use an array
+      return false;
+    }
+    
+    int maxDoc = 0;
+    int sumTotalHits = 0;
+    for (MatchingDocs docs : fc.getMatchingDocs()) {
+      sumTotalHits += docs.totalHits;
+      maxDoc += docs.context.reader().maxDoc();
+    }
+
+    // if our result set is < 10% of the index, we collect sparsely (use hash map):
+    return sumTotalHits < maxDoc/10;
+  }
+
+  /** Increment the count for this ordinal by 1. */
+  protected void increment(int ordinal) {
+    increment(ordinal, 1);
+  }
+
+  /** Increment the count for this ordinal by {@code amount}.. */
+  protected void increment(int ordinal, int amount) {
+    if (sparseValues != null) {
+      sparseValues.addTo(ordinal, amount);
+    } else {
+      values[ordinal] += amount;
+    }
+  }
+
+  private int getValue(int ordinal) {
+    if (sparseValues != null) {
+      return sparseValues.get(ordinal);
+    } else {
+      return values[ordinal];
+    }
+  }
+
   /** Rolls up any single-valued hierarchical dimensions. */
   protected void rollup() throws IOException {
     // Rollup any necessary dims:
+    int[] children = null;
     for(Map.Entry<String,DimConfig> ent : config.getDimConfigs().entrySet()) {
       String dim = ent.getKey();
       DimConfig ft = ent.getValue();
@@ -50,18 +109,23 @@ public abstract class IntTaxonomyFacets extends TaxonomyFacets {
         // It can be -1 if this field was declared in the
         // config but never indexed:
         if (dimRootOrd > 0) {
-          values[dimRootOrd] += rollup(children[dimRootOrd]);
+          if (children == null) {
+            // lazy init
+            children = getChildren();
+          }
+          increment(dimRootOrd, rollup(children[dimRootOrd]));
         }
       }
     }
   }
 
-  private int rollup(int ord) {
+  private int rollup(int ord) throws IOException {
+    int[] children = getChildren();
+    int[] siblings = getSiblings();
     int sum = 0;
     while (ord != TaxonomyReader.INVALID_ORDINAL) {
-      int childValue = values[ord] + rollup(children[ord]);
-      values[ord] = childValue;
-      sum += childValue;
+      increment(ord, rollup(children[ord]));
+      sum += getValue(ord);
       ord = siblings[ord];
     }
     return sum;
@@ -83,7 +147,7 @@ public abstract class IntTaxonomyFacets extends TaxonomyFacets {
     if (ord < 0) {
       return -1;
     }
-    return values[ord];
+    return getValue(ord);
   }
 
   @Override
@@ -102,29 +166,58 @@ public abstract class IntTaxonomyFacets extends TaxonomyFacets {
     
     int bottomValue = 0;
 
-    int ord = children[dimOrd];
     int totValue = 0;
     int childCount = 0;
 
     TopOrdAndIntQueue.OrdAndValue reuse = null;
-    while(ord != TaxonomyReader.INVALID_ORDINAL) {
-      if (values[ord] > 0) {
-        totValue += values[ord];
-        childCount++;
-        if (values[ord] > bottomValue) {
-          if (reuse == null) {
-            reuse = new TopOrdAndIntQueue.OrdAndValue();
-          }
-          reuse.ord = ord;
-          reuse.value = values[ord];
-          reuse = q.insertWithOverflow(reuse);
-          if (q.size() == topN) {
-            bottomValue = q.top().value;
+
+    // TODO: would be faster if we had a "get the following children" API?  then we
+    // can make a single pass over the hashmap
+
+    if (sparseValues != null) {
+      for (IntIntCursor c : sparseValues) {
+        int count = c.value;
+        int ord = c.key;
+        if (parents[ord] == dimOrd && count > 0) {
+          totValue += count;
+          childCount++;
+          if (count > bottomValue) {
+            if (reuse == null) {
+              reuse = new TopOrdAndIntQueue.OrdAndValue();
+            }
+            reuse.ord = ord;
+            reuse.value = count;
+            reuse = q.insertWithOverflow(reuse);
+            if (q.size() == topN) {
+              bottomValue = q.top().value;
+            }
           }
         }
       }
+    } else {
+      int[] children = getChildren();
+      int[] siblings = getSiblings();
+      int ord = children[dimOrd];
+      while(ord != TaxonomyReader.INVALID_ORDINAL) {
+        int value = values[ord];
+        if (value > 0) {
+          totValue += value;
+          childCount++;
+          if (value > bottomValue) {
+            if (reuse == null) {
+              reuse = new TopOrdAndIntQueue.OrdAndValue();
+            }
+            reuse.ord = ord;
+            reuse.value = value;
+            reuse = q.insertWithOverflow(reuse);
+            if (q.size() == topN) {
+              bottomValue = q.top().value;
+            }
+          }
+        }
 
-      ord = siblings[ord];
+        ord = siblings[ord];
+      }
     }
 
     if (totValue == 0) {
@@ -133,7 +226,7 @@ public abstract class IntTaxonomyFacets extends TaxonomyFacets {
 
     if (dimConfig.multiValued) {
       if (dimConfig.requireDimCount) {
-        totValue = values[dimOrd];
+        totValue = getValue(dimOrd);
       } else {
         // Our sum'd value is not correct, in general:
         totValue = -1;
@@ -151,4 +244,4 @@ public abstract class IntTaxonomyFacets extends TaxonomyFacets {
 
     return new FacetResult(dim, path, totValue, labelValues, childCount);
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
index ce82e27..891851e 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
@@ -39,7 +39,7 @@ public class TaxonomyFacetCounts extends IntTaxonomyFacets {
    *  {@link OrdinalsReader}; otherwise use {@link
    *  FastTaxonomyFacetCounts}. */
   public TaxonomyFacetCounts(OrdinalsReader ordinalsReader, TaxonomyReader taxoReader, FacetsConfig config, FacetsCollector fc) throws IOException {
-    super(ordinalsReader.getIndexFieldName(), taxoReader, config);
+    super(ordinalsReader.getIndexFieldName(), taxoReader, config, fc);
     this.ordinalsReader = ordinalsReader;
     count(fc.getMatchingDocs());
   }
@@ -54,7 +54,7 @@ public class TaxonomyFacetCounts extends IntTaxonomyFacets {
       while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
         ords.get(doc, scratch);
         for(int i=0;i<scratch.length;i++) {
-          values[scratch.ints[scratch.offset+i]]++;
+          increment(scratch.ints[scratch.offset+i]);
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
index 6913d0d..e61f484 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
@@ -42,7 +42,7 @@ public class TaxonomyFacetSumIntAssociations extends IntTaxonomyFacets {
   /** Create {@code TaxonomyFacetSumIntAssociations} against
    *  the specified index field. */
   public TaxonomyFacetSumIntAssociations(String indexFieldName, TaxonomyReader taxoReader, FacetsConfig config, FacetsCollector fc) throws IOException {
-    super(indexFieldName, taxoReader, config);
+    super(indexFieldName, taxoReader, config, fc);
     sumValues(fc.getMatchingDocs());
   }
 
@@ -80,7 +80,7 @@ public class TaxonomyFacetSumIntAssociations extends IntTaxonomyFacets {
               ((bytes[offset+2]&0xFF) << 8) |
               (bytes[offset+3]&0xFF);
             offset += 4;
-            values[ord] += value;
+            increment(ord, value);
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacets.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacets.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacets.java
index e1903d1..6bc2c6d 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacets.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacets.java
@@ -54,20 +54,54 @@ public abstract class TaxonomyFacets extends Facets {
 
   /** Maps parent ordinal to its child, or -1 if the parent
    *  is childless. */
-  protected final int[] children;
+  private int[] children;
 
   /** Maps an ordinal to its sibling, or -1 if there is no
    *  sibling. */
-  protected final int[] siblings;
+  private int[] siblings;
+
+  /** Maps an ordinal to its parent, or -1 if there is no
+   *  parent (root node). */
+  protected final int[] parents;
 
   /** Sole constructor. */
   protected TaxonomyFacets(String indexFieldName, TaxonomyReader taxoReader, FacetsConfig config) throws IOException {
     this.indexFieldName = indexFieldName;
     this.taxoReader = taxoReader;
     this.config = config;
-    ParallelTaxonomyArrays pta = taxoReader.getParallelTaxonomyArrays();
-    children = pta.children();
-    siblings = pta.siblings();
+    parents = taxoReader.getParallelTaxonomyArrays().parents();
+  }
+
+  /** Returns int[] mapping each ordinal to its first child; this is a large array and
+   *  is computed (and then saved) the first time this method is invoked. */
+  protected int[] getChildren() throws IOException {
+    if (children == null) {
+      children = taxoReader.getParallelTaxonomyArrays().children();
+    }
+    return children;
+  }       
+
+  /** Returns int[] mapping each ordinal to its next sibling; this is a large array and
+   *  is computed (and then saved) the first time this method is invoked. */
+  protected int[] getSiblings() throws IOException {
+    if (siblings == null) {
+      siblings = taxoReader.getParallelTaxonomyArrays().siblings();
+    }
+    return siblings;
+  }
+
+  /** Returns true if the (costly, and lazily initialized) children int[] was initialized.
+   *
+   * @lucene.experimental */
+  public boolean childrenLoaded() {
+    return children != null;
+  }
+
+  /** Returns true if the (costly, and lazily initialized) sibling int[] was initialized.
+   *
+   * @lucene.experimental */
+  public boolean siblingsLoaded() {
+    return children != null;
   }
 
   /** Throws {@code IllegalArgumentException} if the
@@ -83,6 +117,8 @@ public abstract class TaxonomyFacets extends Facets {
 
   @Override
   public List<FacetResult> getAllDims(int topN) throws IOException {
+    int[] children = getChildren();
+    int[] siblings = getSiblings();
     int ord = children[TaxonomyReader.ROOT_ORDINAL];
     List<FacetResult> results = new ArrayList<>();
     while (ord != TaxonomyReader.INVALID_ORDINAL) {
@@ -101,5 +137,4 @@ public abstract class TaxonomyFacets extends Facets {
     Collections.sort(results, BY_VALUE_THEN_DIM);
     return results;
   }
-  
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/LruTaxonomyWriterCache.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/LruTaxonomyWriterCache.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/LruTaxonomyWriterCache.java
index 6dc8cd2..1182ffd 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/LruTaxonomyWriterCache.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/LruTaxonomyWriterCache.java
@@ -89,6 +89,10 @@ public class LruTaxonomyWriterCache implements TaxonomyWriterCache {
     cache = null;
   }
 
+  public int size() {
+    return cache.getSize();
+  }
+  
   @Override
   public synchronized int get(FacetLabel categoryPath) {
     Integer res = cache.get(categoryPath);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java
index f310426..b72db21 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java
@@ -94,5 +94,7 @@ public interface TaxonomyWriterCache {
    * assume that the cache is still operable after this method returns.
    */
   public void clear();
-  
+
+  /** How many labels are currently stored in the cache. */
+  public int size();
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/UTF8TaxonomyWriterCache.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/UTF8TaxonomyWriterCache.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/UTF8TaxonomyWriterCache.java
index 70bb6ba..1c0adf7 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/UTF8TaxonomyWriterCache.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/UTF8TaxonomyWriterCache.java
@@ -31,128 +31,133 @@ import org.apache.lucene.util.UnicodeUtil;
 
 /** A "cache" that never frees memory, and stores labels in a BytesRefHash (utf-8 encoding). */
 public final class UTF8TaxonomyWriterCache implements TaxonomyWriterCache, Accountable {
-    private final ThreadLocal<BytesRefBuilder> bytes = new ThreadLocal<BytesRefBuilder>() {
-            @Override
-            protected BytesRefBuilder initialValue() {
-                return new BytesRefBuilder();
-            }
-        };
-    private final Counter bytesUsed = Counter.newCounter();
-    private final BytesRefHash map = new BytesRefHash(new ByteBlockPool(new DirectTrackingAllocator(bytesUsed)));
+  private final ThreadLocal<BytesRefBuilder> bytes = new ThreadLocal<BytesRefBuilder>() {
+      @Override
+      protected BytesRefBuilder initialValue() {
+        return new BytesRefBuilder();
+      }
+    };
+  private final Counter bytesUsed = Counter.newCounter();
+  private final BytesRefHash map = new BytesRefHash(new ByteBlockPool(new DirectTrackingAllocator(bytesUsed)));
 
-    private final static int ORDINALS_PAGE_SIZE = 65536;
-    private final static int ORDINALS_PAGE_MASK = ORDINALS_PAGE_SIZE - 1;
+  private final static int ORDINALS_PAGE_SIZE = 65536;
+  private final static int ORDINALS_PAGE_MASK = ORDINALS_PAGE_SIZE - 1;
     
-    private volatile int[][] ordinals;
+  private volatile int[][] ordinals;
 
-    // How many labels we are storing:
-    private int count;
+  // How many labels we are storing:
+  private int count;
 
-    // How many pages in ordinals we've allocated:
-    private int pageCount;
+  // How many pages in ordinals we've allocated:
+  private int pageCount;
 
-    /** Sole constructor. */
-    public UTF8TaxonomyWriterCache() {
-        ordinals = new int[1][];
-        ordinals[0] = new int[ORDINALS_PAGE_SIZE];
-    }
+  /** Sole constructor. */
+  public UTF8TaxonomyWriterCache() {
+    ordinals = new int[1][];
+    ordinals[0] = new int[ORDINALS_PAGE_SIZE];
+  }
 
-    @Override
-    public int get(FacetLabel label) {
-        BytesRef bytes = toBytes(label);
-        int id;
-        synchronized (this) {
-            id = map.find(bytes);
-        }
-        if (id == -1) {
-            return LabelToOrdinal.INVALID_ORDINAL;
-        }
-        int page = id / ORDINALS_PAGE_SIZE;
-        int offset = id % ORDINALS_PAGE_MASK;
-        return ordinals[page][offset];
+  @Override
+  public int get(FacetLabel label) {
+    BytesRef bytes = toBytes(label);
+    int id;
+    synchronized (this) {
+      id = map.find(bytes);
     }
-
-    // Called only from assert
-    private boolean assertSameOrdinal(FacetLabel label, int id, int ord) {
-        id = -id - 1;
-        int page = id / ORDINALS_PAGE_SIZE;
-        int offset = id % ORDINALS_PAGE_MASK;
-        int oldOrd = ordinals[page][offset];
-        if (oldOrd != ord) {
-            throw new IllegalArgumentException("label " + label + " was already cached, with old ord=" + oldOrd + " versus new ord=" + ord);
-        }
-        return true;
+    if (id == -1) {
+      return LabelToOrdinal.INVALID_ORDINAL;
     }
+    int page = id / ORDINALS_PAGE_SIZE;
+    int offset = id % ORDINALS_PAGE_MASK;
+    return ordinals[page][offset];
+  }
 
-    @Override
-    public boolean put(FacetLabel label, int ord) {
-        BytesRef bytes = toBytes(label);
-        int id;
-        synchronized (this) {
-            id = map.add(bytes);
-            if (id < 0) {
-                assert assertSameOrdinal(label, id, ord);
-                return false;
-            }
-            assert id == count;
-            int page = id / ORDINALS_PAGE_SIZE;
-            int offset = id % ORDINALS_PAGE_MASK;
-            if (page == pageCount) {
-                if (page == ordinals.length) {
-                    int[][] newOrdinals = new int[ArrayUtil.oversize(page+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF)][];
-                    System.arraycopy(ordinals, 0, newOrdinals, 0, ordinals.length);
-                    ordinals = newOrdinals;
-                }
-                ordinals[page] = new int[ORDINALS_PAGE_MASK];
-                pageCount++;
-            }
-            ordinals[page][offset] = ord;
-            count++;
-
-            // we never prune from the cache
-            return false;
-        }
+  // Called only from assert
+  private boolean assertSameOrdinal(FacetLabel label, int id, int ord) {
+    id = -id - 1;
+    int page = id / ORDINALS_PAGE_SIZE;
+    int offset = id % ORDINALS_PAGE_MASK;
+    int oldOrd = ordinals[page][offset];
+    if (oldOrd != ord) {
+      throw new IllegalArgumentException("label " + label + " was already cached, with old ord=" + oldOrd + " versus new ord=" + ord);
     }
+    return true;
+  }
 
-    @Override
-    public boolean isFull() {
-        // we are never full
+  @Override
+  public boolean put(FacetLabel label, int ord) {
+    BytesRef bytes = toBytes(label);
+    int id;
+    synchronized (this) {
+      id = map.add(bytes);
+      if (id < 0) {
+        assert assertSameOrdinal(label, id, ord);
         return false;
-    }
+      }
+      assert id == count;
+      int page = id / ORDINALS_PAGE_SIZE;
+      int offset = id % ORDINALS_PAGE_MASK;
+      if (page == pageCount) {
+        if (page == ordinals.length) {
+          int[][] newOrdinals = new int[ArrayUtil.oversize(page+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF)][];
+          System.arraycopy(ordinals, 0, newOrdinals, 0, ordinals.length);
+          ordinals = newOrdinals;
+        }
+        ordinals[page] = new int[ORDINALS_PAGE_MASK];
+        pageCount++;
+      }
+      ordinals[page][offset] = ord;
+      count++;
 
-    @Override
-    public synchronized void clear() {
-        map.clear();
-        map.reinit();
-        ordinals = new int[1][];
-        ordinals[0] = new int[ORDINALS_PAGE_SIZE];
-        count = 0;
-        pageCount = 0;
-        assert bytesUsed.get() == 0;
+      // we never prune from the cache
+      return false;
     }
+  }
+
+  @Override
+  public boolean isFull() {
+    // we are never full
+    return false;
+  }
+
+  @Override
+  public synchronized void clear() {
+    map.clear();
+    map.reinit();
+    ordinals = new int[1][];
+    ordinals[0] = new int[ORDINALS_PAGE_SIZE];
+    count = 0;
+    pageCount = 0;
+    assert bytesUsed.get() == 0;
+  }
     
-    @Override
-    public synchronized long ramBytesUsed() {
-        return bytesUsed.get() + pageCount * ORDINALS_PAGE_SIZE * RamUsageEstimator.NUM_BYTES_INT;
-    }
+  /** How many labels are currently stored in the cache. */
+  public int size() {
+    return count;
+  }
+  
+  @Override
+  public synchronized long ramBytesUsed() {
+    return bytesUsed.get() + pageCount * ORDINALS_PAGE_SIZE * RamUsageEstimator.NUM_BYTES_INT;
+  }
     
-    @Override
-    public void close() {
-    }
+  @Override
+  public void close() {
+  }
 
-    private static final byte DELIM_CHAR = (byte) 0x1F;
+  private static final byte DELIM_CHAR = (byte) 0x1F;
     
-    private BytesRef toBytes(FacetLabel label) {
-        BytesRefBuilder bytes = this.bytes.get();
-        bytes.clear();
-        for (int i = 0; i < label.length; i++) {
-            String part = label.components[i];
-            if (i > 0) {
-                bytes.append(DELIM_CHAR);
-            }
-            bytes.grow(bytes.length() + UnicodeUtil.maxUTF8Length(part.length()));
-            bytes.setLength(UnicodeUtil.UTF16toUTF8(part, 0, part.length(), bytes.bytes(), bytes.length()));
-        }
-        return bytes.get();
+  private BytesRef toBytes(FacetLabel label) {
+    BytesRefBuilder bytes = this.bytes.get();
+    bytes.clear();
+    for (int i = 0; i < label.length; i++) {
+      String part = label.components[i];
+      if (i > 0) {
+        bytes.append(DELIM_CHAR);
+      }
+      bytes.grow(bytes.length() + UnicodeUtil.maxUTF8Length(part.length()));
+      bytes.setLength(UnicodeUtil.UTF16toUTF8(part, 0, part.length(), bytes.bytes(), bytes.length()));
     }
+    return bytes.get();
+  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java
index 3bb480d..a12563e 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java
@@ -105,6 +105,10 @@ public class TestTaxonomyFacetCounts extends FacetTestCase {
 
     Facets facets = getAllFacets(FacetsConfig.DEFAULT_INDEX_FIELD_NAME, searcher, taxoReader, config);
 
+    // Publish Date is hierarchical, so we should have loaded all 3 int[]:
+    assertTrue(((TaxonomyFacets) facets).siblingsLoaded());
+    assertTrue(((TaxonomyFacets) facets).childrenLoaded());
+
     // Retrieve & verify results:
     assertEquals("dim=Publish Date path=[] value=5 childCount=3\n  2010 (2)\n  2012 (2)\n  1999 (1)\n", facets.getTopChildren(10, "Publish Date").toString());
     assertEquals("dim=Author path=[] value=5 childCount=4\n  Lisa (2)\n  Bob (1)\n  Susan (1)\n  Frank (1)\n", facets.getTopChildren(10, "Author").toString());
@@ -330,6 +334,10 @@ public class TestTaxonomyFacetCounts extends FacetTestCase {
     assertEquals(1, facets.getSpecificValue("dim", "test\u001Fone"));
     assertEquals(1, facets.getSpecificValue("dim", "test\u001Etwo"));
 
+    // no hierarchy
+    assertFalse(((TaxonomyFacets) facets).siblingsLoaded());
+    assertFalse(((TaxonomyFacets) facets).childrenLoaded());
+
     FacetResult result = facets.getTopChildren(10, "dim");
     assertEquals("dim=dim path=[] value=-1 childCount=2\n  test\u001Fone (1)\n  test\u001Etwo (1)\n", result.toString());
     writer.close();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java
index bf10a3b..9c628d5 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java
@@ -51,6 +51,8 @@ public class TestConcurrentFacetedIndexing extends FacetTestCase {
     public boolean isFull() { return true; }
     @Override
     public void clear() {}
+    @Override
+    public int size() { return 0; }
     
   };
   

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
index 1f330e0..1b5b826 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
@@ -65,6 +65,8 @@ public class TestDirectoryTaxonomyWriter extends FacetTestCase {
     public boolean isFull() { return true; }
     @Override
     public void clear() {}
+    @Override
+    public int size() { return 0; }
     
   };
   

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestUTF8TaxonomyWriterCache.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestUTF8TaxonomyWriterCache.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestUTF8TaxonomyWriterCache.java
index 31a5e49..c36abe1 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestUTF8TaxonomyWriterCache.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestUTF8TaxonomyWriterCache.java
@@ -45,19 +45,19 @@ public class TestUTF8TaxonomyWriterCache extends FacetTestCase {
       int numParts = TestUtil.nextInt(random(), 1, 5);
       StringBuilder b = new StringBuilder();
       for (int i=0;i<numParts;i++) {
-          String part = null;
-          while (true) {
-            part = TestUtil.randomRealisticUnicodeString(random(), 16);
-            part = part.replace("/", "");
-            if (part.length() > 0) {
-              break;
-            }
+        String part = null;
+        while (true) {
+          part = TestUtil.randomRealisticUnicodeString(random(), 16);
+          part = part.replace("/", "");
+          if (part.length() > 0) {
+            break;
           }
+        }
 
-          if (i > 0) {
-            b.append('/');
-          }
-          b.append(part);
+        if (i > 0) {
+          b.append('/');
+        }
+        b.append(part);
       }
       uniqueValuesSet.add(b.toString());
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index d020231..b06adac 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -9,7 +9,7 @@ com.carrotsearch.randomizedtesting.version = 2.5.3
 /com.carrotsearch.randomizedtesting/junit4-ant = ${com.carrotsearch.randomizedtesting.version}
 /com.carrotsearch.randomizedtesting/randomizedtesting-runner = ${com.carrotsearch.randomizedtesting.version}
 
-/com.carrotsearch/hppc = 0.7.1
+/com.carrotsearch/hppc = 0.7.3
 
 
 /com.cybozu.labs/langdetect = 1.1-20120112

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/licenses/hppc-0.7.3.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/hppc-0.7.3.jar.sha1 b/lucene/licenses/hppc-0.7.3.jar.sha1
new file mode 100644
index 0000000..df3f33a
--- /dev/null
+++ b/lucene/licenses/hppc-0.7.3.jar.sha1
@@ -0,0 +1 @@
+1a9c77da84ac7db6a78b49c60947983490ece324

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/licenses/hppc-LICENSE-ASL.txt
----------------------------------------------------------------------
diff --git a/lucene/licenses/hppc-LICENSE-ASL.txt b/lucene/licenses/hppc-LICENSE-ASL.txt
new file mode 100644
index 0000000..f433b1a
--- /dev/null
+++ b/lucene/licenses/hppc-LICENSE-ASL.txt
@@ -0,0 +1,177 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/lucene/licenses/hppc-NOTICE.txt
----------------------------------------------------------------------
diff --git a/lucene/licenses/hppc-NOTICE.txt b/lucene/licenses/hppc-NOTICE.txt
new file mode 100644
index 0000000..e69de29

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/solr/licenses/hppc-0.7.1.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/hppc-0.7.1.jar.sha1 b/solr/licenses/hppc-0.7.1.jar.sha1
deleted file mode 100644
index d3e00e1..0000000
--- a/solr/licenses/hppc-0.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8b5057f74ea378c0150a1860874a3ebdcb713767

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77e6e291/solr/licenses/hppc-0.7.3.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/hppc-0.7.3.jar.sha1 b/solr/licenses/hppc-0.7.3.jar.sha1
new file mode 100644
index 0000000..df3f33a
--- /dev/null
+++ b/solr/licenses/hppc-0.7.3.jar.sha1
@@ -0,0 +1 @@
+1a9c77da84ac7db6a78b49c60947983490ece324