You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2013/01/18 19:31:23 UTC

svn commit: r1435287 [17/41] - in /lucene/dev/branches/LUCENE-2878: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/analysis/icu/ dev-tools/maven/ dev-tools/maven/lucene/benchmark/ dev-tools/maven/solr/ dev-tools/...

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java Fri Jan 18 18:30:54 2013
@@ -1,7 +1,7 @@
 package org.apache.lucene.util.encoding;
 
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -50,11 +50,10 @@ import java.io.OutputStream;
  */
 public class NOnesIntEncoder extends FourFlagsIntEncoder {
 
+  private final IntsRef internalBuffer;
+  
   /** Number of consecutive '1's to be translated into single target value '2'. */
-  private int n;
-
-  /** Counts the number of consecutive ones seen. */
-  private int onesCounter = 0;
+  private final int n;
 
   /**
    * Constructs an encoder with a given value of N (N: Number of consecutive
@@ -62,38 +61,44 @@ public class NOnesIntEncoder extends Fou
    */
   public NOnesIntEncoder(int n) {
     this.n = n;
+    internalBuffer = new IntsRef(n);
   }
 
   @Override
-  public void close() throws IOException {
-    // We might have ones in our buffer, encode them as neccesary.
-    while (onesCounter-- > 0) {
-      super.encode(1);
+  public void encode(IntsRef values, BytesRef buf) {
+    internalBuffer.length = 0;
+    // make sure the internal buffer is large enough
+    if (values.length > internalBuffer.ints.length) {
+      internalBuffer.grow(values.length);
     }
-
-    super.close();
-  }
-
-  @Override
-  public void encode(int value) throws IOException {
-    if (value == 1) {
-      // Increment the number of consecutive ones seen so far
-      if (++onesCounter == n) {
-        super.encode(2);
-        onesCounter = 0;
+    
+    int onesCounter = 0;
+    int upto = values.offset + values.length;
+    for (int i = values.offset; i < upto; i++) {
+      int value = values.ints[i];
+      if (value == 1) {
+        // every N 1's should be encoded as '2'
+        if (++onesCounter == n) {
+          internalBuffer.ints[internalBuffer.length++] = 2;
+          onesCounter = 0;
+        }
+      } else {
+        // there might have been 1's that we need to encode
+        while (onesCounter > 0) {
+          --onesCounter;
+          internalBuffer.ints[internalBuffer.length++] = 1;
+        }
+        
+        // encode value as value+1
+        internalBuffer.ints[internalBuffer.length++] = value + 1;
       }
-      return;
     }
-
-    // If it's not one - there might have been ones we had to encode prior to
-    // this value
+    // there might have been 1's that we need to encode
     while (onesCounter > 0) {
       --onesCounter;
-      super.encode(1);
+      internalBuffer.ints[internalBuffer.length++] = 1;
     }
-
-    // encode value + 1 --> the translation.
-    super.encode(value + 1);
+    super.encode(internalBuffer, buf);
   }
 
   @Override
@@ -102,14 +107,8 @@ public class NOnesIntEncoder extends Fou
   }
 
   @Override
-  public void reInit(OutputStream out) {
-    super.reInit(out);
-    onesCounter = 0;
-  }
-
-  @Override
   public String toString() {
-    return "NOnes (" + n + ") (" + super.toString() + ")";
+    return "NOnes(" + n + ") (" + super.toString() + ")";
   }
 
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java Fri Jan 18 18:30:54 2013
@@ -1,7 +1,9 @@
 package org.apache.lucene.util.encoding;
 
-import java.io.IOException;
-import java.io.StreamCorruptedException;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,41 +23,29 @@ import java.io.StreamCorruptedException;
  */
 
 /**
- * A simple stream decoder which can decode values encoded with
- * {@link SimpleIntEncoder}.
+ * Decodes values encoded with {@link SimpleIntEncoder}.
  * 
  * @lucene.experimental
  */
-public class SimpleIntDecoder extends IntDecoder {
-
-  /**
-   * reusable buffer - allocated only once as this is not a thread-safe object
-   */
-  private byte[] buffer = new byte[4];
+public final class SimpleIntDecoder extends IntDecoder {
 
   @Override
-  public long decode() throws IOException {
-
-    // we need exactly 4 bytes to decode an int in this decoder impl, otherwise, throw an exception
-    int offset = 0;
-    while (offset < 4) {
-      int nRead = in.read(buffer, offset, 4 - offset);
-      if (nRead == -1) {
-        if (offset > 0) {
-          throw new StreamCorruptedException(
-              "Need 4 bytes for decoding an int, got only " + offset);
-        }
-        return EOS;
-      }
-      offset += nRead;
+  public void decode(BytesRef buf, IntsRef values) {
+    values.offset = values.length = 0;
+    int numValues = buf.length / 4; // every value is 4 bytes
+    if (values.ints.length < numValues) { // offset and length are 0
+      values.ints = new int[ArrayUtil.oversize(numValues, RamUsageEstimator.NUM_BYTES_INT)];
+    }
+    
+    int offset = buf.offset;
+    int upto = buf.offset + buf.length;
+    while (offset < upto) {
+      values.ints[values.length++] = 
+          ((buf.bytes[offset++] & 0xFF) << 24) | 
+          ((buf.bytes[offset++] & 0xFF) << 16) | 
+          ((buf.bytes[offset++] & 0xFF) <<  8) | 
+          (buf.bytes[offset++] & 0xFF);
     }
-
-    int v = buffer[3] & 0xff;
-    v |= (buffer[2] << 8) & 0xff00;
-    v |= (buffer[1] << 16) & 0xff0000;
-    v |= (buffer[0] << 24) & 0xff000000;
-
-    return v;
   }
 
   @Override

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java Fri Jan 18 18:30:54 2013
@@ -1,6 +1,7 @@
 package org.apache.lucene.util.encoding;
 
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -24,24 +25,25 @@ import java.io.IOException;
  * 
  * @lucene.experimental
  */
-public class SimpleIntEncoder extends IntEncoder {
+public final class SimpleIntEncoder extends IntEncoder {
 
-  /**
-   * This method makes sure the value wasn't previously encoded by checking
-   * against the Set. If the value wasn't encoded, it's added to the Set, and
-   * encoded with {#link Vint8#encode}
-   * 
-   * @param value
-   *            an integer to be encoded
-   * @throws IOException
-   *             possibly thrown by the OutputStream
-   */
   @Override
-  public void encode(int value) throws IOException {
-    out.write(value >>> 24);
-    out.write((value >> 16) & 0xFF);
-    out.write((value >> 8) & 0xFF);
-    out.write(value & 0xFF);
+  public void encode(IntsRef values, BytesRef buf) {
+    buf.offset = buf.length = 0;
+    // ensure there's enough room in the buffer
+    int bytesNeeded = values.length * 4;
+    if (buf.bytes.length < bytesNeeded) {
+      buf.grow(bytesNeeded);
+    }
+    
+    int upto = values.offset + values.length;
+    for (int i = values.offset; i < upto; i++) {
+      int value = values.ints[i];
+      buf.bytes[buf.length++] = (byte) (value >>> 24);
+      buf.bytes[buf.length++] = (byte) ((value >> 16) & 0xFF);
+      buf.bytes[buf.length++] = (byte) ((value >> 8) & 0xFF);
+      buf.bytes[buf.length++] = (byte) (value & 0xFF);
+    }
   }
 
   @Override

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java Fri Jan 18 18:30:54 2013
@@ -1,9 +1,10 @@
 package org.apache.lucene.util.encoding;
 
-import java.io.IOException;
-import java.io.OutputStream;
 import java.util.Arrays;
 
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -23,17 +24,11 @@ import java.util.Arrays;
 
 /**
  * An {@link IntEncoderFilter} which sorts the values to encode in ascending
- * order before encoding them. Encoding therefore happens upon calling
- * {@link #close()}. Since this encoder is usually chained with another encoder
- * that relies on sorted values, it does not offer a default constructor.
+ * order before encoding them.
  * 
  * @lucene.experimental
  */
-public class SortingIntEncoder extends IntEncoderFilter {
-
-  private float grow = 2.0f;
-  private int index = 0;
-  private int[] set = new int[1024];
+public final class SortingIntEncoder extends IntEncoderFilter {
 
   /** Initializes with the given encoder. */
   public SortingIntEncoder(IntEncoder encoder) {
@@ -41,29 +36,9 @@ public class SortingIntEncoder extends I
   }
 
   @Override
-  public void close() throws IOException {
-    if (index == 0) {
-      return;
-    }
-
-    Arrays.sort(set, 0, index);
-    for (int i = 0; i < index; i++) {
-      encoder.encode(set[i]);
-    }
-    encoder.close();
-    index = 0;
-
-    super.close();
-  }
-
-  @Override
-  public void encode(int value) throws IOException {
-    if (index == set.length) {
-      int[] newSet = new int[(int) (set.length * grow)];
-      System.arraycopy(set, 0, newSet, 0, set.length);
-      set = newSet;
-    }
-    set[index++] = value;
+  public void encode(IntsRef values, BytesRef buf) {
+    Arrays.sort(values.ints, values.offset, values.offset + values.length);
+    encoder.encode(values, buf);
   }
 
   @Override
@@ -72,14 +47,8 @@ public class SortingIntEncoder extends I
   }
   
   @Override
-  public void reInit(OutputStream out) {
-    super.reInit(out);
-    index = 0;
-  }
-
-  @Override
   public String toString() {
-    return "Sorting (" + encoder.toString() + ")";
+    return "Sorting(" + encoder.toString() + ")";
   }
   
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java Fri Jan 18 18:30:54 2013
@@ -1,7 +1,7 @@
 package org.apache.lucene.util.encoding;
 
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,7 +22,7 @@ import java.io.OutputStream;
 
 /**
  * An {@link IntEncoderFilter} which ensures only unique values are encoded. The
- * implementation assumes the values given to {@link #encode(int)} are sorted.
+ * implementation assumes the values given to {@link #encode(IntsRef, BytesRef)} are sorted.
  * If this is not the case, you can chain this encoder with
  * {@link SortingIntEncoder}.
  * 
@@ -30,26 +30,24 @@ import java.io.OutputStream;
  */
 public final class UniqueValuesIntEncoder extends IntEncoderFilter {
 
-  /**
-   * Denotes an illegal value which we can use to init 'prev' to. Since all
-   * encoded values are integers, this value is init to MAX_INT+1 and is of type
-   * long. Therefore we are guaranteed not to get this value in encode.
-   */
-  private static final long ILLEGAL_VALUE = Integer.MAX_VALUE + 1;
-
-  private long prev = ILLEGAL_VALUE;
-  
   /** Constructs a new instance with the given encoder. */
   public UniqueValuesIntEncoder(IntEncoder encoder) {
     super(encoder);
   }
 
   @Override
-  public void encode(int value) throws IOException {
-    if (prev != value) {
-      encoder.encode(value);
-      prev = value;
+  public void encode(IntsRef values, BytesRef buf) {
+    int prev = values.ints[values.offset];
+    int idx = values.offset + 1;
+    int upto = values.offset + values.length;
+    for (int i = idx; i < upto; i++) {
+      if (values.ints[i] != prev) {
+        values.ints[idx++] = values.ints[i];
+        prev = values.ints[i];
+      }
     }
+    values.length = idx - values.offset;
+    encoder.encode(values, buf);
   }
 
   @Override
@@ -58,14 +56,8 @@ public final class UniqueValuesIntEncode
   }
   
   @Override
-  public void reInit(OutputStream out) {
-    super.reInit(out);
-    prev = ILLEGAL_VALUE;
-  }
-
-  @Override
   public String toString() {
-    return "Unique (" + encoder.toString() + ")";
+    return "Unique(" + encoder.toString() + ")";
   }
   
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java Fri Jan 18 18:30:54 2013
@@ -1,6 +1,9 @@
 package org.apache.lucene.util.encoding;
 
-import java.io.IOException;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,33 +23,36 @@ import java.io.IOException;
  */
 
 /**
- * An {@link IntDecoder} which can decode values encoded by
- * {@link VInt8IntEncoder}.
+ * Decodes values encoded by {@link VInt8IntEncoder}.
  * 
  * @lucene.experimental
  */
-public class VInt8IntDecoder extends IntDecoder {
-
-  private boolean legalEOS = true;
+public final class VInt8IntDecoder extends IntDecoder {
 
   @Override
-  public long decode() throws IOException {
+  public void decode(BytesRef buf, IntsRef values) {
+    values.offset = values.length = 0;
+
+    // grow the buffer up front, even if by a large number of values (buf.length)
+    // that saves the need to check inside the loop for every decoded value if
+    // the buffer needs to grow.
+    if (values.ints.length < buf.length) {
+      values.ints = new int[ArrayUtil.oversize(buf.length, RamUsageEstimator.NUM_BYTES_INT)];
+    }
+
+    // it is better if the decoding is inlined like so, and not e.g.
+    // in a utility method
+    int upto = buf.offset + buf.length;
     int value = 0;
-    while (true) {
-      int first = in.read();
-      if (first < 0) {
-        if (!legalEOS) {
-          throw new IOException("Unexpected End-Of-Stream");
-        }
-        return EOS;
-      }
-      value |= first & 0x7F;
-      if ((first & 0x80) == 0) {
-        legalEOS = true;
-        return value;
+    int offset = buf.offset;
+    while (offset < upto) {
+      byte b = buf.bytes[offset++];
+      if (b >= 0) {
+        values.ints[values.length++] = (value << 7) | b;
+        value = 0;
+      } else {
+        value = (value << 7) | (b & 0x7F);
       }
-      legalEOS = false;
-      value <<= 7;
     }
   }
 

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java Fri Jan 18 18:30:54 2013
@@ -1,6 +1,7 @@
 package org.apache.lucene.util.encoding;
 
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -46,30 +47,47 @@ import java.io.IOException;
  * 
  * @lucene.experimental
  */
-public class VInt8IntEncoder extends IntEncoder {
+public final class VInt8IntEncoder extends IntEncoder {
 
   @Override
-  public void encode(int value) throws IOException {
-    if ((value & ~0x7F) == 0) {
-      out.write(value);
-    } else if ((value & ~0x3FFF) == 0) {
-      out.write(0x80 | (value >> 7));
-      out.write(0x7F & value);
-    } else if ((value & ~0x1FFFFF) == 0) {
-      out.write(0x80 | (value >> 14));
-      out.write(0x80 | (value >> 7));
-      out.write(0x7F & value);
-    } else if ((value & ~0xFFFFFFF) == 0) {
-      out.write(0x80 | (value >> 21));
-      out.write(0x80 | (value >> 14));
-      out.write(0x80 | (value >> 7));
-      out.write(0x7F & value);
-    } else {
-      out.write(0x80 | (value >> 28));
-      out.write(0x80 | (value >> 21));
-      out.write(0x80 | (value >> 14));
-      out.write(0x80 | (value >> 7));
-      out.write(0x7F & value);
+  public void encode(IntsRef values, BytesRef buf) {
+    buf.offset = buf.length = 0;
+    int maxBytesNeeded = 5 * values.length; // at most 5 bytes per VInt
+    if (buf.bytes.length < maxBytesNeeded) {
+      buf.grow(maxBytesNeeded);
+    }
+    
+    int upto = values.offset + values.length;
+    for (int i = values.offset; i < upto; i++) {
+      // it is better if the encoding is inlined like so, and not e.g.
+      // in a utility method
+      int value = values.ints[i];
+      if ((value & ~0x7F) == 0) {
+        buf.bytes[buf.length] = (byte) value;
+        buf.length++;
+      } else if ((value & ~0x3FFF) == 0) {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 1] = (byte) (value & 0x7F);
+        buf.length += 2;
+      } else if ((value & ~0x1FFFFF) == 0) {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+        buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 2] = (byte) (value & 0x7F);
+        buf.length += 3;
+      } else if ((value & ~0xFFFFFFF) == 0) {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+        buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+        buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 3] = (byte) (value & 0x7F);
+        buf.length += 4;
+      } else {
+        buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xF0000000) >> 28));
+        buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+        buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+        buf.bytes[buf.length + 3] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+        buf.bytes[buf.length + 4] = (byte) (value & 0x7F);
+        buf.length += 5;
+      }
     }
   }
 

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html Fri Jan 18 18:30:54 2013
@@ -25,49 +25,8 @@ mechanisms to create new ones. The super
 encoders there is a matching {@link
 org.apache.lucene.util.encoding.IntDecoder} implementation (not all
 encoders need a decoder).
-<p>An encoder encodes the integers that are passed to {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} into a
-set output stream (see {@link
-org.apache.lucene.util.encoding.IntEncoder#reInit(OutputStream)
-reInit}). One should always call {@link
-org.apache.lucene.util.encoding.IntEncoder#close() close} when all
-integers have been encoded, to ensure proper finish by the encoder. Some
-encoders buffer values in-memory and encode in batches in order to
-optimize the encoding, and not closing them may result in loss of
-information or corrupt stream.
-<p>A proper and typical usage of an encoder looks like this:
-<blockquote><pre class="prettyprint"><code>
-int[] data = &lt;the values to encode&gt;
-IntEncoder encoder = new VInt8IntEncoder();
-OutputStream out = new ByteArrayOutputStream();
-encoder.reInit(out);
-for (int val : data) {
-  encoder.encode(val);
-}
-encoder.close();
-
-// Print the bytes in binary
-byte[] bytes = out.toByteArray();
-for (byte b : bytes) {
-  System.out.println(Integer.toBinaryString(b));
-}
-</code></pre></blockquote>
-Each encoder also implements {@link
-org.apache.lucene.util.encoding.IntEncoder#createMatchingDecoder()
-createMatchingDecoder} which returns the matching decoder for this encoder.
-As mentioned above, not all encoders have a matching decoder (like some
-encoder filters which are explained next), however every encoder should
-return a decoder following a call to that method. To complete the
-example above, one can easily iterate over the decoded values like this:
-<blockquote><pre class="prettyprint"><code>
-IntDecoder d = e.createMatchingDecoder();
-d.reInit(new ByteArrayInputStream(bytes));
-long val;
-while ((val = d.decode()) != IntDecoder.EOS) {
-  System.out.println(val);
-}
-</code></pre></blockquote>
-<p>Some encoders don't perform any encoding at all, or do not include an
+<p>
+Some encoders don't perform any encoding at all, or do not include an
 encoding logic. Those are called {@link
 org.apache.lucene.util.encoding.IntEncoderFilter}s. A filter is an
 encoder which delegates the encoding task to a given encoder, however
@@ -76,91 +35,6 @@ example is {@link org.apache.lucene.util
 which encodes the gaps between values rather than the values themselves.
 Another example is {@link
 org.apache.lucene.util.encoding.SortingIntEncoder} which sorts all the
-values in ascending order before they are sent for encoding. This
-encoder aggregates the values in its {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} implementation 
-and decoding only happens upon calling {@link
-org.apache.lucene.util.encoding.IntEncoder#close() close}.
-<h4>Extending IntEncoder</h4>
-Extending {@link org.apache.lucene.util.encoding.IntEncoder} is a very
-easy task. One only needs to implement {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} and
-{@link org.apache.lucene.util.encoding.IntEncoder#createMatchingDecoder()
-createMatchingDecoder} as the base implementation takes care of
-re-initializing the output stream and closing it. The following example
-illustrates how can one write an encoder (and a matching decoder) which
-'tags' the stream with type/ID of the encoder. Such tagging is important
-in scenarios where an application uses different encoders for different
-streams, and wants to manage some sort of mapping between an encoder ID
-to an IntEncoder/Decoder implementation, so a proper decoder will be
-initialized on the fly:
-<blockquote><pre class="prettyprint"><code>
-public class TaggingIntEncoder extends IntEncoderFilter {
-  
-  public TaggingIntEncoder(IntEncoder encoder) {
-    super(encoder);
-  }
-  
-  &#64;Override
-  public void encode(int value) throws IOException {
-    encoder.encode(value);
-  }
-
-  &#64;Override
-  public IntDecoder createMatchingDecoder() {
-    return new TaggingIntDecoder();
-  }
-	
-  &#64;Override
-  public void reInit(OutputStream out) {
-    super.reInit(os);
-    // Assumes the application has a static EncodersMap class which is able to 
-    // return a unique ID for a given encoder.
-    int encoderID = EncodersMap.getID(encoder);
-    this.out.write(encoderID);
-  }
-
-  &#64;Override
-  public String toString() {
-    return "Tagging (" + encoder.toString() + ")";
-  }
-
-}
-</code></pre></blockquote>
-And the matching decoder:
-<blockquote><pre class="prettyprint"><code>
-public class TaggingIntDecoder extends IntDecoder {
-  
-  // Will be initialized upon calling reInit.
-  private IntDecoder decoder;
-  
-  &#64;Override
-  public void reInit(InputStream in) {
-    super.reInit(in);
-    
-    // Read the ID of the encoder that tagged this stream.
-    int encoderID = in.read();
-    
-    // Assumes EncodersMap can return the proper IntEncoder given the ID.
-    decoder = EncodersMap.getEncoder(encoderID).createMatchingDecoder();
-  }
-	
-  &#64;Override
-  public long decode() throws IOException {
-    return decoder.decode();
-  }
-
-  &#64;Override
-  public String toString() {
-    return "Tagging (" + decoder == null ? "none" : decoder.toString() + ")";
-  }
-
-}
-</code></pre></blockquote>
-The example implements <code>TaggingIntEncoder</code> as a filter over another 
-encoder. Even though it does not do any filtering on the actual values, it feels 
-right to present it as a filter. Anyway, this is just an example code and one 
-can choose to implement it however it makes sense to the application. For 
-simplicity, error checking was omitted from the sample code.
+values in ascending order before they are sent for encoding.
 </body>
 </html>
\ No newline at end of file

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java Fri Jan 18 18:30:54 2013
@@ -15,6 +15,18 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.facet.index.FacetFields;
+import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
+import org.apache.lucene.facet.search.params.FacetRequest;
+import org.apache.lucene.facet.search.params.FacetSearchParams;
+import org.apache.lucene.facet.search.results.FacetResult;
+import org.apache.lucene.facet.search.results.FacetResultNode;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexReader;
@@ -28,24 +40,11 @@ import org.apache.lucene.index.TermsEnum
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.Directory;
-
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.facet.index.CategoryDocumentBuilder;
-import org.apache.lucene.facet.index.params.CategoryListParams;
-import org.apache.lucene.facet.index.params.DefaultFacetIndexingParams;
-import org.apache.lucene.facet.index.params.FacetIndexingParams;
-import org.apache.lucene.facet.search.params.FacetRequest;
-import org.apache.lucene.facet.search.params.FacetSearchParams;
-import org.apache.lucene.facet.search.results.FacetResult;
-import org.apache.lucene.facet.search.results.FacetResultNode;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
-import org.apache.lucene.facet.taxonomy.TaxonomyReader;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -66,7 +65,7 @@ import org.junit.BeforeClass;
  * limitations under the License.
  */
 
-/** Base faceted search test. */
+@SuppressCodecs({"SimpleText"})
 public abstract class FacetTestBase extends LuceneTestCase {
   
   /** Holds a search and taxonomy Directories pair. */
@@ -184,29 +183,28 @@ public abstract class FacetTestBase exte
 
   /** Returns a default facet indexing params */
   protected FacetIndexingParams getFacetIndexingParams(final int partSize) {
-    return new DefaultFacetIndexingParams() {
+    return new FacetIndexingParams() {
       @Override
-      protected int fixedPartitionSize() {
+      public int getPartitionSize() {
         return partSize;
       }
     };
   }
   
   /**
-   * Faceted Search Params for the test.
-   * Sub classes should override in order to test with different faceted search params.
+   * Faceted Search Params for the test. Sub classes should override in order to
+   * test with different faceted search params.
    */
-  protected FacetSearchParams getFacetedSearchParams() {
-    return getFacetedSearchParams(Integer.MAX_VALUE);
+  protected FacetSearchParams getFacetSearchParams(FacetIndexingParams iParams, FacetRequest... facetRequests) {
+    return new FacetSearchParams(Arrays.asList(facetRequests), iParams);
   }
 
   /**
-   * Faceted Search Params with specified partition size.
-   * @see #getFacetedSearchParams()
+   * Faceted Search Params for the test. Sub classes should override in order to
+   * test with different faceted search params.
    */
-  protected FacetSearchParams getFacetedSearchParams(int partitionSize) {
-    FacetSearchParams res = new FacetSearchParams(getFacetIndexingParams(partitionSize));
-    return res; 
+  protected FacetSearchParams getFacetSearchParams(List<FacetRequest> facetRequests, FacetIndexingParams iParams) {
+    return new FacetSearchParams(facetRequests, iParams);
   }
 
   /**
@@ -258,9 +256,8 @@ public abstract class FacetTestBase exte
   protected final void indexDoc(FacetIndexingParams iParams, RandomIndexWriter iw,
       TaxonomyWriter tw, String content, List<CategoryPath> categories) throws IOException {
     Document d = new Document();
-    CategoryDocumentBuilder builder = new CategoryDocumentBuilder(tw, iParams);
-    builder.setCategoryPaths(categories);
-    builder.build(d);
+    FacetFields facetFields = new FacetFields(tw, iParams);
+    facetFields.addFields(d, categories);
     d.add(new TextField("content", content, Field.Store.YES));
     iw.addDocument(d);
   }
@@ -270,13 +267,12 @@ public abstract class FacetTestBase exte
     FacetIndexingParams iParams = getFacetIndexingParams(Integer.MAX_VALUE);
     String delim = String.valueOf(iParams.getFacetDelimChar());
     Map<CategoryPath, Integer> res = new HashMap<CategoryPath, Integer>();
-    HashSet<Term> handledTerms = new HashSet<Term>();
+    HashSet<String> handledTerms = new HashSet<String>();
     for (CategoryListParams clp : iParams.getAllCategoryListParams()) {
-      Term baseTerm = new Term(clp.getTerm().field());
-      if (!handledTerms.add(baseTerm)) {
+      if (!handledTerms.add(clp.field)) {
         continue; // already handled this term (for another list) 
       }
-      Terms terms = MultiFields.getTerms(indexReader, baseTerm.field());
+      Terms terms = MultiFields.getTerms(indexReader, clp.field);
       if (terms == null) {
         continue;
       }
@@ -284,7 +280,7 @@ public abstract class FacetTestBase exte
       TermsEnum te = terms.iterator(null);
       DocsEnum de = null;
       while (te.next() != null) {
-        de = _TestUtil.docs(random(), te, liveDocs, de, 0);
+        de = _TestUtil.docs(random(), te, liveDocs, de, DocsEnum.FLAG_NONE);
         int cnt = 0;
         while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
           cnt++;
@@ -301,7 +297,7 @@ public abstract class FacetTestBase exte
       FacetResultNode topResNode = fr.getFacetResultNode();
       FacetRequest freq = fr.getFacetRequest();
       if (VERBOSE) {
-        System.out.println(freq.getCategoryPath().toString()+ "\t\t" + topResNode);
+        System.out.println(freq.categoryPath.toString()+ "\t\t" + topResNode);
       }
       assertCountsAndCardinality(facetCountsTruth, topResNode, freq.getNumResults());
     }
@@ -321,14 +317,13 @@ public abstract class FacetTestBase exte
   }
   
   /** Validate results equality */
-  protected static void assertSameResults(List<FacetResult> expected,
-                                          List<FacetResult> actual) {
+  protected static void assertSameResults(List<FacetResult> expected, List<FacetResult> actual) {
     String expectedResults = resStringValueOnly(expected);
     String actualResults = resStringValueOnly(actual);
     if (!expectedResults.equals(actualResults)) {
       System.err.println("Results are not the same!");
       System.err.println("Expected:\n" + expectedResults);
-      System.err.println("Actual" + actualResults);
+      System.err.println("Actual:\n" + actualResults);
       throw new NotSameResultError();
     }
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java Fri Jan 18 18:30:54 2013
@@ -2,26 +2,9 @@ package org.apache.lucene.facet;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
+import java.util.List;
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.store.Directory;
-
-import org.apache.lucene.search.MultiCollector;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.CategoryDocumentBuilder;
-import org.apache.lucene.facet.index.params.DefaultFacetIndexingParams;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.FacetsCollector;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
@@ -32,6 +15,16 @@ import org.apache.lucene.facet.taxonomy.
 import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
 import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
 import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MultiCollector;
+import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -91,19 +84,18 @@ public class FacetTestUtils {
   }
 
   public static Collector[] search(IndexSearcher searcher,
-      TaxonomyReader taxonomyReader, DefaultFacetIndexingParams iParams,
-      int k, String... facetNames) throws IOException {
+      TaxonomyReader taxonomyReader, FacetIndexingParams iParams, int k,
+      String... facetNames) throws IOException {
     
     Collector[] collectors = new Collector[2];
     
-    FacetSearchParams facetSearchParams = new FacetSearchParams(iParams);
-    Collection<FacetRequest> fRequests = new ArrayList<FacetRequest>();
+    List<FacetRequest> fRequests = new ArrayList<FacetRequest>();
     for (String facetName : facetNames) {
       CategoryPath cp = new CategoryPath(facetName);
       FacetRequest fq = new CountFacetRequest(cp, k);
-      facetSearchParams.addFacetRequest(fq);
       fRequests.add(fq);
     }
+    FacetSearchParams facetSearchParams = new FacetSearchParams(fRequests, iParams);
 
     TopScoreDocCollector topDocsCollector = TopScoreDocCollector.create(
         searcher.getIndexReader().maxDoc(), true);
@@ -118,17 +110,6 @@ public class FacetTestUtils {
     return collectors;
   }
   
-  public static void add(FacetIndexingParams iParams, RandomIndexWriter iw,
-      TaxonomyWriter tw, String... strings) throws IOException {
-    ArrayList<CategoryPath> cps = new ArrayList<CategoryPath>();
-    CategoryPath cp = new CategoryPath(strings);
-    cps.add(cp);
-    Document d = new Document();
-    new CategoryDocumentBuilder(tw, iParams).setCategoryPaths(cps).build(d);
-    d.add(new TextField("content", "alpha", Field.Store.YES));
-    iw.addDocument(d);
-  }
-
   public static class IndexTaxonomyReaderPair {
     public DirectoryReader indexReader;
     public DirectoryTaxonomyReader taxReader;

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java Fri Jan 18 18:30:54 2013
@@ -4,7 +4,7 @@ import org.junit.Test;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.facet.example.ExampleResult;
-import org.apache.lucene.facet.example.association.AssociationMain;
+import org.apache.lucene.facet.example.association.CategoryAssociationsMain;
 import org.apache.lucene.facet.search.results.FacetResultNode;
 
 /*
@@ -35,8 +35,8 @@ public class TestAssociationExample exte
 
   @Test
   public void testAssociationExamples() throws Exception {
-    assertExampleResult(new AssociationMain().runSumIntAssociationSample(), EXPECTED_INT_SUM_RESULTS);
-    assertExampleResult(new AssociationMain().runSumFloatAssociationSample(), EXPECTED_FLOAT_SUM_RESULTS);
+    assertExampleResult(new CategoryAssociationsMain().runSumIntAssociationSample(), EXPECTED_INT_SUM_RESULTS);
+    assertExampleResult(new CategoryAssociationsMain().runSumFloatAssociationSample(), EXPECTED_FLOAT_SUM_RESULTS);
   }
 
   private void assertExampleResult(ExampleResult res, double[] expectedResults) {

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java Fri Jan 18 18:30:54 2013
@@ -34,7 +34,7 @@ import org.apache.lucene.facet.search.re
 public class TestSimpleExample extends LuceneTestCase {
 
   @Test
-  public void testSimple () throws Exception {
+  public void testSimple() throws Exception {
     ExampleResult res = new SimpleMain().runSimple();
     assertNotNull("Null result!", res);
     assertNotNull("Null facet result!", res.getFacetResults());
@@ -47,7 +47,7 @@ public class TestSimpleExample extends L
    * As result, facets that without drill down got count of 2 will now get a count of 1. 
    */
   @Test
-  public void testDrillDown () throws Exception {
+  public void testDrillDown() throws Exception {
     ExampleResult res = new SimpleMain().runDrillDown();
     assertNotNull("Null result!", res);
     assertNotNull("Null facet result!", res.getFacetResults());

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java Fri Jan 18 18:30:54 2013
@@ -1,16 +1,12 @@
 package org.apache.lucene.facet.index.categorypolicy;
 
-import org.apache.lucene.store.Directory;
-import org.junit.Test;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.categorypolicy.DefaultOrdinalPolicy;
-import org.apache.lucene.facet.index.categorypolicy.NonTopLevelOrdinalPolicy;
-import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
 import org.apache.lucene.facet.taxonomy.TaxonomyReader;
 import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
 import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -34,13 +30,11 @@ public class OrdinalPolicyTest extends L
   @Test
   public void testDefaultOrdinalPolicy() {
     // check ordinal policy
-    OrdinalPolicy ordinalPolicy = new DefaultOrdinalPolicy();
-    assertFalse("default ordinal policy should not match root", ordinalPolicy
-        .shouldAdd(TaxonomyReader.ROOT_ORDINAL));
+    OrdinalPolicy ordinalPolicy = OrdinalPolicy.ALL_PARENTS;
+    assertFalse("default ordinal policy should not match root", ordinalPolicy.shouldAdd(TaxonomyReader.ROOT_ORDINAL));
     for (int i = 0; i < 300; i++) {
       int ordinal = 1 + random().nextInt(Integer.MAX_VALUE - 1);
-      assertTrue("default ordinal policy should match " + ordinal,
-          ordinalPolicy.shouldAdd(ordinal));
+      assertTrue("default ordinal policy should match " + ordinal, ordinalPolicy.shouldAdd(ordinal));
     }
   }
 
@@ -54,8 +48,7 @@ public class OrdinalPolicyTest extends L
     String[] topLevelStrings = new String[10];
     for (int i = 0; i < 10; i++) {
       topLevelStrings[i] = Integer.valueOf(random().nextInt(30)).toString();
-      topLevelOrdinals[i] = taxonomy.addCategory(new CategoryPath(
-          topLevelStrings[i]));
+      topLevelOrdinals[i] = taxonomy.addCategory(new CategoryPath(topLevelStrings[i]));
     }
     int[] nonTopLevelOrdinals = new int[300];
     for (int i = 0; i < 300; i++) {
@@ -65,22 +58,18 @@ public class OrdinalPolicyTest extends L
       for (int j = 1; j < components.length; j++) {
         components[j] = (Integer.valueOf(random().nextInt(30))).toString();
       }
-      nonTopLevelOrdinals[i] = taxonomy.addCategory(new CategoryPath(
-          components));
+      nonTopLevelOrdinals[i] = taxonomy.addCategory(new CategoryPath(components));
     }
     // check ordinal policy
     OrdinalPolicy ordinalPolicy = new NonTopLevelOrdinalPolicy();
     ordinalPolicy.init(taxonomy);
-    assertFalse("top level ordinal policy should not match root", ordinalPolicy
-        .shouldAdd(TaxonomyReader.ROOT_ORDINAL));
+    assertFalse("top level ordinal policy should not match root", ordinalPolicy.shouldAdd(TaxonomyReader.ROOT_ORDINAL));
     for (int i = 0; i < 10; i++) {
-      assertFalse("top level ordinal policy should not match "
-          + topLevelOrdinals[i],
+      assertFalse("top level ordinal policy should not match " + topLevelOrdinals[i], 
           ordinalPolicy.shouldAdd(topLevelOrdinals[i]));
     }
     for (int i = 0; i < 300; i++) {
-      assertTrue("top level ordinal policy should match "
-          + nonTopLevelOrdinals[i],
+      assertTrue("top level ordinal policy should match " + nonTopLevelOrdinals[i],
           ordinalPolicy.shouldAdd(nonTopLevelOrdinals[i]));
     }
 

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java Fri Jan 18 18:30:54 2013
@@ -1,15 +1,11 @@
 package org.apache.lucene.facet.index.categorypolicy;
 
-import org.apache.lucene.store.Directory;
-import org.junit.Test;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.categorypolicy.DefaultPathPolicy;
-import org.apache.lucene.facet.index.categorypolicy.NonTopLevelPathPolicy;
-import org.apache.lucene.facet.index.categorypolicy.PathPolicy;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
 import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
 import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -33,10 +29,9 @@ public class PathPolicyTest extends Luce
   @Test
   public void testDefaultPathPolicy() {
     // check path policy
-    CategoryPath cp = new CategoryPath();
-    PathPolicy pathPolicy = new DefaultPathPolicy();
-    assertFalse("default path policy should not accept root", 
-        pathPolicy.shouldAdd(cp));
+    CategoryPath cp = CategoryPath.EMPTY;
+    PathPolicy pathPolicy = PathPolicy.ALL_CATEGORIES;
+    assertFalse("default path policy should not accept root", pathPolicy.shouldAdd(cp));
     for (int i = 0; i < 300; i++) {
       int nComponents = 1 + random().nextInt(10);
       String[] components = new String[nComponents];
@@ -44,9 +39,7 @@ public class PathPolicyTest extends Luce
         components[j] = (Integer.valueOf(random().nextInt(30))).toString();
       }
       cp = new CategoryPath(components);
-      assertTrue("default path policy should accept "
-          + cp.toString('/'),
-          pathPolicy.shouldAdd(cp));
+      assertTrue("default path policy should accept " + cp.toString('/'), pathPolicy.shouldAdd(cp));
     }
   }
 
@@ -78,7 +71,7 @@ public class PathPolicyTest extends Luce
     // check ordinal policy
     PathPolicy pathPolicy = new NonTopLevelPathPolicy();
     assertFalse("top level path policy should not match root",
-        pathPolicy.shouldAdd(new CategoryPath()));
+        pathPolicy.shouldAdd(CategoryPath.EMPTY));
     for (int i = 0; i < 10; i++) {
       assertFalse("top level path policy should not match "
           + topLevelPaths[i],

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java Fri Jan 18 18:30:54 2013
@@ -1,10 +1,12 @@
 package org.apache.lucene.facet.index.params;
 
-import org.apache.lucene.index.Term;
-import org.junit.Test;
-
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.util.encoding.DGapVInt8IntEncoder;
+import org.apache.lucene.util.encoding.IntDecoder;
+import org.apache.lucene.util.encoding.IntEncoder;
+import org.apache.lucene.util.encoding.SortingIntEncoder;
+import org.apache.lucene.util.encoding.UniqueValuesIntEncoder;
+import org.junit.Test;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -28,9 +30,11 @@ public class CategoryListParamsTest exte
   @Test
   public void testDefaultSettings() {
     CategoryListParams clp = new CategoryListParams();
-    assertEquals("wrong default term", new Term("$facets", "$fulltree$"), clp.getTerm());
-    assertEquals("unexpected default encoder", "Sorting (Unique (DGap (VInt8)))", clp.createEncoder().toString());
-    assertEquals("unexpected default decoder", "DGap (VInt8)", clp.createEncoder().createMatchingDecoder().toString());
+    assertEquals("wrong default field", "$facets", clp.field);
+    IntEncoder encoder = new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapVInt8IntEncoder()));
+    IntDecoder decoder = encoder.createMatchingDecoder();
+    assertEquals("unexpected default encoder", encoder.toString(), clp.createEncoder().toString());
+    assertEquals("unexpected default decoder", decoder.toString(), clp.createEncoder().createMatchingDecoder().toString());
   }
   
   /**
@@ -64,8 +68,8 @@ public class CategoryListParamsTest exte
         clParams1.hashCode(), clParams2.hashCode());
 
     // Test 2 CategoryListParams with the same specified Term
-    clParams1 = new CategoryListParams(new Term("test"));
-    clParams2 = new CategoryListParams(new Term("test"));
+    clParams1 = new CategoryListParams("test");
+    clParams2 = new CategoryListParams("test");
     assertEquals(
         "2 CategoryListParams with the same term should equal each other.",
         clParams1, clParams2);
@@ -73,8 +77,8 @@ public class CategoryListParamsTest exte
         clParams1.hashCode(), clParams2.hashCode());
     
     // Test 2 CategoryListParams with DIFFERENT terms
-    clParams1 = new CategoryListParams(new Term("test1"));
-    clParams2 = new CategoryListParams(new Term("test2"));
+    clParams1 = new CategoryListParams("test1");
+    clParams2 = new CategoryListParams("test2");
     assertFalse(
         "2 CategoryListParams with the different terms should NOT equal each other.",
         clParams1.equals(clParams2));

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java Fri Jan 18 18:30:54 2013
@@ -1,15 +1,13 @@
 package org.apache.lucene.facet.index.params;
 
-import org.apache.lucene.index.Term;
-import org.junit.Test;
+import java.util.Collections;
 
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.params.CategoryListParams;
-import org.apache.lucene.facet.index.params.FacetIndexingParams;
-import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
 import org.apache.lucene.facet.search.DrillDown;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
 import org.apache.lucene.facet.util.PartitionsUtils;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -32,50 +30,33 @@ public class PerDimensionIndexingParamsT
 
   @Test
   public void testTopLevelSettings() {
-    FacetIndexingParams ifip = new PerDimensionIndexingParams();
-    assertNotNull("Missing default category list", ifip
-        .getAllCategoryListParams());
-    assertEquals(
-        "Expected default category list term is $facets:$fulltree$",
-        new Term("$facets", "$fulltree$"), ifip.getCategoryListParams(
-            null).getTerm());
-    String expectedDDText = "a"
-        + ifip.getFacetDelimChar() + "b";
+    FacetIndexingParams ifip = new PerDimensionIndexingParams(Collections.<CategoryPath, CategoryListParams>emptyMap());
+    assertNotNull("Missing default category list", ifip.getAllCategoryListParams());
+    assertEquals("Expected default category list field is $facets", "$facets", ifip.getCategoryListParams(null).field);
+    String expectedDDText = "a" + ifip.getFacetDelimChar() + "b";
     CategoryPath cp = new CategoryPath("a", "b");
-    assertEquals("wrong drill-down term", new Term("$facets",
-        expectedDDText), DrillDown.term(ifip,cp));
+    assertEquals("wrong drill-down term", new Term("$facets", expectedDDText), DrillDown.term(ifip,cp));
     char[] buf = new char[20];
     int numchars = ifip.drillDownTermText(cp, buf);
     assertEquals("3 characters should be written", 3, numchars);
-    assertEquals("wrong drill-down term text", expectedDDText, new String(
-        buf, 0, numchars));
+    assertEquals("wrong drill-down term text", expectedDDText, new String(buf, 0, numchars));
     
     CategoryListParams clParams = ifip.getCategoryListParams(null);
-    assertEquals("partition for all ordinals is the first", "$fulltree$", 
-        PartitionsUtils.partitionNameByOrdinal(ifip, clParams , 250));
-    assertEquals("for partition 0, the same name should be returned",
-        "$fulltree$", PartitionsUtils.partitionName(clParams, 0));
-    assertEquals(
-        "for any other, it's the concatenation of name + partition",
-        "$fulltree$1", PartitionsUtils.partitionName(clParams, 1));
-    assertEquals("default partition number is always 0", 0, 
-        PartitionsUtils.partitionNumber(ifip,100));
-    
-    assertEquals("default partition size is unbounded", Integer.MAX_VALUE,
-        ifip.getPartitionSize());
+    assertEquals("partition for all ordinals is the first", "", PartitionsUtils.partitionNameByOrdinal(ifip, 250));
+    assertEquals("for partition 0, the same name should be returned", "", PartitionsUtils.partitionName(0));
+    assertEquals("for any other, it's the concatenation of name + partition", PartitionsUtils.PART_NAME_PREFIX + "1", PartitionsUtils.partitionName(1));
+    assertEquals("default partition number is always 0", 0, PartitionsUtils.partitionNumber(ifip,100));
+    assertEquals("default partition size is unbounded", Integer.MAX_VALUE, ifip.getPartitionSize());
   }
 
   @Test
   public void testCategoryListParamsAddition() {
-    PerDimensionIndexingParams tlfip = new PerDimensionIndexingParams();
-    CategoryListParams clp = new CategoryListParams(
-        new Term("clp", "value"));
-    tlfip.addCategoryListParams(new CategoryPath("a"), clp);
-    assertEquals("Expected category list term is " + clp.getTerm(), clp
-        .getTerm(), tlfip.getCategoryListParams(new CategoryPath("a"))
-        .getTerm());
-    assertNotSame("Unexpected default category list " + clp.getTerm(), clp,
-        tlfip.getCategoryListParams(null));
+    CategoryListParams clp = new CategoryListParams("clp");
+    PerDimensionIndexingParams tlfip = new PerDimensionIndexingParams(
+        Collections.<CategoryPath,CategoryListParams> singletonMap(new CategoryPath("a"), clp));
+    assertEquals("Expected category list field is " + clp.field, 
+        clp.field, tlfip.getCategoryListParams(new CategoryPath("a")).field);
+    assertNotSame("Unexpected default category list " + clp.field, clp, tlfip.getCategoryListParams(null));
   }
 
 }
\ No newline at end of file

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java Fri Jan 18 18:30:54 2013
@@ -29,12 +29,11 @@ import org.apache.lucene.facet.taxonomy.
 public class AdaptiveAccumulatorTest extends BaseSampleTestTopK {
 
   @Override
-  protected FacetsAccumulator getSamplingAccumulator(Sampler sampler,
-      TaxonomyReader taxoReader, IndexReader indexReader,
-      FacetSearchParams searchParams) {
-    AdaptiveFacetsAccumulator res = new AdaptiveFacetsAccumulator(searchParams,
-        indexReader, taxoReader);
+  protected FacetsAccumulator getSamplingAccumulator(Sampler sampler, TaxonomyReader taxoReader, 
+      IndexReader indexReader, FacetSearchParams searchParams) {
+    AdaptiveFacetsAccumulator res = new AdaptiveFacetsAccumulator(searchParams, indexReader, taxoReader);
     res.setSampler(sampler);
     return res;
   }
+  
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java Fri Jan 18 18:30:54 2013
@@ -1,6 +1,7 @@
 package org.apache.lucene.facet.search;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -12,6 +13,7 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.facet.FacetTestBase;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
+import org.apache.lucene.facet.search.params.FacetRequest;
 import org.apache.lucene.facet.search.params.FacetSearchParams;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
 import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
@@ -61,6 +63,10 @@ public abstract class BaseTestTopK exten
       return;
     }
     currDoc = doc;
+    // the code below tries to achieve non-uniform distribution of
+    // categories. Perhaps we can use random.nextGaussian() instead,
+    // something like nextGaussian() * stdev + maxCategory/2. Or
+    // try to generate a Zipf distribution.
     nextInt = random().nextInt(categoriesPow2);
     nextInt = (int)Math.sqrt(nextInt);
   }
@@ -93,12 +99,12 @@ public abstract class BaseTestTopK exten
   }
   
   protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
-    FacetSearchParams res = getFacetedSearchParams(partitionSize);
-    res.addFacetRequest(new CountFacetRequest(new CategoryPath("a"), numResults));
-    res.addFacetRequest(new CountFacetRequest(new CategoryPath("a", "1"), numResults));
-    res.addFacetRequest(new CountFacetRequest(new CategoryPath("a", "1", "10"), numResults));
-    res.addFacetRequest(new CountFacetRequest(new CategoryPath("a", "2",  "26", "267"), numResults));
-    return res;
+    List<FacetRequest> facetRequests = new ArrayList<FacetRequest>();
+    facetRequests.add(new CountFacetRequest(new CategoryPath("a"), numResults));
+    facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1"), numResults));
+    facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1", "10"), numResults));
+    facetRequests.add(new CountFacetRequest(new CategoryPath("a", "2",  "26", "267"), numResults));
+    return getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
   }
 
   @Override

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java Fri Jan 18 18:30:54 2013
@@ -1,26 +1,19 @@
 package org.apache.lucene.facet.search;
 
-import java.io.IOException;
-import java.io.Reader;
 import java.util.HashSet;
 import java.util.Set;
 
-import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.StraightBytesDocValuesField;
+import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.UnsafeByteArrayOutputStream;
 import org.apache.lucene.util.encoding.DGapIntEncoder;
 import org.apache.lucene.util.encoding.IntEncoder;
 import org.apache.lucene.util.encoding.SortingIntEncoder;
@@ -47,113 +40,69 @@ import org.junit.Test;
 
 public class CategoryListIteratorTest extends LuceneTestCase {
 
-  private static final class DataTokenStream extends TokenStream {
-
-    private int idx;
-    private PayloadAttribute payload = addAttribute(PayloadAttribute.class);
-    private byte[] buf = new byte[20];
-    UnsafeByteArrayOutputStream ubaos = new UnsafeByteArrayOutputStream(buf);
-    IntEncoder encoder;
-    private boolean exhausted = false;
-    private CharTermAttribute term = addAttribute(CharTermAttribute.class);
-
-    public DataTokenStream(String text, IntEncoder encoder) {
-      this.encoder = encoder;
-      term.setEmpty().append(text);
-    }
-
-    public void setIdx(int idx) {
-      this.idx = idx;
-      exhausted = false;
-    }
-
-    @Override
-    public boolean incrementToken() throws IOException {
-      if (exhausted) {
-        return false;
-      }
-
-      int[] values = data[idx];
-      ubaos.reInit(buf);
-      encoder.reInit(ubaos);
-      for (int val : values) {
-        encoder.encode(val);
-      }
-      encoder.close();
-      payload.setPayload(new BytesRef(buf, 0, ubaos.length()));
-
-      exhausted = true;
-      return true;
-    }
-
-  }
-
-  static final int[][] data = new int[][] {
-    new int[] { 1, 2 }, new int[] { 3, 4 }, new int[] { 1, 3 }, new int[] { 1, 2, 3, 4 },
+  static final IntsRef[] data = new IntsRef[] {
+    new IntsRef(new int[] { 1, 2 }, 0, 2), 
+    new IntsRef(new int[] { 3, 4 }, 0, 2),
+    new IntsRef(new int[] { 1, 3 }, 0, 2),
+    new IntsRef(new int[] { 1, 2, 3, 4 }, 0, 4)
   };
 
   @Test
-  public void testPayloadIntDecodingIterator() throws Exception {
+  public void testPayloadCategoryListIteraor() throws Exception {
     Directory dir = newDirectory();
-    DataTokenStream dts = new DataTokenStream("1",new SortingIntEncoder(
-        new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
+    final IntEncoder encoder = new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder())));
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, 
         new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)).setMergePolicy(newLogMergePolicy()));
+    BytesRef buf = new BytesRef();
     for (int i = 0; i < data.length; i++) {
-      dts.setIdx(i);
       Document doc = new Document();
-      doc.add(new TextField("f", dts));
+      encoder.encode(IntsRef.deepCopyOf(data[i]), buf);
+      doc.add(new StraightBytesDocValuesField("f", buf));
       writer.addDocument(doc);
     }
     IndexReader reader = writer.getReader();
     writer.close();
 
-    CategoryListIterator cli = new PayloadIntDecodingIterator(reader, new Term(
-        "f","1"), dts.encoder.createMatchingDecoder());
-    cli.init();
     int totalCategories = 0;
-    for (int i = 0; i < data.length; i++) {
-      Set<Integer> values = new HashSet<Integer>();
-      for (int j = 0; j < data[i].length; j++) {
-        values.add(data[i][j]);
-      }
-      cli.skipTo(i);
-      long cat;
-      while ((cat = cli.nextCategory()) < Integer.MAX_VALUE) {
-        assertTrue("expected category not found: " + cat, values.contains((int) cat));
-        totalCategories ++;
+    IntsRef ordinals = new IntsRef();
+    CategoryListIterator cli = new DocValuesCategoryListIterator("f", encoder.createMatchingDecoder());
+    for (AtomicReaderContext context : reader.leaves()) {
+      assertTrue("failed to initalize iterator", cli.setNextReader(context));
+      int maxDoc = context.reader().maxDoc();
+      int dataIdx = context.docBase;
+      for (int doc = 0; doc < maxDoc; doc++, dataIdx++) {
+        Set<Integer> values = new HashSet<Integer>();
+        for (int j = 0; j < data[dataIdx].length; j++) {
+          values.add(data[dataIdx].ints[j]);
+        }
+        cli.getOrdinals(doc, ordinals);
+        assertTrue("no ordinals for document " + doc, ordinals.length > 0);
+        for (int j = 0; j < ordinals.length; j++) {
+          assertTrue("expected category not found: " + ordinals.ints[j], values.contains(ordinals.ints[j]));
+        }
+        totalCategories += ordinals.length;
       }
     }
-    assertEquals("Missing categories!",10,totalCategories);
+    assertEquals("Missing categories!", 10, totalCategories);
     reader.close();
     dir.close();
   }
 
-  /**
-   * Test that a document with no payloads does not confuse the payload decoder.
-   */
   @Test
   public void testPayloadIteratorWithInvalidDoc() throws Exception {
     Directory dir = newDirectory();
-    DataTokenStream dts = new DataTokenStream("1",new SortingIntEncoder(
-        new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
-    // this test requires that no payloads ever be randomly present!
-    final Analyzer noPayloadsAnalyzer = new Analyzer() {
-      @Override
-      public TokenStreamComponents createComponents(String fieldName, Reader reader) {
-        return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.KEYWORD, false));
-      }
-    };
+    final IntEncoder encoder = new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder())));
     // NOTE: test is wired to LogMP... because test relies on certain docids having payloads
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, 
-        newIndexWriterConfig(TEST_VERSION_CURRENT, noPayloadsAnalyzer).setMergePolicy(newLogMergePolicy()));
+        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
     for (int i = 0; i < data.length; i++) {
       Document doc = new Document();
       if (i == 0) {
-        dts.setIdx(i);
-        doc.add(new TextField("f", dts)); // only doc 0 has payloads!
+        BytesRef buf = new BytesRef();
+        encoder.encode(IntsRef.deepCopyOf(data[i]), buf );
+        doc.add(new StraightBytesDocValuesField("f", buf));
       } else {
-        doc.add(new TextField("f", "1", Field.Store.NO));
+        doc.add(new StraightBytesDocValuesField("f", new BytesRef()));
       }
       writer.addDocument(doc);
       writer.commit();
@@ -162,30 +111,31 @@ public class CategoryListIteratorTest ex
     IndexReader reader = writer.getReader();
     writer.close();
 
-    CategoryListIterator cli = new PayloadIntDecodingIterator(reader, new Term(
-        "f","1"), dts.encoder.createMatchingDecoder());
-    assertTrue("Failed to initialize payload iterator", cli.init());
-    int totalCats = 0;
-    for (int i = 0; i < data.length; i++) {
-      // doc no. i
-      Set<Integer> values = new HashSet<Integer>();
-      for (int j = 0; j < data[i].length; j++) {
-        values.add(data[i][j]);
-      }
-      boolean hasDoc = cli.skipTo(i);
-      if (hasDoc) {
-        assertTrue("Document " + i + " must not have a payload!", i == 0);
-        long cat;
-        while ((cat = cli.nextCategory()) < Integer.MAX_VALUE) {
-          assertTrue("expected category not found: " + cat, values.contains((int) cat));
-          ++totalCats;
+    int totalCategories = 0;
+    IntsRef ordinals = new IntsRef();
+    CategoryListIterator cli = new DocValuesCategoryListIterator("f", encoder.createMatchingDecoder());
+    for (AtomicReaderContext context : reader.leaves()) {
+      assertTrue("failed to initalize iterator", cli.setNextReader(context));
+      int maxDoc = context.reader().maxDoc();
+      int dataIdx = context.docBase;
+      for (int doc = 0; doc < maxDoc; doc++, dataIdx++) {
+        Set<Integer> values = new HashSet<Integer>();
+        for (int j = 0; j < data[dataIdx].length; j++) {
+          values.add(data[dataIdx].ints[j]);
+        }
+        cli.getOrdinals(doc, ordinals);
+        if (dataIdx == 0) {
+          assertTrue("document 0 must have ordinals", ordinals.length > 0);
+          for (int j = 0; j < ordinals.length; j++) {
+            assertTrue("expected category not found: " + ordinals.ints[j], values.contains(ordinals.ints[j]));
+          }
+          totalCategories += ordinals.length;
+        } else {
+          assertTrue("only document 0 should have ordinals", ordinals.length == 0);
         }
-      } else {
-        assertFalse("Document " + i + " must have a payload!", i == 0);
       }
-
     }
-    assertEquals("Wrong number of total categories!", 2, totalCats);
+    assertEquals("Wrong number of total categories!", 2, totalCategories);
 
     reader.close();
     dir.close();

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java Fri Jan 18 18:30:54 2013
@@ -2,34 +2,36 @@ package org.apache.lucene.facet.search;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.facet.index.FacetFields;
+import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
+import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.CategoryDocumentBuilder;
-import org.apache.lucene.facet.index.params.CategoryListParams;
-import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
-import org.apache.lucene.facet.search.params.FacetSearchParams;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -49,22 +51,18 @@ import org.apache.lucene.facet.taxonomy.
 
 public class DrillDownTest extends LuceneTestCase {
   
-  private FacetSearchParams defaultParams = new FacetSearchParams();
-  private FacetSearchParams nonDefaultParams;
+  private FacetIndexingParams defaultParams = FacetIndexingParams.ALL_PARENTS;
+  private PerDimensionIndexingParams nonDefaultParams;
   private static IndexReader reader;
   private static DirectoryTaxonomyReader taxo;
   private static Directory dir;
   private static Directory taxoDir;
   
   public DrillDownTest() {
-    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();
-    CategoryListParams aClParams = new CategoryListParams(new Term("testing_facets_a", "a"));
-    CategoryListParams bClParams = new CategoryListParams(new Term("testing_facets_b", "b"));
-    
-    iParams.addCategoryListParams(new CategoryPath("a"), aClParams);
-    iParams.addCategoryListParams(new CategoryPath("b"), bClParams);
-    
-    nonDefaultParams = new FacetSearchParams(iParams);
+    Map<CategoryPath,CategoryListParams> paramsMap = new HashMap<CategoryPath,CategoryListParams>();
+    paramsMap.put(new CategoryPath("a"), new CategoryListParams("testing_facets_a"));
+    paramsMap.put(new CategoryPath("b"), new CategoryListParams("testing_facets_b"));
+    nonDefaultParams = new PerDimensionIndexingParams(paramsMap);
   }
 
   @BeforeClass
@@ -91,8 +89,10 @@ public class DrillDownTest extends Lucen
       if (i % 5 == 0) { // 20
         paths.add(new CategoryPath("b"));
       }
-      CategoryDocumentBuilder builder = new CategoryDocumentBuilder(taxoWriter);
-      builder.setCategoryPaths(paths).build(doc);
+      FacetFields facetFields = new FacetFields(taxoWriter);
+      if (paths.size() > 0) {
+        facetFields.addFields(doc, paths);
+      }
       writer.addDocument(doc);
     }
     
@@ -113,8 +113,8 @@ public class DrillDownTest extends Lucen
   }
   
   @Test
-  public void testTermDefault() {
-    String defaultField = CategoryListParams.DEFAULT_TERM.field();
+  public void testDefaultField() {
+    String defaultField = CategoryListParams.DEFAULT_FIELD;
     
     Term termA = DrillDown.term(defaultParams, new CategoryPath("a"));
     assertEquals(new Term(defaultField, "a"), termA);
@@ -128,7 +128,7 @@ public class DrillDownTest extends Lucen
     IndexSearcher searcher = newSearcher(reader);
 
     // Making sure the query yields 25 documents with the facet "a"
-    Query q = DrillDown.query(defaultParams, new CategoryPath("a"));
+    Query q = DrillDown.query(defaultParams, null, new CategoryPath("a"));
     TopDocs docs = searcher.search(q, 100);
     assertEquals(25, docs.totalHits);
     
@@ -139,7 +139,7 @@ public class DrillDownTest extends Lucen
     assertEquals(5, docs.totalHits);
 
     // Making sure that a query of both facet "a" and facet "b" yields 5 results
-    Query q3 = DrillDown.query(defaultParams, new CategoryPath("a"), new CategoryPath("b"));
+    Query q3 = DrillDown.query(defaultParams, null, new CategoryPath("a"), new CategoryPath("b"));
     docs = searcher.search(q3, 100);
     assertEquals(5, docs.totalHits);
     
@@ -156,18 +156,18 @@ public class DrillDownTest extends Lucen
     IndexSearcher searcher = newSearcher(reader);
 
     // Create the base query to start with
-    Query q = DrillDown.query(defaultParams, new CategoryPath("a"));
+    Query q = DrillDown.query(defaultParams, null, new CategoryPath("a"));
     
     // Making sure the query yields 5 documents with the facet "b" and the
     // previous (facet "a") query as a base query
-    Query q2 = DrillDown.query(q, new CategoryPath("b"));
+    Query q2 = DrillDown.query(defaultParams, q, new CategoryPath("b"));
     TopDocs docs = searcher.search(q2, 100);
     assertEquals(5, docs.totalHits);
 
     // Check that content:foo (which yields 50% results) and facet/b (which yields 20%)
     // would gather together 10 results (10%..) 
     Query fooQuery = new TermQuery(new Term("content", "foo"));
-    Query q4 = DrillDown.query(fooQuery, new CategoryPath("b"));
+    Query q4 = DrillDown.query(defaultParams, fooQuery, new CategoryPath("b"));
     docs = searcher.search(q4, 100);
     assertEquals(10, docs.totalHits);
   }
@@ -187,5 +187,38 @@ public class DrillDownTest extends Lucen
     dir.close();
     taxoDir.close();
   }
+  
+  @Test
+  public void testScoring() throws IOException {
+    // verify that drill-down queries do not modify scores
+    IndexSearcher searcher = newSearcher(reader);
+
+    float[] scores = new float[reader.maxDoc()];
+    
+    Query q = new TermQuery(new Term("content", "foo"));
+    TopDocs docs = searcher.search(q, reader.maxDoc()); // fetch all available docs to this query
+    for (ScoreDoc sd : docs.scoreDocs) {
+      scores[sd.doc] = sd.score;
+    }
+    
+    // create a drill-down query with category "a", scores should not change
+    q = DrillDown.query(defaultParams, q, new CategoryPath("a"));
+    docs = searcher.search(q, reader.maxDoc()); // fetch all available docs to this query
+    for (ScoreDoc sd : docs.scoreDocs) {
+      assertEquals("score of doc=" + sd.doc + " modified", scores[sd.doc], sd.score, 0f);
+    }
+  }
+  
+  @Test
+  public void testScoringNoBaseQuery() throws IOException {
+    // verify that drill-down queries (with no base query) returns 0.0 score
+    IndexSearcher searcher = newSearcher(reader);
+    
+    Query q = DrillDown.query(defaultParams, null, new CategoryPath("a"));
+    TopDocs docs = searcher.search(q, reader.maxDoc()); // fetch all available docs to this query
+    for (ScoreDoc sd : docs.scoreDocs) {
+      assertEquals(0f, sd.score, 0f);
+    }
+  }
     
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java Fri Jan 18 18:30:54 2013
@@ -1,11 +1,7 @@
 package org.apache.lucene.facet.search;
 
-import org.junit.Test;
-
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.search.FacetArrays;
-import org.apache.lucene.facet.search.FloatArrayAllocator;
-import org.apache.lucene.facet.search.IntArrayAllocator;
+import org.junit.Test;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -27,26 +23,39 @@ import org.apache.lucene.facet.search.In
 public class TestFacetArrays extends LuceneTestCase {
 
   @Test
-  public void testSimple() {
-    FacetArrays arrays = new FacetArrays(new IntArrayAllocator(1, 1), new FloatArrayAllocator(1, 1));
-
-    int[] intArray = arrays.getIntArray();
-    // Set the element, then free
-    intArray[0] = 1;
-    arrays.free();
-
-    // We should expect a cleared array back
-    intArray = arrays.getIntArray();
-    assertEquals("Expected a cleared array back, but the array is still filled", 0, intArray[0]);
-
-    float[] floatArray = arrays.getFloatArray();
-    // Set the element, then free
-    floatArray[0] = 1.0f;
-    arrays.free();
-
-    // We should expect a cleared array back
-    floatArray = arrays.getFloatArray();
-    assertEquals("Expected a cleared array back, but the array is still filled", 0.0f, floatArray[0], 0.0);
+  public void testFacetArrays() {
+    for (boolean reusing : new boolean[] { false, true }) {
+      final FacetArrays arrays;
+      if (reusing) {
+        arrays = new ReusingFacetArrays(new ArraysPool(1, 1));
+      } else {
+        arrays = new FacetArrays(1);
+      }
+      
+      int[] intArray = arrays.getIntArray();
+      // Set the element, then free
+      intArray[0] = 1;
+      arrays.free();
+      
+      // We should expect a cleared array back
+      int[] newIntArray = arrays.getIntArray();
+      assertEquals("Expected a cleared array back, but the array is still filled", 0, newIntArray[0]);
+      
+      float[] floatArray = arrays.getFloatArray();
+      // Set the element, then free
+      floatArray[0] = 1.0f;
+      arrays.free();
+      
+      // We should expect a cleared array back
+      float[] newFloatArray = arrays.getFloatArray();
+      assertEquals("Expected a cleared array back, but the array is still filled", 0.0f, newFloatArray[0], 0.0);
+      
+      if (reusing) {
+        // same instance should be returned after free()
+        assertSame("ReusingFacetArrays did not reuse the array!", intArray, newIntArray);
+        assertSame("ReusingFacetArrays did not reuse the array!", floatArray, newFloatArray);
+      }
+    }
   }
   
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java Fri Jan 18 18:30:54 2013
@@ -123,18 +123,15 @@ public class TestFacetsAccumulatorWithCo
     
   }
   
-  @Override
-  protected FacetSearchParams getFacetedSearchParams() {
-    FacetSearchParams res = super.getFacetedSearchParams();
-    res.addFacetRequest(new CountFacetRequest(new CategoryPath("root","a"), 10));
-    return res;
+  private FacetSearchParams getFacetSearchParams() {
+    return new FacetSearchParams(new CountFacetRequest(new CategoryPath("root","a"), 10));
   }
   
   /** compute facets with certain facet requests and docs */
   private List<FacetResult> findFacets(ScoredDocIDs sDocids, boolean withComplement) throws IOException {
     
     FacetsAccumulator fAccumulator = 
-      new StandardFacetsAccumulator(getFacetedSearchParams(), indexReader, taxoReader);
+      new StandardFacetsAccumulator(getFacetSearchParams(), indexReader, taxoReader);
     
     fAccumulator.setComplementThreshold(
         withComplement ?