You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2013/01/18 19:31:23 UTC
svn commit: r1435287 [17/41] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/maven/solr/ dev-tools/...
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java Fri Jan 18 18:30:54 2013
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -50,11 +50,10 @@ import java.io.OutputStream;
*/
public class NOnesIntEncoder extends FourFlagsIntEncoder {
+ private final IntsRef internalBuffer;
+
/** Number of consecutive '1's to be translated into single target value '2'. */
- private int n;
-
- /** Counts the number of consecutive ones seen. */
- private int onesCounter = 0;
+ private final int n;
/**
* Constructs an encoder with a given value of N (N: Number of consecutive
@@ -62,38 +61,44 @@ public class NOnesIntEncoder extends Fou
*/
public NOnesIntEncoder(int n) {
this.n = n;
+ internalBuffer = new IntsRef(n);
}
@Override
- public void close() throws IOException {
- // We might have ones in our buffer, encode them as neccesary.
- while (onesCounter-- > 0) {
- super.encode(1);
+ public void encode(IntsRef values, BytesRef buf) {
+ internalBuffer.length = 0;
+ // make sure the internal buffer is large enough
+ if (values.length > internalBuffer.ints.length) {
+ internalBuffer.grow(values.length);
}
-
- super.close();
- }
-
- @Override
- public void encode(int value) throws IOException {
- if (value == 1) {
- // Increment the number of consecutive ones seen so far
- if (++onesCounter == n) {
- super.encode(2);
- onesCounter = 0;
+
+ int onesCounter = 0;
+ int upto = values.offset + values.length;
+ for (int i = values.offset; i < upto; i++) {
+ int value = values.ints[i];
+ if (value == 1) {
+ // every N 1's should be encoded as '2'
+ if (++onesCounter == n) {
+ internalBuffer.ints[internalBuffer.length++] = 2;
+ onesCounter = 0;
+ }
+ } else {
+ // there might have been 1's that we need to encode
+ while (onesCounter > 0) {
+ --onesCounter;
+ internalBuffer.ints[internalBuffer.length++] = 1;
+ }
+
+ // encode value as value+1
+ internalBuffer.ints[internalBuffer.length++] = value + 1;
}
- return;
}
-
- // If it's not one - there might have been ones we had to encode prior to
- // this value
+ // there might have been 1's that we need to encode
while (onesCounter > 0) {
--onesCounter;
- super.encode(1);
+ internalBuffer.ints[internalBuffer.length++] = 1;
}
-
- // encode value + 1 --> the translation.
- super.encode(value + 1);
+ super.encode(internalBuffer, buf);
}
@Override
@@ -102,14 +107,8 @@ public class NOnesIntEncoder extends Fou
}
@Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- onesCounter = 0;
- }
-
- @Override
public String toString() {
- return "NOnes (" + n + ") (" + super.toString() + ")";
+ return "NOnes(" + n + ") (" + super.toString() + ")";
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java Fri Jan 18 18:30:54 2013
@@ -1,7 +1,9 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.StreamCorruptedException;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,41 +23,29 @@ import java.io.StreamCorruptedException;
*/
/**
- * A simple stream decoder which can decode values encoded with
- * {@link SimpleIntEncoder}.
+ * Decodes values encoded with {@link SimpleIntEncoder}.
*
* @lucene.experimental
*/
-public class SimpleIntDecoder extends IntDecoder {
-
- /**
- * reusable buffer - allocated only once as this is not a thread-safe object
- */
- private byte[] buffer = new byte[4];
+public final class SimpleIntDecoder extends IntDecoder {
@Override
- public long decode() throws IOException {
-
- // we need exactly 4 bytes to decode an int in this decoder impl, otherwise, throw an exception
- int offset = 0;
- while (offset < 4) {
- int nRead = in.read(buffer, offset, 4 - offset);
- if (nRead == -1) {
- if (offset > 0) {
- throw new StreamCorruptedException(
- "Need 4 bytes for decoding an int, got only " + offset);
- }
- return EOS;
- }
- offset += nRead;
+ public void decode(BytesRef buf, IntsRef values) {
+ values.offset = values.length = 0;
+ int numValues = buf.length / 4; // every value is 4 bytes
+ if (values.ints.length < numValues) { // offset and length are 0
+ values.ints = new int[ArrayUtil.oversize(numValues, RamUsageEstimator.NUM_BYTES_INT)];
+ }
+
+ int offset = buf.offset;
+ int upto = buf.offset + buf.length;
+ while (offset < upto) {
+ values.ints[values.length++] =
+ ((buf.bytes[offset++] & 0xFF) << 24) |
+ ((buf.bytes[offset++] & 0xFF) << 16) |
+ ((buf.bytes[offset++] & 0xFF) << 8) |
+ (buf.bytes[offset++] & 0xFF);
}
-
- int v = buffer[3] & 0xff;
- v |= (buffer[2] << 8) & 0xff00;
- v |= (buffer[1] << 16) & 0xff0000;
- v |= (buffer[0] << 24) & 0xff000000;
-
- return v;
}
@Override
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java Fri Jan 18 18:30:54 2013
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -24,24 +25,25 @@ import java.io.IOException;
*
* @lucene.experimental
*/
-public class SimpleIntEncoder extends IntEncoder {
+public final class SimpleIntEncoder extends IntEncoder {
- /**
- * This method makes sure the value wasn't previously encoded by checking
- * against the Set. If the value wasn't encoded, it's added to the Set, and
- * encoded with {#link Vint8#encode}
- *
- * @param value
- * an integer to be encoded
- * @throws IOException
- * possibly thrown by the OutputStream
- */
@Override
- public void encode(int value) throws IOException {
- out.write(value >>> 24);
- out.write((value >> 16) & 0xFF);
- out.write((value >> 8) & 0xFF);
- out.write(value & 0xFF);
+ public void encode(IntsRef values, BytesRef buf) {
+ buf.offset = buf.length = 0;
+ // ensure there's enough room in the buffer
+ int bytesNeeded = values.length * 4;
+ if (buf.bytes.length < bytesNeeded) {
+ buf.grow(bytesNeeded);
+ }
+
+ int upto = values.offset + values.length;
+ for (int i = values.offset; i < upto; i++) {
+ int value = values.ints[i];
+ buf.bytes[buf.length++] = (byte) (value >>> 24);
+ buf.bytes[buf.length++] = (byte) ((value >> 16) & 0xFF);
+ buf.bytes[buf.length++] = (byte) ((value >> 8) & 0xFF);
+ buf.bytes[buf.length++] = (byte) (value & 0xFF);
+ }
}
@Override
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java Fri Jan 18 18:30:54 2013
@@ -1,9 +1,10 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
import java.util.Arrays;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -23,17 +24,11 @@ import java.util.Arrays;
/**
* An {@link IntEncoderFilter} which sorts the values to encode in ascending
- * order before encoding them. Encoding therefore happens upon calling
- * {@link #close()}. Since this encoder is usually chained with another encoder
- * that relies on sorted values, it does not offer a default constructor.
+ * order before encoding them.
*
* @lucene.experimental
*/
-public class SortingIntEncoder extends IntEncoderFilter {
-
- private float grow = 2.0f;
- private int index = 0;
- private int[] set = new int[1024];
+public final class SortingIntEncoder extends IntEncoderFilter {
/** Initializes with the given encoder. */
public SortingIntEncoder(IntEncoder encoder) {
@@ -41,29 +36,9 @@ public class SortingIntEncoder extends I
}
@Override
- public void close() throws IOException {
- if (index == 0) {
- return;
- }
-
- Arrays.sort(set, 0, index);
- for (int i = 0; i < index; i++) {
- encoder.encode(set[i]);
- }
- encoder.close();
- index = 0;
-
- super.close();
- }
-
- @Override
- public void encode(int value) throws IOException {
- if (index == set.length) {
- int[] newSet = new int[(int) (set.length * grow)];
- System.arraycopy(set, 0, newSet, 0, set.length);
- set = newSet;
- }
- set[index++] = value;
+ public void encode(IntsRef values, BytesRef buf) {
+ Arrays.sort(values.ints, values.offset, values.offset + values.length);
+ encoder.encode(values, buf);
}
@Override
@@ -72,14 +47,8 @@ public class SortingIntEncoder extends I
}
@Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- index = 0;
- }
-
- @Override
public String toString() {
- return "Sorting (" + encoder.toString() + ")";
+ return "Sorting(" + encoder.toString() + ")";
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java Fri Jan 18 18:30:54 2013
@@ -1,7 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,7 +22,7 @@ import java.io.OutputStream;
/**
* An {@link IntEncoderFilter} which ensures only unique values are encoded. The
- * implementation assumes the values given to {@link #encode(int)} are sorted.
+ * implementation assumes the values given to {@link #encode(IntsRef, BytesRef)} are sorted.
* If this is not the case, you can chain this encoder with
* {@link SortingIntEncoder}.
*
@@ -30,26 +30,24 @@ import java.io.OutputStream;
*/
public final class UniqueValuesIntEncoder extends IntEncoderFilter {
- /**
- * Denotes an illegal value which we can use to init 'prev' to. Since all
- * encoded values are integers, this value is init to MAX_INT+1 and is of type
- * long. Therefore we are guaranteed not to get this value in encode.
- */
- private static final long ILLEGAL_VALUE = Integer.MAX_VALUE + 1;
-
- private long prev = ILLEGAL_VALUE;
-
/** Constructs a new instance with the given encoder. */
public UniqueValuesIntEncoder(IntEncoder encoder) {
super(encoder);
}
@Override
- public void encode(int value) throws IOException {
- if (prev != value) {
- encoder.encode(value);
- prev = value;
+ public void encode(IntsRef values, BytesRef buf) {
+ int prev = values.ints[values.offset];
+ int idx = values.offset + 1;
+ int upto = values.offset + values.length;
+ for (int i = idx; i < upto; i++) {
+ if (values.ints[i] != prev) {
+ values.ints[idx++] = values.ints[i];
+ prev = values.ints[i];
+ }
}
+ values.length = idx - values.offset;
+ encoder.encode(values, buf);
}
@Override
@@ -58,14 +56,8 @@ public final class UniqueValuesIntEncode
}
@Override
- public void reInit(OutputStream out) {
- super.reInit(out);
- prev = ILLEGAL_VALUE;
- }
-
- @Override
public String toString() {
- return "Unique (" + encoder.toString() + ")";
+ return "Unique(" + encoder.toString() + ")";
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java Fri Jan 18 18:30:54 2013
@@ -1,6 +1,9 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,33 +23,36 @@ import java.io.IOException;
*/
/**
- * An {@link IntDecoder} which can decode values encoded by
- * {@link VInt8IntEncoder}.
+ * Decodes values encoded by {@link VInt8IntEncoder}.
*
* @lucene.experimental
*/
-public class VInt8IntDecoder extends IntDecoder {
-
- private boolean legalEOS = true;
+public final class VInt8IntDecoder extends IntDecoder {
@Override
- public long decode() throws IOException {
+ public void decode(BytesRef buf, IntsRef values) {
+ values.offset = values.length = 0;
+
+ // grow the buffer up front, even if by a large number of values (buf.length)
+ // that saves the need to check inside the loop for every decoded value if
+ // the buffer needs to grow.
+ if (values.ints.length < buf.length) {
+ values.ints = new int[ArrayUtil.oversize(buf.length, RamUsageEstimator.NUM_BYTES_INT)];
+ }
+
+ // it is better if the decoding is inlined like so, and not e.g.
+ // in a utility method
+ int upto = buf.offset + buf.length;
int value = 0;
- while (true) {
- int first = in.read();
- if (first < 0) {
- if (!legalEOS) {
- throw new IOException("Unexpected End-Of-Stream");
- }
- return EOS;
- }
- value |= first & 0x7F;
- if ((first & 0x80) == 0) {
- legalEOS = true;
- return value;
+ int offset = buf.offset;
+ while (offset < upto) {
+ byte b = buf.bytes[offset++];
+ if (b >= 0) {
+ values.ints[values.length++] = (value << 7) | b;
+ value = 0;
+ } else {
+ value = (value << 7) | (b & 0x7F);
}
- legalEOS = false;
- value <<= 7;
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java Fri Jan 18 18:30:54 2013
@@ -1,6 +1,7 @@
package org.apache.lucene.util.encoding;
-import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -46,30 +47,47 @@ import java.io.IOException;
*
* @lucene.experimental
*/
-public class VInt8IntEncoder extends IntEncoder {
+public final class VInt8IntEncoder extends IntEncoder {
@Override
- public void encode(int value) throws IOException {
- if ((value & ~0x7F) == 0) {
- out.write(value);
- } else if ((value & ~0x3FFF) == 0) {
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
- } else if ((value & ~0x1FFFFF) == 0) {
- out.write(0x80 | (value >> 14));
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
- } else if ((value & ~0xFFFFFFF) == 0) {
- out.write(0x80 | (value >> 21));
- out.write(0x80 | (value >> 14));
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
- } else {
- out.write(0x80 | (value >> 28));
- out.write(0x80 | (value >> 21));
- out.write(0x80 | (value >> 14));
- out.write(0x80 | (value >> 7));
- out.write(0x7F & value);
+ public void encode(IntsRef values, BytesRef buf) {
+ buf.offset = buf.length = 0;
+ int maxBytesNeeded = 5 * values.length; // at most 5 bytes per VInt
+ if (buf.bytes.length < maxBytesNeeded) {
+ buf.grow(maxBytesNeeded);
+ }
+
+ int upto = values.offset + values.length;
+ for (int i = values.offset; i < upto; i++) {
+ // it is better if the encoding is inlined like so, and not e.g.
+ // in a utility method
+ int value = values.ints[i];
+ if ((value & ~0x7F) == 0) {
+ buf.bytes[buf.length] = (byte) value;
+ buf.length++;
+ } else if ((value & ~0x3FFF) == 0) {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 1] = (byte) (value & 0x7F);
+ buf.length += 2;
+ } else if ((value & ~0x1FFFFF) == 0) {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 2] = (byte) (value & 0x7F);
+ buf.length += 3;
+ } else if ((value & ~0xFFFFFFF) == 0) {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+ buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 3] = (byte) (value & 0x7F);
+ buf.length += 4;
+ } else {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xF0000000) >> 28));
+ buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+ buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ buf.bytes[buf.length + 3] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 4] = (byte) (value & 0x7F);
+ buf.length += 5;
+ }
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/java/org/apache/lucene/util/encoding/package.html Fri Jan 18 18:30:54 2013
@@ -25,49 +25,8 @@ mechanisms to create new ones. The super
encoders there is a matching {@link
org.apache.lucene.util.encoding.IntDecoder} implementation (not all
encoders need a decoder).
-<p>An encoder encodes the integers that are passed to {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} into a
-set output stream (see {@link
-org.apache.lucene.util.encoding.IntEncoder#reInit(OutputStream)
-reInit}). One should always call {@link
-org.apache.lucene.util.encoding.IntEncoder#close() close} when all
-integers have been encoded, to ensure proper finish by the encoder. Some
-encoders buffer values in-memory and encode in batches in order to
-optimize the encoding, and not closing them may result in loss of
-information or corrupt stream.
-<p>A proper and typical usage of an encoder looks like this:
-<blockquote><pre class="prettyprint"><code>
-int[] data = <the values to encode>
-IntEncoder encoder = new VInt8IntEncoder();
-OutputStream out = new ByteArrayOutputStream();
-encoder.reInit(out);
-for (int val : data) {
- encoder.encode(val);
-}
-encoder.close();
-
-// Print the bytes in binary
-byte[] bytes = out.toByteArray();
-for (byte b : bytes) {
- System.out.println(Integer.toBinaryString(b));
-}
-</code></pre></blockquote>
-Each encoder also implements {@link
-org.apache.lucene.util.encoding.IntEncoder#createMatchingDecoder()
-createMatchingDecoder} which returns the matching decoder for this encoder.
-As mentioned above, not all encoders have a matching decoder (like some
-encoder filters which are explained next), however every encoder should
-return a decoder following a call to that method. To complete the
-example above, one can easily iterate over the decoded values like this:
-<blockquote><pre class="prettyprint"><code>
-IntDecoder d = e.createMatchingDecoder();
-d.reInit(new ByteArrayInputStream(bytes));
-long val;
-while ((val = d.decode()) != IntDecoder.EOS) {
- System.out.println(val);
-}
-</code></pre></blockquote>
-<p>Some encoders don't perform any encoding at all, or do not include an
+<p>
+Some encoders don't perform any encoding at all, or do not include an
encoding logic. Those are called {@link
org.apache.lucene.util.encoding.IntEncoderFilter}s. A filter is an
encoder which delegates the encoding task to a given encoder, however
@@ -76,91 +35,6 @@ example is {@link org.apache.lucene.util
which encodes the gaps between values rather than the values themselves.
Another example is {@link
org.apache.lucene.util.encoding.SortingIntEncoder} which sorts all the
-values in ascending order before they are sent for encoding. This
-encoder aggregates the values in its {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} implementation
-and decoding only happens upon calling {@link
-org.apache.lucene.util.encoding.IntEncoder#close() close}.
-<h4>Extending IntEncoder</h4>
-Extending {@link org.apache.lucene.util.encoding.IntEncoder} is a very
-easy task. One only needs to implement {@link
-org.apache.lucene.util.encoding.IntEncoder#encode(int) encode} and
-{@link org.apache.lucene.util.encoding.IntEncoder#createMatchingDecoder()
-createMatchingDecoder} as the base implementation takes care of
-re-initializing the output stream and closing it. The following example
-illustrates how can one write an encoder (and a matching decoder) which
-'tags' the stream with type/ID of the encoder. Such tagging is important
-in scenarios where an application uses different encoders for different
-streams, and wants to manage some sort of mapping between an encoder ID
-to an IntEncoder/Decoder implementation, so a proper decoder will be
-initialized on the fly:
-<blockquote><pre class="prettyprint"><code>
-public class TaggingIntEncoder extends IntEncoderFilter {
-
- public TaggingIntEncoder(IntEncoder encoder) {
- super(encoder);
- }
-
- @Override
- public void encode(int value) throws IOException {
- encoder.encode(value);
- }
-
- @Override
- public IntDecoder createMatchingDecoder() {
- return new TaggingIntDecoder();
- }
-
- @Override
- public void reInit(OutputStream out) {
- super.reInit(os);
- // Assumes the application has a static EncodersMap class which is able to
- // return a unique ID for a given encoder.
- int encoderID = EncodersMap.getID(encoder);
- this.out.write(encoderID);
- }
-
- @Override
- public String toString() {
- return "Tagging (" + encoder.toString() + ")";
- }
-
-}
-</code></pre></blockquote>
-And the matching decoder:
-<blockquote><pre class="prettyprint"><code>
-public class TaggingIntDecoder extends IntDecoder {
-
- // Will be initialized upon calling reInit.
- private IntDecoder decoder;
-
- @Override
- public void reInit(InputStream in) {
- super.reInit(in);
-
- // Read the ID of the encoder that tagged this stream.
- int encoderID = in.read();
-
- // Assumes EncodersMap can return the proper IntEncoder given the ID.
- decoder = EncodersMap.getEncoder(encoderID).createMatchingDecoder();
- }
-
- @Override
- public long decode() throws IOException {
- return decoder.decode();
- }
-
- @Override
- public String toString() {
- return "Tagging (" + decoder == null ? "none" : decoder.toString() + ")";
- }
-
-}
-</code></pre></blockquote>
-The example implements <code>TaggingIntEncoder</code> as a filter over another
-encoder. Even though it does not do any filtering on the actual values, it feels
-right to present it as a filter. Anyway, this is just an example code and one
-can choose to implement it however it makes sense to the application. For
-simplicity, error checking was omitted from the sample code.
+values in ascending order before they are sent for encoding.
</body>
</html>
\ No newline at end of file
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java Fri Jan 18 18:30:54 2013
@@ -15,6 +15,18 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.facet.index.FacetFields;
+import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
+import org.apache.lucene.facet.search.params.FacetRequest;
+import org.apache.lucene.facet.search.params.FacetSearchParams;
+import org.apache.lucene.facet.search.results.FacetResult;
+import org.apache.lucene.facet.search.results.FacetResultNode;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
@@ -28,24 +40,11 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
-
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.facet.index.CategoryDocumentBuilder;
-import org.apache.lucene.facet.index.params.CategoryListParams;
-import org.apache.lucene.facet.index.params.DefaultFacetIndexingParams;
-import org.apache.lucene.facet.index.params.FacetIndexingParams;
-import org.apache.lucene.facet.search.params.FacetRequest;
-import org.apache.lucene.facet.search.params.FacetSearchParams;
-import org.apache.lucene.facet.search.results.FacetResult;
-import org.apache.lucene.facet.search.results.FacetResultNode;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
-import org.apache.lucene.facet.taxonomy.TaxonomyReader;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -66,7 +65,7 @@ import org.junit.BeforeClass;
* limitations under the License.
*/
-/** Base faceted search test. */
+@SuppressCodecs({"SimpleText"})
public abstract class FacetTestBase extends LuceneTestCase {
/** Holds a search and taxonomy Directories pair. */
@@ -184,29 +183,28 @@ public abstract class FacetTestBase exte
/** Returns a default facet indexing params */
protected FacetIndexingParams getFacetIndexingParams(final int partSize) {
- return new DefaultFacetIndexingParams() {
+ return new FacetIndexingParams() {
@Override
- protected int fixedPartitionSize() {
+ public int getPartitionSize() {
return partSize;
}
};
}
/**
- * Faceted Search Params for the test.
- * Sub classes should override in order to test with different faceted search params.
+ * Faceted Search Params for the test. Sub classes should override in order to
+ * test with different faceted search params.
*/
- protected FacetSearchParams getFacetedSearchParams() {
- return getFacetedSearchParams(Integer.MAX_VALUE);
+ protected FacetSearchParams getFacetSearchParams(FacetIndexingParams iParams, FacetRequest... facetRequests) {
+ return new FacetSearchParams(Arrays.asList(facetRequests), iParams);
}
/**
- * Faceted Search Params with specified partition size.
- * @see #getFacetedSearchParams()
+ * Faceted Search Params for the test. Sub classes should override in order to
+ * test with different faceted search params.
*/
- protected FacetSearchParams getFacetedSearchParams(int partitionSize) {
- FacetSearchParams res = new FacetSearchParams(getFacetIndexingParams(partitionSize));
- return res;
+ protected FacetSearchParams getFacetSearchParams(List<FacetRequest> facetRequests, FacetIndexingParams iParams) {
+ return new FacetSearchParams(facetRequests, iParams);
}
/**
@@ -258,9 +256,8 @@ public abstract class FacetTestBase exte
protected final void indexDoc(FacetIndexingParams iParams, RandomIndexWriter iw,
TaxonomyWriter tw, String content, List<CategoryPath> categories) throws IOException {
Document d = new Document();
- CategoryDocumentBuilder builder = new CategoryDocumentBuilder(tw, iParams);
- builder.setCategoryPaths(categories);
- builder.build(d);
+ FacetFields facetFields = new FacetFields(tw, iParams);
+ facetFields.addFields(d, categories);
d.add(new TextField("content", content, Field.Store.YES));
iw.addDocument(d);
}
@@ -270,13 +267,12 @@ public abstract class FacetTestBase exte
FacetIndexingParams iParams = getFacetIndexingParams(Integer.MAX_VALUE);
String delim = String.valueOf(iParams.getFacetDelimChar());
Map<CategoryPath, Integer> res = new HashMap<CategoryPath, Integer>();
- HashSet<Term> handledTerms = new HashSet<Term>();
+ HashSet<String> handledTerms = new HashSet<String>();
for (CategoryListParams clp : iParams.getAllCategoryListParams()) {
- Term baseTerm = new Term(clp.getTerm().field());
- if (!handledTerms.add(baseTerm)) {
+ if (!handledTerms.add(clp.field)) {
continue; // already handled this term (for another list)
}
- Terms terms = MultiFields.getTerms(indexReader, baseTerm.field());
+ Terms terms = MultiFields.getTerms(indexReader, clp.field);
if (terms == null) {
continue;
}
@@ -284,7 +280,7 @@ public abstract class FacetTestBase exte
TermsEnum te = terms.iterator(null);
DocsEnum de = null;
while (te.next() != null) {
- de = _TestUtil.docs(random(), te, liveDocs, de, 0);
+ de = _TestUtil.docs(random(), te, liveDocs, de, DocsEnum.FLAG_NONE);
int cnt = 0;
while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
cnt++;
@@ -301,7 +297,7 @@ public abstract class FacetTestBase exte
FacetResultNode topResNode = fr.getFacetResultNode();
FacetRequest freq = fr.getFacetRequest();
if (VERBOSE) {
- System.out.println(freq.getCategoryPath().toString()+ "\t\t" + topResNode);
+ System.out.println(freq.categoryPath.toString()+ "\t\t" + topResNode);
}
assertCountsAndCardinality(facetCountsTruth, topResNode, freq.getNumResults());
}
@@ -321,14 +317,13 @@ public abstract class FacetTestBase exte
}
/** Validate results equality */
- protected static void assertSameResults(List<FacetResult> expected,
- List<FacetResult> actual) {
+ protected static void assertSameResults(List<FacetResult> expected, List<FacetResult> actual) {
String expectedResults = resStringValueOnly(expected);
String actualResults = resStringValueOnly(actual);
if (!expectedResults.equals(actualResults)) {
System.err.println("Results are not the same!");
System.err.println("Expected:\n" + expectedResults);
- System.err.println("Actual" + actualResults);
+ System.err.println("Actual:\n" + actualResults);
throw new NotSameResultError();
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java Fri Jan 18 18:30:54 2013
@@ -2,26 +2,9 @@ package org.apache.lucene.facet;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
+import java.util.List;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.store.Directory;
-
-import org.apache.lucene.search.MultiCollector;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.CategoryDocumentBuilder;
-import org.apache.lucene.facet.index.params.DefaultFacetIndexingParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.search.params.CountFacetRequest;
@@ -32,6 +15,16 @@ import org.apache.lucene.facet.taxonomy.
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MultiCollector;
+import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -91,19 +84,18 @@ public class FacetTestUtils {
}
public static Collector[] search(IndexSearcher searcher,
- TaxonomyReader taxonomyReader, DefaultFacetIndexingParams iParams,
- int k, String... facetNames) throws IOException {
+ TaxonomyReader taxonomyReader, FacetIndexingParams iParams, int k,
+ String... facetNames) throws IOException {
Collector[] collectors = new Collector[2];
- FacetSearchParams facetSearchParams = new FacetSearchParams(iParams);
- Collection<FacetRequest> fRequests = new ArrayList<FacetRequest>();
+ List<FacetRequest> fRequests = new ArrayList<FacetRequest>();
for (String facetName : facetNames) {
CategoryPath cp = new CategoryPath(facetName);
FacetRequest fq = new CountFacetRequest(cp, k);
- facetSearchParams.addFacetRequest(fq);
fRequests.add(fq);
}
+ FacetSearchParams facetSearchParams = new FacetSearchParams(fRequests, iParams);
TopScoreDocCollector topDocsCollector = TopScoreDocCollector.create(
searcher.getIndexReader().maxDoc(), true);
@@ -118,17 +110,6 @@ public class FacetTestUtils {
return collectors;
}
- public static void add(FacetIndexingParams iParams, RandomIndexWriter iw,
- TaxonomyWriter tw, String... strings) throws IOException {
- ArrayList<CategoryPath> cps = new ArrayList<CategoryPath>();
- CategoryPath cp = new CategoryPath(strings);
- cps.add(cp);
- Document d = new Document();
- new CategoryDocumentBuilder(tw, iParams).setCategoryPaths(cps).build(d);
- d.add(new TextField("content", "alpha", Field.Store.YES));
- iw.addDocument(d);
- }
-
public static class IndexTaxonomyReaderPair {
public DirectoryReader indexReader;
public DirectoryTaxonomyReader taxReader;
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java Fri Jan 18 18:30:54 2013
@@ -4,7 +4,7 @@ import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.facet.example.ExampleResult;
-import org.apache.lucene.facet.example.association.AssociationMain;
+import org.apache.lucene.facet.example.association.CategoryAssociationsMain;
import org.apache.lucene.facet.search.results.FacetResultNode;
/*
@@ -35,8 +35,8 @@ public class TestAssociationExample exte
@Test
public void testAssociationExamples() throws Exception {
- assertExampleResult(new AssociationMain().runSumIntAssociationSample(), EXPECTED_INT_SUM_RESULTS);
- assertExampleResult(new AssociationMain().runSumFloatAssociationSample(), EXPECTED_FLOAT_SUM_RESULTS);
+ assertExampleResult(new CategoryAssociationsMain().runSumIntAssociationSample(), EXPECTED_INT_SUM_RESULTS);
+ assertExampleResult(new CategoryAssociationsMain().runSumFloatAssociationSample(), EXPECTED_FLOAT_SUM_RESULTS);
}
private void assertExampleResult(ExampleResult res, double[] expectedResults) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java Fri Jan 18 18:30:54 2013
@@ -34,7 +34,7 @@ import org.apache.lucene.facet.search.re
public class TestSimpleExample extends LuceneTestCase {
@Test
- public void testSimple () throws Exception {
+ public void testSimple() throws Exception {
ExampleResult res = new SimpleMain().runSimple();
assertNotNull("Null result!", res);
assertNotNull("Null facet result!", res.getFacetResults());
@@ -47,7 +47,7 @@ public class TestSimpleExample extends L
* As result, facets that without drill down got count of 2 will now get a count of 1.
*/
@Test
- public void testDrillDown () throws Exception {
+ public void testDrillDown() throws Exception {
ExampleResult res = new SimpleMain().runDrillDown();
assertNotNull("Null result!", res);
assertNotNull("Null facet result!", res.getFacetResults());
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java Fri Jan 18 18:30:54 2013
@@ -1,16 +1,12 @@
package org.apache.lucene.facet.index.categorypolicy;
-import org.apache.lucene.store.Directory;
-import org.junit.Test;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.categorypolicy.DefaultOrdinalPolicy;
-import org.apache.lucene.facet.index.categorypolicy.NonTopLevelOrdinalPolicy;
-import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -34,13 +30,11 @@ public class OrdinalPolicyTest extends L
@Test
public void testDefaultOrdinalPolicy() {
// check ordinal policy
- OrdinalPolicy ordinalPolicy = new DefaultOrdinalPolicy();
- assertFalse("default ordinal policy should not match root", ordinalPolicy
- .shouldAdd(TaxonomyReader.ROOT_ORDINAL));
+ OrdinalPolicy ordinalPolicy = OrdinalPolicy.ALL_PARENTS;
+ assertFalse("default ordinal policy should not match root", ordinalPolicy.shouldAdd(TaxonomyReader.ROOT_ORDINAL));
for (int i = 0; i < 300; i++) {
int ordinal = 1 + random().nextInt(Integer.MAX_VALUE - 1);
- assertTrue("default ordinal policy should match " + ordinal,
- ordinalPolicy.shouldAdd(ordinal));
+ assertTrue("default ordinal policy should match " + ordinal, ordinalPolicy.shouldAdd(ordinal));
}
}
@@ -54,8 +48,7 @@ public class OrdinalPolicyTest extends L
String[] topLevelStrings = new String[10];
for (int i = 0; i < 10; i++) {
topLevelStrings[i] = Integer.valueOf(random().nextInt(30)).toString();
- topLevelOrdinals[i] = taxonomy.addCategory(new CategoryPath(
- topLevelStrings[i]));
+ topLevelOrdinals[i] = taxonomy.addCategory(new CategoryPath(topLevelStrings[i]));
}
int[] nonTopLevelOrdinals = new int[300];
for (int i = 0; i < 300; i++) {
@@ -65,22 +58,18 @@ public class OrdinalPolicyTest extends L
for (int j = 1; j < components.length; j++) {
components[j] = (Integer.valueOf(random().nextInt(30))).toString();
}
- nonTopLevelOrdinals[i] = taxonomy.addCategory(new CategoryPath(
- components));
+ nonTopLevelOrdinals[i] = taxonomy.addCategory(new CategoryPath(components));
}
// check ordinal policy
OrdinalPolicy ordinalPolicy = new NonTopLevelOrdinalPolicy();
ordinalPolicy.init(taxonomy);
- assertFalse("top level ordinal policy should not match root", ordinalPolicy
- .shouldAdd(TaxonomyReader.ROOT_ORDINAL));
+ assertFalse("top level ordinal policy should not match root", ordinalPolicy.shouldAdd(TaxonomyReader.ROOT_ORDINAL));
for (int i = 0; i < 10; i++) {
- assertFalse("top level ordinal policy should not match "
- + topLevelOrdinals[i],
+ assertFalse("top level ordinal policy should not match " + topLevelOrdinals[i],
ordinalPolicy.shouldAdd(topLevelOrdinals[i]));
}
for (int i = 0; i < 300; i++) {
- assertTrue("top level ordinal policy should match "
- + nonTopLevelOrdinals[i],
+ assertTrue("top level ordinal policy should match " + nonTopLevelOrdinals[i],
ordinalPolicy.shouldAdd(nonTopLevelOrdinals[i]));
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java Fri Jan 18 18:30:54 2013
@@ -1,15 +1,11 @@
package org.apache.lucene.facet.index.categorypolicy;
-import org.apache.lucene.store.Directory;
-import org.junit.Test;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.categorypolicy.DefaultPathPolicy;
-import org.apache.lucene.facet.index.categorypolicy.NonTopLevelPathPolicy;
-import org.apache.lucene.facet.index.categorypolicy.PathPolicy;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -33,10 +29,9 @@ public class PathPolicyTest extends Luce
@Test
public void testDefaultPathPolicy() {
// check path policy
- CategoryPath cp = new CategoryPath();
- PathPolicy pathPolicy = new DefaultPathPolicy();
- assertFalse("default path policy should not accept root",
- pathPolicy.shouldAdd(cp));
+ CategoryPath cp = CategoryPath.EMPTY;
+ PathPolicy pathPolicy = PathPolicy.ALL_CATEGORIES;
+ assertFalse("default path policy should not accept root", pathPolicy.shouldAdd(cp));
for (int i = 0; i < 300; i++) {
int nComponents = 1 + random().nextInt(10);
String[] components = new String[nComponents];
@@ -44,9 +39,7 @@ public class PathPolicyTest extends Luce
components[j] = (Integer.valueOf(random().nextInt(30))).toString();
}
cp = new CategoryPath(components);
- assertTrue("default path policy should accept "
- + cp.toString('/'),
- pathPolicy.shouldAdd(cp));
+ assertTrue("default path policy should accept " + cp.toString('/'), pathPolicy.shouldAdd(cp));
}
}
@@ -78,7 +71,7 @@ public class PathPolicyTest extends Luce
// check ordinal policy
PathPolicy pathPolicy = new NonTopLevelPathPolicy();
assertFalse("top level path policy should not match root",
- pathPolicy.shouldAdd(new CategoryPath()));
+ pathPolicy.shouldAdd(CategoryPath.EMPTY));
for (int i = 0; i < 10; i++) {
assertFalse("top level path policy should not match "
+ topLevelPaths[i],
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java Fri Jan 18 18:30:54 2013
@@ -1,10 +1,12 @@
package org.apache.lucene.facet.index.params;
-import org.apache.lucene.index.Term;
-import org.junit.Test;
-
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.util.encoding.DGapVInt8IntEncoder;
+import org.apache.lucene.util.encoding.IntDecoder;
+import org.apache.lucene.util.encoding.IntEncoder;
+import org.apache.lucene.util.encoding.SortingIntEncoder;
+import org.apache.lucene.util.encoding.UniqueValuesIntEncoder;
+import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -28,9 +30,11 @@ public class CategoryListParamsTest exte
@Test
public void testDefaultSettings() {
CategoryListParams clp = new CategoryListParams();
- assertEquals("wrong default term", new Term("$facets", "$fulltree$"), clp.getTerm());
- assertEquals("unexpected default encoder", "Sorting (Unique (DGap (VInt8)))", clp.createEncoder().toString());
- assertEquals("unexpected default decoder", "DGap (VInt8)", clp.createEncoder().createMatchingDecoder().toString());
+ assertEquals("wrong default field", "$facets", clp.field);
+ IntEncoder encoder = new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapVInt8IntEncoder()));
+ IntDecoder decoder = encoder.createMatchingDecoder();
+ assertEquals("unexpected default encoder", encoder.toString(), clp.createEncoder().toString());
+ assertEquals("unexpected default decoder", decoder.toString(), clp.createEncoder().createMatchingDecoder().toString());
}
/**
@@ -64,8 +68,8 @@ public class CategoryListParamsTest exte
clParams1.hashCode(), clParams2.hashCode());
// Test 2 CategoryListParams with the same specified Term
- clParams1 = new CategoryListParams(new Term("test"));
- clParams2 = new CategoryListParams(new Term("test"));
+ clParams1 = new CategoryListParams("test");
+ clParams2 = new CategoryListParams("test");
assertEquals(
"2 CategoryListParams with the same term should equal each other.",
clParams1, clParams2);
@@ -73,8 +77,8 @@ public class CategoryListParamsTest exte
clParams1.hashCode(), clParams2.hashCode());
// Test 2 CategoryListParams with DIFFERENT terms
- clParams1 = new CategoryListParams(new Term("test1"));
- clParams2 = new CategoryListParams(new Term("test2"));
+ clParams1 = new CategoryListParams("test1");
+ clParams2 = new CategoryListParams("test2");
assertFalse(
"2 CategoryListParams with the different terms should NOT equal each other.",
clParams1.equals(clParams2));
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java Fri Jan 18 18:30:54 2013
@@ -1,15 +1,13 @@
package org.apache.lucene.facet.index.params;
-import org.apache.lucene.index.Term;
-import org.junit.Test;
+import java.util.Collections;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.params.CategoryListParams;
-import org.apache.lucene.facet.index.params.FacetIndexingParams;
-import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
import org.apache.lucene.facet.search.DrillDown;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.util.PartitionsUtils;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -32,50 +30,33 @@ public class PerDimensionIndexingParamsT
@Test
public void testTopLevelSettings() {
- FacetIndexingParams ifip = new PerDimensionIndexingParams();
- assertNotNull("Missing default category list", ifip
- .getAllCategoryListParams());
- assertEquals(
- "Expected default category list term is $facets:$fulltree$",
- new Term("$facets", "$fulltree$"), ifip.getCategoryListParams(
- null).getTerm());
- String expectedDDText = "a"
- + ifip.getFacetDelimChar() + "b";
+ FacetIndexingParams ifip = new PerDimensionIndexingParams(Collections.<CategoryPath, CategoryListParams>emptyMap());
+ assertNotNull("Missing default category list", ifip.getAllCategoryListParams());
+ assertEquals("Expected default category list field is $facets", "$facets", ifip.getCategoryListParams(null).field);
+ String expectedDDText = "a" + ifip.getFacetDelimChar() + "b";
CategoryPath cp = new CategoryPath("a", "b");
- assertEquals("wrong drill-down term", new Term("$facets",
- expectedDDText), DrillDown.term(ifip,cp));
+ assertEquals("wrong drill-down term", new Term("$facets", expectedDDText), DrillDown.term(ifip,cp));
char[] buf = new char[20];
int numchars = ifip.drillDownTermText(cp, buf);
assertEquals("3 characters should be written", 3, numchars);
- assertEquals("wrong drill-down term text", expectedDDText, new String(
- buf, 0, numchars));
+ assertEquals("wrong drill-down term text", expectedDDText, new String(buf, 0, numchars));
CategoryListParams clParams = ifip.getCategoryListParams(null);
- assertEquals("partition for all ordinals is the first", "$fulltree$",
- PartitionsUtils.partitionNameByOrdinal(ifip, clParams , 250));
- assertEquals("for partition 0, the same name should be returned",
- "$fulltree$", PartitionsUtils.partitionName(clParams, 0));
- assertEquals(
- "for any other, it's the concatenation of name + partition",
- "$fulltree$1", PartitionsUtils.partitionName(clParams, 1));
- assertEquals("default partition number is always 0", 0,
- PartitionsUtils.partitionNumber(ifip,100));
-
- assertEquals("default partition size is unbounded", Integer.MAX_VALUE,
- ifip.getPartitionSize());
+ assertEquals("partition for all ordinals is the first", "", PartitionsUtils.partitionNameByOrdinal(ifip, 250));
+ assertEquals("for partition 0, the same name should be returned", "", PartitionsUtils.partitionName(0));
+ assertEquals("for any other, it's the concatenation of name + partition", PartitionsUtils.PART_NAME_PREFIX + "1", PartitionsUtils.partitionName(1));
+ assertEquals("default partition number is always 0", 0, PartitionsUtils.partitionNumber(ifip,100));
+ assertEquals("default partition size is unbounded", Integer.MAX_VALUE, ifip.getPartitionSize());
}
@Test
public void testCategoryListParamsAddition() {
- PerDimensionIndexingParams tlfip = new PerDimensionIndexingParams();
- CategoryListParams clp = new CategoryListParams(
- new Term("clp", "value"));
- tlfip.addCategoryListParams(new CategoryPath("a"), clp);
- assertEquals("Expected category list term is " + clp.getTerm(), clp
- .getTerm(), tlfip.getCategoryListParams(new CategoryPath("a"))
- .getTerm());
- assertNotSame("Unexpected default category list " + clp.getTerm(), clp,
- tlfip.getCategoryListParams(null));
+ CategoryListParams clp = new CategoryListParams("clp");
+ PerDimensionIndexingParams tlfip = new PerDimensionIndexingParams(
+ Collections.<CategoryPath,CategoryListParams> singletonMap(new CategoryPath("a"), clp));
+ assertEquals("Expected category list field is " + clp.field,
+ clp.field, tlfip.getCategoryListParams(new CategoryPath("a")).field);
+ assertNotSame("Unexpected default category list " + clp.field, clp, tlfip.getCategoryListParams(null));
}
}
\ No newline at end of file
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java Fri Jan 18 18:30:54 2013
@@ -29,12 +29,11 @@ import org.apache.lucene.facet.taxonomy.
public class AdaptiveAccumulatorTest extends BaseSampleTestTopK {
@Override
- protected FacetsAccumulator getSamplingAccumulator(Sampler sampler,
- TaxonomyReader taxoReader, IndexReader indexReader,
- FacetSearchParams searchParams) {
- AdaptiveFacetsAccumulator res = new AdaptiveFacetsAccumulator(searchParams,
- indexReader, taxoReader);
+ protected FacetsAccumulator getSamplingAccumulator(Sampler sampler, TaxonomyReader taxoReader,
+ IndexReader indexReader, FacetSearchParams searchParams) {
+ AdaptiveFacetsAccumulator res = new AdaptiveFacetsAccumulator(searchParams, indexReader, taxoReader);
res.setSampler(sampler);
return res;
}
+
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java Fri Jan 18 18:30:54 2013
@@ -1,6 +1,7 @@
package org.apache.lucene.facet.search;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -12,6 +13,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.facet.FacetTestBase;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.CountFacetRequest;
+import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
@@ -61,6 +63,10 @@ public abstract class BaseTestTopK exten
return;
}
currDoc = doc;
+ // the code below tries to achieve non-uniform distribution of
+ // categories. Perhaps we can use random.nextGaussian() instead,
+ // something like nextGaussian() * stdev + maxCategory/2. Or
+ // try to generate a Zipf distribution.
nextInt = random().nextInt(categoriesPow2);
nextInt = (int)Math.sqrt(nextInt);
}
@@ -93,12 +99,12 @@ public abstract class BaseTestTopK exten
}
protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
- FacetSearchParams res = getFacetedSearchParams(partitionSize);
- res.addFacetRequest(new CountFacetRequest(new CategoryPath("a"), numResults));
- res.addFacetRequest(new CountFacetRequest(new CategoryPath("a", "1"), numResults));
- res.addFacetRequest(new CountFacetRequest(new CategoryPath("a", "1", "10"), numResults));
- res.addFacetRequest(new CountFacetRequest(new CategoryPath("a", "2", "26", "267"), numResults));
- return res;
+ List<FacetRequest> facetRequests = new ArrayList<FacetRequest>();
+ facetRequests.add(new CountFacetRequest(new CategoryPath("a"), numResults));
+ facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1"), numResults));
+ facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1", "10"), numResults));
+ facetRequests.add(new CountFacetRequest(new CategoryPath("a", "2", "26", "267"), numResults));
+ return getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
}
@Override
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java Fri Jan 18 18:30:54 2013
@@ -1,26 +1,19 @@
package org.apache.lucene.facet.search;
-import java.io.IOException;
-import java.io.Reader;
import java.util.HashSet;
import java.util.Set;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.StraightBytesDocValuesField;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.UnsafeByteArrayOutputStream;
import org.apache.lucene.util.encoding.DGapIntEncoder;
import org.apache.lucene.util.encoding.IntEncoder;
import org.apache.lucene.util.encoding.SortingIntEncoder;
@@ -47,113 +40,69 @@ import org.junit.Test;
public class CategoryListIteratorTest extends LuceneTestCase {
- private static final class DataTokenStream extends TokenStream {
-
- private int idx;
- private PayloadAttribute payload = addAttribute(PayloadAttribute.class);
- private byte[] buf = new byte[20];
- UnsafeByteArrayOutputStream ubaos = new UnsafeByteArrayOutputStream(buf);
- IntEncoder encoder;
- private boolean exhausted = false;
- private CharTermAttribute term = addAttribute(CharTermAttribute.class);
-
- public DataTokenStream(String text, IntEncoder encoder) {
- this.encoder = encoder;
- term.setEmpty().append(text);
- }
-
- public void setIdx(int idx) {
- this.idx = idx;
- exhausted = false;
- }
-
- @Override
- public boolean incrementToken() throws IOException {
- if (exhausted) {
- return false;
- }
-
- int[] values = data[idx];
- ubaos.reInit(buf);
- encoder.reInit(ubaos);
- for (int val : values) {
- encoder.encode(val);
- }
- encoder.close();
- payload.setPayload(new BytesRef(buf, 0, ubaos.length()));
-
- exhausted = true;
- return true;
- }
-
- }
-
- static final int[][] data = new int[][] {
- new int[] { 1, 2 }, new int[] { 3, 4 }, new int[] { 1, 3 }, new int[] { 1, 2, 3, 4 },
+ static final IntsRef[] data = new IntsRef[] {
+ new IntsRef(new int[] { 1, 2 }, 0, 2),
+ new IntsRef(new int[] { 3, 4 }, 0, 2),
+ new IntsRef(new int[] { 1, 3 }, 0, 2),
+ new IntsRef(new int[] { 1, 2, 3, 4 }, 0, 4)
};
@Test
- public void testPayloadIntDecodingIterator() throws Exception {
+ public void testPayloadCategoryListIteraor() throws Exception {
Directory dir = newDirectory();
- DataTokenStream dts = new DataTokenStream("1",new SortingIntEncoder(
- new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
+ final IntEncoder encoder = new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder())));
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)).setMergePolicy(newLogMergePolicy()));
+ BytesRef buf = new BytesRef();
for (int i = 0; i < data.length; i++) {
- dts.setIdx(i);
Document doc = new Document();
- doc.add(new TextField("f", dts));
+ encoder.encode(IntsRef.deepCopyOf(data[i]), buf);
+ doc.add(new StraightBytesDocValuesField("f", buf));
writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
writer.close();
- CategoryListIterator cli = new PayloadIntDecodingIterator(reader, new Term(
- "f","1"), dts.encoder.createMatchingDecoder());
- cli.init();
int totalCategories = 0;
- for (int i = 0; i < data.length; i++) {
- Set<Integer> values = new HashSet<Integer>();
- for (int j = 0; j < data[i].length; j++) {
- values.add(data[i][j]);
- }
- cli.skipTo(i);
- long cat;
- while ((cat = cli.nextCategory()) < Integer.MAX_VALUE) {
- assertTrue("expected category not found: " + cat, values.contains((int) cat));
- totalCategories ++;
+ IntsRef ordinals = new IntsRef();
+ CategoryListIterator cli = new DocValuesCategoryListIterator("f", encoder.createMatchingDecoder());
+ for (AtomicReaderContext context : reader.leaves()) {
+ assertTrue("failed to initalize iterator", cli.setNextReader(context));
+ int maxDoc = context.reader().maxDoc();
+ int dataIdx = context.docBase;
+ for (int doc = 0; doc < maxDoc; doc++, dataIdx++) {
+ Set<Integer> values = new HashSet<Integer>();
+ for (int j = 0; j < data[dataIdx].length; j++) {
+ values.add(data[dataIdx].ints[j]);
+ }
+ cli.getOrdinals(doc, ordinals);
+ assertTrue("no ordinals for document " + doc, ordinals.length > 0);
+ for (int j = 0; j < ordinals.length; j++) {
+ assertTrue("expected category not found: " + ordinals.ints[j], values.contains(ordinals.ints[j]));
+ }
+ totalCategories += ordinals.length;
}
}
- assertEquals("Missing categories!",10,totalCategories);
+ assertEquals("Missing categories!", 10, totalCategories);
reader.close();
dir.close();
}
- /**
- * Test that a document with no payloads does not confuse the payload decoder.
- */
@Test
public void testPayloadIteratorWithInvalidDoc() throws Exception {
Directory dir = newDirectory();
- DataTokenStream dts = new DataTokenStream("1",new SortingIntEncoder(
- new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
- // this test requires that no payloads ever be randomly present!
- final Analyzer noPayloadsAnalyzer = new Analyzer() {
- @Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.KEYWORD, false));
- }
- };
+ final IntEncoder encoder = new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder())));
// NOTE: test is wired to LogMP... because test relies on certain docids having payloads
RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, noPayloadsAnalyzer).setMergePolicy(newLogMergePolicy()));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
for (int i = 0; i < data.length; i++) {
Document doc = new Document();
if (i == 0) {
- dts.setIdx(i);
- doc.add(new TextField("f", dts)); // only doc 0 has payloads!
+ BytesRef buf = new BytesRef();
+ encoder.encode(IntsRef.deepCopyOf(data[i]), buf );
+ doc.add(new StraightBytesDocValuesField("f", buf));
} else {
- doc.add(new TextField("f", "1", Field.Store.NO));
+ doc.add(new StraightBytesDocValuesField("f", new BytesRef()));
}
writer.addDocument(doc);
writer.commit();
@@ -162,30 +111,31 @@ public class CategoryListIteratorTest ex
IndexReader reader = writer.getReader();
writer.close();
- CategoryListIterator cli = new PayloadIntDecodingIterator(reader, new Term(
- "f","1"), dts.encoder.createMatchingDecoder());
- assertTrue("Failed to initialize payload iterator", cli.init());
- int totalCats = 0;
- for (int i = 0; i < data.length; i++) {
- // doc no. i
- Set<Integer> values = new HashSet<Integer>();
- for (int j = 0; j < data[i].length; j++) {
- values.add(data[i][j]);
- }
- boolean hasDoc = cli.skipTo(i);
- if (hasDoc) {
- assertTrue("Document " + i + " must not have a payload!", i == 0);
- long cat;
- while ((cat = cli.nextCategory()) < Integer.MAX_VALUE) {
- assertTrue("expected category not found: " + cat, values.contains((int) cat));
- ++totalCats;
+ int totalCategories = 0;
+ IntsRef ordinals = new IntsRef();
+ CategoryListIterator cli = new DocValuesCategoryListIterator("f", encoder.createMatchingDecoder());
+ for (AtomicReaderContext context : reader.leaves()) {
+ assertTrue("failed to initalize iterator", cli.setNextReader(context));
+ int maxDoc = context.reader().maxDoc();
+ int dataIdx = context.docBase;
+ for (int doc = 0; doc < maxDoc; doc++, dataIdx++) {
+ Set<Integer> values = new HashSet<Integer>();
+ for (int j = 0; j < data[dataIdx].length; j++) {
+ values.add(data[dataIdx].ints[j]);
+ }
+ cli.getOrdinals(doc, ordinals);
+ if (dataIdx == 0) {
+ assertTrue("document 0 must have ordinals", ordinals.length > 0);
+ for (int j = 0; j < ordinals.length; j++) {
+ assertTrue("expected category not found: " + ordinals.ints[j], values.contains(ordinals.ints[j]));
+ }
+ totalCategories += ordinals.length;
+ } else {
+ assertTrue("only document 0 should have ordinals", ordinals.length == 0);
}
- } else {
- assertFalse("Document " + i + " must have a payload!", i == 0);
}
-
}
- assertEquals("Wrong number of total categories!", 2, totalCats);
+ assertEquals("Wrong number of total categories!", 2, totalCategories);
reader.close();
dir.close();
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java Fri Jan 18 18:30:54 2013
@@ -2,34 +2,36 @@ package org.apache.lucene.facet.search;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.facet.index.FacetFields;
+import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
+import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.index.CategoryDocumentBuilder;
-import org.apache.lucene.facet.index.params.CategoryListParams;
-import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
-import org.apache.lucene.facet.search.params.FacetSearchParams;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -49,22 +51,18 @@ import org.apache.lucene.facet.taxonomy.
public class DrillDownTest extends LuceneTestCase {
- private FacetSearchParams defaultParams = new FacetSearchParams();
- private FacetSearchParams nonDefaultParams;
+ private FacetIndexingParams defaultParams = FacetIndexingParams.ALL_PARENTS;
+ private PerDimensionIndexingParams nonDefaultParams;
private static IndexReader reader;
private static DirectoryTaxonomyReader taxo;
private static Directory dir;
private static Directory taxoDir;
public DrillDownTest() {
- PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();
- CategoryListParams aClParams = new CategoryListParams(new Term("testing_facets_a", "a"));
- CategoryListParams bClParams = new CategoryListParams(new Term("testing_facets_b", "b"));
-
- iParams.addCategoryListParams(new CategoryPath("a"), aClParams);
- iParams.addCategoryListParams(new CategoryPath("b"), bClParams);
-
- nonDefaultParams = new FacetSearchParams(iParams);
+ Map<CategoryPath,CategoryListParams> paramsMap = new HashMap<CategoryPath,CategoryListParams>();
+ paramsMap.put(new CategoryPath("a"), new CategoryListParams("testing_facets_a"));
+ paramsMap.put(new CategoryPath("b"), new CategoryListParams("testing_facets_b"));
+ nonDefaultParams = new PerDimensionIndexingParams(paramsMap);
}
@BeforeClass
@@ -91,8 +89,10 @@ public class DrillDownTest extends Lucen
if (i % 5 == 0) { // 20
paths.add(new CategoryPath("b"));
}
- CategoryDocumentBuilder builder = new CategoryDocumentBuilder(taxoWriter);
- builder.setCategoryPaths(paths).build(doc);
+ FacetFields facetFields = new FacetFields(taxoWriter);
+ if (paths.size() > 0) {
+ facetFields.addFields(doc, paths);
+ }
writer.addDocument(doc);
}
@@ -113,8 +113,8 @@ public class DrillDownTest extends Lucen
}
@Test
- public void testTermDefault() {
- String defaultField = CategoryListParams.DEFAULT_TERM.field();
+ public void testDefaultField() {
+ String defaultField = CategoryListParams.DEFAULT_FIELD;
Term termA = DrillDown.term(defaultParams, new CategoryPath("a"));
assertEquals(new Term(defaultField, "a"), termA);
@@ -128,7 +128,7 @@ public class DrillDownTest extends Lucen
IndexSearcher searcher = newSearcher(reader);
// Making sure the query yields 25 documents with the facet "a"
- Query q = DrillDown.query(defaultParams, new CategoryPath("a"));
+ Query q = DrillDown.query(defaultParams, null, new CategoryPath("a"));
TopDocs docs = searcher.search(q, 100);
assertEquals(25, docs.totalHits);
@@ -139,7 +139,7 @@ public class DrillDownTest extends Lucen
assertEquals(5, docs.totalHits);
// Making sure that a query of both facet "a" and facet "b" yields 5 results
- Query q3 = DrillDown.query(defaultParams, new CategoryPath("a"), new CategoryPath("b"));
+ Query q3 = DrillDown.query(defaultParams, null, new CategoryPath("a"), new CategoryPath("b"));
docs = searcher.search(q3, 100);
assertEquals(5, docs.totalHits);
@@ -156,18 +156,18 @@ public class DrillDownTest extends Lucen
IndexSearcher searcher = newSearcher(reader);
// Create the base query to start with
- Query q = DrillDown.query(defaultParams, new CategoryPath("a"));
+ Query q = DrillDown.query(defaultParams, null, new CategoryPath("a"));
// Making sure the query yields 5 documents with the facet "b" and the
// previous (facet "a") query as a base query
- Query q2 = DrillDown.query(q, new CategoryPath("b"));
+ Query q2 = DrillDown.query(defaultParams, q, new CategoryPath("b"));
TopDocs docs = searcher.search(q2, 100);
assertEquals(5, docs.totalHits);
// Check that content:foo (which yields 50% results) and facet/b (which yields 20%)
// would gather together 10 results (10%..)
Query fooQuery = new TermQuery(new Term("content", "foo"));
- Query q4 = DrillDown.query(fooQuery, new CategoryPath("b"));
+ Query q4 = DrillDown.query(defaultParams, fooQuery, new CategoryPath("b"));
docs = searcher.search(q4, 100);
assertEquals(10, docs.totalHits);
}
@@ -187,5 +187,38 @@ public class DrillDownTest extends Lucen
dir.close();
taxoDir.close();
}
+
+ @Test
+ public void testScoring() throws IOException {
+ // verify that drill-down queries do not modify scores
+ IndexSearcher searcher = newSearcher(reader);
+
+ float[] scores = new float[reader.maxDoc()];
+
+ Query q = new TermQuery(new Term("content", "foo"));
+ TopDocs docs = searcher.search(q, reader.maxDoc()); // fetch all available docs to this query
+ for (ScoreDoc sd : docs.scoreDocs) {
+ scores[sd.doc] = sd.score;
+ }
+
+ // create a drill-down query with category "a", scores should not change
+ q = DrillDown.query(defaultParams, q, new CategoryPath("a"));
+ docs = searcher.search(q, reader.maxDoc()); // fetch all available docs to this query
+ for (ScoreDoc sd : docs.scoreDocs) {
+ assertEquals("score of doc=" + sd.doc + " modified", scores[sd.doc], sd.score, 0f);
+ }
+ }
+
+ @Test
+ public void testScoringNoBaseQuery() throws IOException {
+ // verify that drill-down queries (with no base query) returns 0.0 score
+ IndexSearcher searcher = newSearcher(reader);
+
+ Query q = DrillDown.query(defaultParams, null, new CategoryPath("a"));
+ TopDocs docs = searcher.search(q, reader.maxDoc()); // fetch all available docs to this query
+ for (ScoreDoc sd : docs.scoreDocs) {
+ assertEquals(0f, sd.score, 0f);
+ }
+ }
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java Fri Jan 18 18:30:54 2013
@@ -1,11 +1,7 @@
package org.apache.lucene.facet.search;
-import org.junit.Test;
-
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.search.FacetArrays;
-import org.apache.lucene.facet.search.FloatArrayAllocator;
-import org.apache.lucene.facet.search.IntArrayAllocator;
+import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -27,26 +23,39 @@ import org.apache.lucene.facet.search.In
public class TestFacetArrays extends LuceneTestCase {
@Test
- public void testSimple() {
- FacetArrays arrays = new FacetArrays(new IntArrayAllocator(1, 1), new FloatArrayAllocator(1, 1));
-
- int[] intArray = arrays.getIntArray();
- // Set the element, then free
- intArray[0] = 1;
- arrays.free();
-
- // We should expect a cleared array back
- intArray = arrays.getIntArray();
- assertEquals("Expected a cleared array back, but the array is still filled", 0, intArray[0]);
-
- float[] floatArray = arrays.getFloatArray();
- // Set the element, then free
- floatArray[0] = 1.0f;
- arrays.free();
-
- // We should expect a cleared array back
- floatArray = arrays.getFloatArray();
- assertEquals("Expected a cleared array back, but the array is still filled", 0.0f, floatArray[0], 0.0);
+ public void testFacetArrays() {
+ for (boolean reusing : new boolean[] { false, true }) {
+ final FacetArrays arrays;
+ if (reusing) {
+ arrays = new ReusingFacetArrays(new ArraysPool(1, 1));
+ } else {
+ arrays = new FacetArrays(1);
+ }
+
+ int[] intArray = arrays.getIntArray();
+ // Set the element, then free
+ intArray[0] = 1;
+ arrays.free();
+
+ // We should expect a cleared array back
+ int[] newIntArray = arrays.getIntArray();
+ assertEquals("Expected a cleared array back, but the array is still filled", 0, newIntArray[0]);
+
+ float[] floatArray = arrays.getFloatArray();
+ // Set the element, then free
+ floatArray[0] = 1.0f;
+ arrays.free();
+
+ // We should expect a cleared array back
+ float[] newFloatArray = arrays.getFloatArray();
+ assertEquals("Expected a cleared array back, but the array is still filled", 0.0f, newFloatArray[0], 0.0);
+
+ if (reusing) {
+ // same instance should be returned after free()
+ assertSame("ReusingFacetArrays did not reuse the array!", intArray, newIntArray);
+ assertSame("ReusingFacetArrays did not reuse the array!", floatArray, newFloatArray);
+ }
+ }
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java Fri Jan 18 18:30:54 2013
@@ -123,18 +123,15 @@ public class TestFacetsAccumulatorWithCo
}
- @Override
- protected FacetSearchParams getFacetedSearchParams() {
- FacetSearchParams res = super.getFacetedSearchParams();
- res.addFacetRequest(new CountFacetRequest(new CategoryPath("root","a"), 10));
- return res;
+ private FacetSearchParams getFacetSearchParams() {
+ return new FacetSearchParams(new CountFacetRequest(new CategoryPath("root","a"), 10));
}
/** compute facets with certain facet requests and docs */
private List<FacetResult> findFacets(ScoredDocIDs sDocids, boolean withComplement) throws IOException {
FacetsAccumulator fAccumulator =
- new StandardFacetsAccumulator(getFacetedSearchParams(), indexReader, taxoReader);
+ new StandardFacetsAccumulator(getFacetSearchParams(), indexReader, taxoReader);
fAccumulator.setComplementThreshold(
withComplement ?