You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ha...@apache.org on 2013/07/16 19:00:18 UTC
svn commit: r1503797 [5/19] - in /lucene/dev/branches/lucene3069: ./
dev-tools/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/suggest/
dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/core/src/test/ dev-tools/maven/ dev-tools/...
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/SortField.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/SortField.java Tue Jul 16 17:00:05 2013
@@ -71,18 +71,10 @@ public class SortField {
* lower values are at the front. */
DOUBLE,
- /** Sort using term values as encoded Shorts. Sort values are Short and
- * lower values are at the front. */
- SHORT,
-
/** Sort using a custom Comparator. Sort values are any Comparable and
* sorting is done according to natural order. */
CUSTOM,
- /** Sort using term values as encoded Bytes. Sort values are Byte and
- * lower values are at the front. */
- BYTE,
-
/** Sort using term values as Strings, but comparing by
* value (using String.compareTo) for all comparisons.
* This is typically slower than {@link #STRING}, which
@@ -164,8 +156,6 @@ public class SortField {
public SortField(String field, FieldCache.Parser parser, boolean reverse) {
if (parser instanceof FieldCache.IntParser) initFieldType(field, Type.INT);
else if (parser instanceof FieldCache.FloatParser) initFieldType(field, Type.FLOAT);
- else if (parser instanceof FieldCache.ShortParser) initFieldType(field, Type.SHORT);
- else if (parser instanceof FieldCache.ByteParser) initFieldType(field, Type.BYTE);
else if (parser instanceof FieldCache.LongParser) initFieldType(field, Type.LONG);
else if (parser instanceof FieldCache.DoubleParser) initFieldType(field, Type.DOUBLE);
else {
@@ -177,7 +167,7 @@ public class SortField {
}
public SortField setMissingValue(Object missingValue) {
- if (type != Type.BYTE && type != Type.SHORT && type != Type.INT && type != Type.FLOAT && type != Type.LONG && type != Type.DOUBLE) {
+ if (type != Type.INT && type != Type.FLOAT && type != Type.LONG && type != Type.DOUBLE) {
throw new IllegalArgumentException( "Missing value only works for numeric types" );
}
this.missingValue = missingValue;
@@ -274,14 +264,6 @@ public class SortField {
buffer.append("<string_val" + ": \"").append(field).append("\">");
break;
- case BYTE:
- buffer.append("<byte: \"").append(field).append("\">");
- break;
-
- case SHORT:
- buffer.append("<short: \"").append(field).append("\">");
- break;
-
case INT:
buffer.append("<int" + ": \"").append(field).append("\">");
break;
@@ -389,12 +371,6 @@ public class SortField {
case DOUBLE:
return new FieldComparator.DoubleComparator(numHits, field, parser, (Double) missingValue);
- case BYTE:
- return new FieldComparator.ByteComparator(numHits, field, parser, (Byte) missingValue);
-
- case SHORT:
- return new FieldComparator.ShortComparator(numHits, field, parser, (Short) missingValue);
-
case CUSTOM:
assert comparatorSource != null;
return comparatorSource.newComparator(field, numHits, sortPos, reverse);
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java Tue Jul 16 17:00:05 2013
@@ -19,10 +19,40 @@ package org.apache.lucene.search.similar
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.SmallFloat;
-/** Expert: Default scoring implementation. */
+/**
+ * Expert: Default scoring implementation which {@link #encodeNormValue(float)
+ * encodes} norm values as a single byte before being stored. At search time,
+ * the norm byte value is read from the index
+ * {@link org.apache.lucene.store.Directory directory} and
+ * {@link #decodeNormValue(long) decoded} back to a float <i>norm</i> value.
+ * This encoding/decoding, while reducing index size, comes with the price of
+ * precision loss - it is not guaranteed that <i>decode(encode(x)) = x</i>. For
+ * instance, <i>decode(encode(0.89)) = 0.75</i>.
+ * <p>
+ * Compression of norm values to a single byte saves memory at search time,
+ * because once a field is referenced at search time, its norms - for all
+ * documents - are maintained in memory.
+ * <p>
+ * The rationale supporting such lossy compression of norm values is that given
+ * the difficulty (and inaccuracy) of users to express their true information
+ * need by a query, only big differences matter. <br>
+ * <br>
+ * Last, note that search time is too late to modify this <i>norm</i> part of
+ * scoring, e.g. by using a different {@link Similarity} for search.
+ */
public class DefaultSimilarity extends TFIDFSimilarity {
+ /** Cache of decoded bytes. */
+ private static final float[] NORM_TABLE = new float[256];
+
+ static {
+ for (int i = 0; i < 256; i++) {
+ NORM_TABLE[i] = SmallFloat.byte315ToFloat((byte)i);
+ }
+ }
+
/** Sole constructor: parameter-free */
public DefaultSimilarity() {}
@@ -38,6 +68,35 @@ public class DefaultSimilarity extends T
return (float)(1.0 / Math.sqrt(sumOfSquaredWeights));
}
+ /**
+ * Encodes a normalization factor for storage in an index.
+ * <p>
+ * The encoding uses a three-bit mantissa, a five-bit exponent, and the
+ * zero-exponent point at 15, thus representing values from around 7x10^9 to
+ * 2x10^-9 with about one significant decimal digit of accuracy. Zero is also
+ * represented. Negative numbers are rounded up to zero. Values too large to
+ * represent are rounded down to the largest representable value. Positive
+ * values too small to represent are rounded up to the smallest positive
+ * representable value.
+ *
+ * @see org.apache.lucene.document.Field#setBoost(float)
+ * @see org.apache.lucene.util.SmallFloat
+ */
+ @Override
+ public final long encodeNormValue(float f) {
+ return SmallFloat.floatToByte315(f);
+ }
+
+ /**
+ * Decodes the norm value, assuming it is a single byte.
+ *
+ * @see #encodeNormValue(float)
+ */
+ @Override
+ public final float decodeNormValue(long norm) {
+ return NORM_TABLE[(int) (norm & 0xFF)]; // & 0xFF maps negative bytes to positive above 127
+ }
+
/** Implemented as
* <code>state.getBoost()*lengthNorm(numTerms)</code>, where
* <code>numTerms</code> is {@link FieldInvertState#getLength()} if {@link
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java Tue Jul 16 17:00:05 2013
@@ -28,7 +28,6 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.SmallFloat;
/**
@@ -496,27 +495,8 @@ import org.apache.lucene.util.SmallFloat
* <td></td>
* </tr>
* </table>
- * <br> <br>
- * However the resulted <i>norm</i> value is {@link #encodeNormValue(float) encoded} as a single byte
- * before being stored.
- * At search time, the norm byte value is read from the index
- * {@link org.apache.lucene.store.Directory directory} and
- * {@link #decodeNormValue(byte) decoded} back to a float <i>norm</i> value.
- * This encoding/decoding, while reducing index size, comes with the price of
- * precision loss - it is not guaranteed that <i>decode(encode(x)) = x</i>.
- * For instance, <i>decode(encode(0.89)) = 0.75</i>.
- * <br> <br>
- * Compression of norm values to a single byte saves memory at search time,
- * because once a field is referenced at search time, its norms - for
- * all documents - are maintained in memory.
- * <br> <br>
- * The rationale supporting such lossy compression of norm values is that
- * given the difficulty (and inaccuracy) of users to express their true information
- * need by a query, only big differences matter.
- * <br> <br>
- * Last, note that search time is too late to modify this <i>norm</i> part of scoring, e.g. by
- * using a different {@link Similarity} for search.
- * <br> <br>
+ * Note that search time is too late to modify this <i>norm</i> part of scoring,
+ * e.g. by using a different {@link Similarity} for search.
* </li>
* </ol>
*
@@ -666,38 +646,15 @@ public abstract class TFIDFSimilarity ex
return encodeNormValue(normValue);
}
- /** Cache of decoded bytes. */
- private static final float[] NORM_TABLE = new float[256];
-
- static {
- for (int i = 0; i < 256; i++) {
- NORM_TABLE[i] = SmallFloat.byte315ToFloat((byte)i);
- }
- }
-
- /** Decodes a normalization factor stored in an index.
+ /**
+ * Decodes a normalization factor stored in an index.
+ *
* @see #encodeNormValue(float)
*/
- public float decodeNormValue(byte b) {
- return NORM_TABLE[b & 0xFF]; // & 0xFF maps negative bytes to positive above 127
- }
+ public abstract float decodeNormValue(long norm);
- /** Encodes a normalization factor for storage in an index.
- *
- * <p>The encoding uses a three-bit mantissa, a five-bit exponent, and
- * the zero-exponent point at 15, thus
- * representing values from around 7x10^9 to 2x10^-9 with about one
- * significant decimal digit of accuracy. Zero is also represented.
- * Negative numbers are rounded up to zero. Values too large to represent
- * are rounded down to the largest representable value. Positive values too
- * small to represent are rounded up to the smallest positive representable
- * value.
- * @see org.apache.lucene.document.Field#setBoost(float)
- * @see org.apache.lucene.util.SmallFloat
- */
- public byte encodeNormValue(float f) {
- return SmallFloat.floatToByte315(f);
- }
+ /** Encodes a normalization factor for storage in an index. */
+ public abstract long encodeNormValue(float f);
/** Computes the amount of a sloppy phrase match, based on an edit distance.
* This value is summed for each sloppy phrase match in a document to form
@@ -756,7 +713,7 @@ public abstract class TFIDFSimilarity ex
public float score(int doc, float freq) {
final float raw = tf(freq) * weightValue; // compute tf(f)*weight
- return norms == null ? raw : raw * decodeNormValue((byte)norms.get(doc)); // normalize for field
+ return norms == null ? raw : raw * decodeNormValue(norms.get(doc)); // normalize for field
}
@Override
@@ -843,8 +800,7 @@ public abstract class TFIDFSimilarity ex
fieldExpl.addDetail(stats.idf);
Explanation fieldNormExpl = new Explanation();
- float fieldNorm =
- norms!=null ? decodeNormValue((byte) norms.get(doc)) : 1.0f;
+ float fieldNorm = norms != null ? decodeNormValue(norms.get(doc)) : 1.0f;
fieldNormExpl.setValue(fieldNorm);
fieldNormExpl.setDescription("fieldNorm(doc="+doc+")");
fieldExpl.addDetail(fieldNormExpl);
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/BitUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/BitUtil.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/BitUtil.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/BitUtil.java Tue Jul 16 17:00:05 2013
@@ -22,8 +22,93 @@ package org.apache.lucene.util; // from
*/
public final class BitUtil {
+ private static final byte[] BYTE_COUNTS = { // table of bits/byte
+ 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
+ };
+
+ // The General Idea: instead of having an array per byte that has
+ // the offsets of the next set bit, that array could be
+ // packed inside a 32 bit integer (8 4 bit numbers). That
+ // should be faster than accessing an array for each index, and
+ // the total array size is kept smaller (256*sizeof(int))=1K
+ /***** the python code that generated bitlist
+ def bits2int(val):
+ arr=0
+ for shift in range(8,0,-1):
+ if val & 0x80:
+ arr = (arr << 4) | shift
+ val = val << 1
+ return arr
+
+ def int_table():
+ tbl = [ hex(bits2int(val)).strip('L') for val in range(256) ]
+ return ','.join(tbl)
+ ******/
+ private static final int[] BIT_LISTS = {
+ 0x0, 0x1, 0x2, 0x21, 0x3, 0x31, 0x32, 0x321, 0x4, 0x41, 0x42, 0x421, 0x43,
+ 0x431, 0x432, 0x4321, 0x5, 0x51, 0x52, 0x521, 0x53, 0x531, 0x532, 0x5321,
+ 0x54, 0x541, 0x542, 0x5421, 0x543, 0x5431, 0x5432, 0x54321, 0x6, 0x61, 0x62,
+ 0x621, 0x63, 0x631, 0x632, 0x6321, 0x64, 0x641, 0x642, 0x6421, 0x643,
+ 0x6431, 0x6432, 0x64321, 0x65, 0x651, 0x652, 0x6521, 0x653, 0x6531, 0x6532,
+ 0x65321, 0x654, 0x6541, 0x6542, 0x65421, 0x6543, 0x65431, 0x65432, 0x654321,
+ 0x7, 0x71, 0x72, 0x721, 0x73, 0x731, 0x732, 0x7321, 0x74, 0x741, 0x742,
+ 0x7421, 0x743, 0x7431, 0x7432, 0x74321, 0x75, 0x751, 0x752, 0x7521, 0x753,
+ 0x7531, 0x7532, 0x75321, 0x754, 0x7541, 0x7542, 0x75421, 0x7543, 0x75431,
+ 0x75432, 0x754321, 0x76, 0x761, 0x762, 0x7621, 0x763, 0x7631, 0x7632,
+ 0x76321, 0x764, 0x7641, 0x7642, 0x76421, 0x7643, 0x76431, 0x76432, 0x764321,
+ 0x765, 0x7651, 0x7652, 0x76521, 0x7653, 0x76531, 0x76532, 0x765321, 0x7654,
+ 0x76541, 0x76542, 0x765421, 0x76543, 0x765431, 0x765432, 0x7654321, 0x8,
+ 0x81, 0x82, 0x821, 0x83, 0x831, 0x832, 0x8321, 0x84, 0x841, 0x842, 0x8421,
+ 0x843, 0x8431, 0x8432, 0x84321, 0x85, 0x851, 0x852, 0x8521, 0x853, 0x8531,
+ 0x8532, 0x85321, 0x854, 0x8541, 0x8542, 0x85421, 0x8543, 0x85431, 0x85432,
+ 0x854321, 0x86, 0x861, 0x862, 0x8621, 0x863, 0x8631, 0x8632, 0x86321, 0x864,
+ 0x8641, 0x8642, 0x86421, 0x8643, 0x86431, 0x86432, 0x864321, 0x865, 0x8651,
+ 0x8652, 0x86521, 0x8653, 0x86531, 0x86532, 0x865321, 0x8654, 0x86541,
+ 0x86542, 0x865421, 0x86543, 0x865431, 0x865432, 0x8654321, 0x87, 0x871,
+ 0x872, 0x8721, 0x873, 0x8731, 0x8732, 0x87321, 0x874, 0x8741, 0x8742,
+ 0x87421, 0x8743, 0x87431, 0x87432, 0x874321, 0x875, 0x8751, 0x8752, 0x87521,
+ 0x8753, 0x87531, 0x87532, 0x875321, 0x8754, 0x87541, 0x87542, 0x875421,
+ 0x87543, 0x875431, 0x875432, 0x8754321, 0x876, 0x8761, 0x8762, 0x87621,
+ 0x8763, 0x87631, 0x87632, 0x876321, 0x8764, 0x87641, 0x87642, 0x876421,
+ 0x87643, 0x876431, 0x876432, 0x8764321, 0x8765, 0x87651, 0x87652, 0x876521,
+ 0x87653, 0x876531, 0x876532, 0x8765321, 0x87654, 0x876541, 0x876542,
+ 0x8765421, 0x876543, 0x8765431, 0x8765432, 0x87654321
+ };
+
private BitUtil() {} // no instance
+ /** Return the number of bits sets in b. */
+ public static int bitCount(byte b) {
+ return BYTE_COUNTS[b & 0xFF];
+ }
+
+ /** Return the list of bits which are set in b encoded as followed:
+ * <code>(i >>> (4 * n)) & 0x0F</code> is the offset of the n-th set bit of
+ * the given byte plus one, or 0 if there are n or less bits set in the given
+ * byte. For example <code>bitList(12)</code> returns 0x43:<ul>
+ * <li><code>0x43 & 0x0F</code> is 3, meaning the the first bit set is at offset 3-1 = 2,</li>
+ * <li><code>(0x43 >>> 4) & 0x0F</code> is 4, meaning there is a second bit set at offset 4-1=3,</li>
+ * <li><code>(0x43 >>> 8) & 0x0F</code> is 0, meaning there is no more bit set in this byte.</li>
+ * </ul>*/
+ public static int bitList(byte b) {
+ return BIT_LISTS[b & 0xFF];
+ }
+
// The pop methods used to rely on bit-manipulation tricks for speed but it
// turns out that it is faster to use the Long.bitCount method (which is an
// intrinsic since Java 6u18) in a naive loop, see LUCENE-2221
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/Constants.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/Constants.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/Constants.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/Constants.java Tue Jul 16 17:00:05 2013
@@ -46,6 +46,8 @@ public final class Constants {
public static final boolean SUN_OS = OS_NAME.startsWith("SunOS");
/** True iff running on Mac OS X */
public static final boolean MAC_OS_X = OS_NAME.startsWith("Mac OS X");
+ /** True iff running on FreeBSD */
+ public static final boolean FREE_BSD = OS_NAME.startsWith("FreeBSD");
public static final String OS_ARCH = System.getProperty("os.arch");
public static final String OS_VERSION = System.getProperty("os.version");
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/OpenBitSetIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/OpenBitSetIterator.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/OpenBitSetIterator.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/OpenBitSetIterator.java Tue Jul 16 17:00:05 2013
@@ -25,55 +25,6 @@ import org.apache.lucene.search.DocIdSet
*/
public class OpenBitSetIterator extends DocIdSetIterator {
- // The General Idea: instead of having an array per byte that has
- // the offsets of the next set bit, that array could be
- // packed inside a 32 bit integer (8 4 bit numbers). That
- // should be faster than accessing an array for each index, and
- // the total array size is kept smaller (256*sizeof(int))=1K
- protected final static int[] bitlist={
- 0x0, 0x1, 0x2, 0x21, 0x3, 0x31, 0x32, 0x321, 0x4, 0x41, 0x42, 0x421, 0x43,
- 0x431, 0x432, 0x4321, 0x5, 0x51, 0x52, 0x521, 0x53, 0x531, 0x532, 0x5321,
- 0x54, 0x541, 0x542, 0x5421, 0x543, 0x5431, 0x5432, 0x54321, 0x6, 0x61, 0x62,
- 0x621, 0x63, 0x631, 0x632, 0x6321, 0x64, 0x641, 0x642, 0x6421, 0x643,
- 0x6431, 0x6432, 0x64321, 0x65, 0x651, 0x652, 0x6521, 0x653, 0x6531, 0x6532,
- 0x65321, 0x654, 0x6541, 0x6542, 0x65421, 0x6543, 0x65431, 0x65432, 0x654321,
- 0x7, 0x71, 0x72, 0x721, 0x73, 0x731, 0x732, 0x7321, 0x74, 0x741, 0x742,
- 0x7421, 0x743, 0x7431, 0x7432, 0x74321, 0x75, 0x751, 0x752, 0x7521, 0x753,
- 0x7531, 0x7532, 0x75321, 0x754, 0x7541, 0x7542, 0x75421, 0x7543, 0x75431,
- 0x75432, 0x754321, 0x76, 0x761, 0x762, 0x7621, 0x763, 0x7631, 0x7632,
- 0x76321, 0x764, 0x7641, 0x7642, 0x76421, 0x7643, 0x76431, 0x76432, 0x764321,
- 0x765, 0x7651, 0x7652, 0x76521, 0x7653, 0x76531, 0x76532, 0x765321, 0x7654,
- 0x76541, 0x76542, 0x765421, 0x76543, 0x765431, 0x765432, 0x7654321, 0x8,
- 0x81, 0x82, 0x821, 0x83, 0x831, 0x832, 0x8321, 0x84, 0x841, 0x842, 0x8421,
- 0x843, 0x8431, 0x8432, 0x84321, 0x85, 0x851, 0x852, 0x8521, 0x853, 0x8531,
- 0x8532, 0x85321, 0x854, 0x8541, 0x8542, 0x85421, 0x8543, 0x85431, 0x85432,
- 0x854321, 0x86, 0x861, 0x862, 0x8621, 0x863, 0x8631, 0x8632, 0x86321, 0x864,
- 0x8641, 0x8642, 0x86421, 0x8643, 0x86431, 0x86432, 0x864321, 0x865, 0x8651,
- 0x8652, 0x86521, 0x8653, 0x86531, 0x86532, 0x865321, 0x8654, 0x86541,
- 0x86542, 0x865421, 0x86543, 0x865431, 0x865432, 0x8654321, 0x87, 0x871,
- 0x872, 0x8721, 0x873, 0x8731, 0x8732, 0x87321, 0x874, 0x8741, 0x8742,
- 0x87421, 0x8743, 0x87431, 0x87432, 0x874321, 0x875, 0x8751, 0x8752, 0x87521,
- 0x8753, 0x87531, 0x87532, 0x875321, 0x8754, 0x87541, 0x87542, 0x875421,
- 0x87543, 0x875431, 0x875432, 0x8754321, 0x876, 0x8761, 0x8762, 0x87621,
- 0x8763, 0x87631, 0x87632, 0x876321, 0x8764, 0x87641, 0x87642, 0x876421,
- 0x87643, 0x876431, 0x876432, 0x8764321, 0x8765, 0x87651, 0x87652, 0x876521,
- 0x87653, 0x876531, 0x876532, 0x8765321, 0x87654, 0x876541, 0x876542,
- 0x8765421, 0x876543, 0x8765431, 0x8765432, 0x87654321
- };
- /***** the python code that generated bitlist
- def bits2int(val):
- arr=0
- for shift in range(8,0,-1):
- if val & 0x80:
- arr = (arr << 4) | shift
- val = val << 1
- return arr
-
- def int_table():
- tbl = [ hex(bits2int(val)).strip('L') for val in range(256) ]
- return ','.join(tbl)
- ******/
-
// hmmm, what about an iterator that finds zeros though,
// or a reverse iterator... should they be separate classes
// for efficiency, or have a common root interface? (or
@@ -101,7 +52,7 @@ public class OpenBitSetIterator extends
if ((int)word ==0) {wordShift +=32; word = word >>>32; }
if ((word & 0x0000FFFF) == 0) { wordShift +=16; word >>>=16; }
if ((word & 0x000000FF) == 0) { wordShift +=8; word >>>=8; }
- indexArray = bitlist[(int)word & 0xff];
+ indexArray = BitUtil.bitList((byte) word);
}
/***** alternate shift implementations
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java Tue Jul 16 17:00:05 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.util;
*/
import java.lang.management.ManagementFactory;
+import java.lang.management.PlatformManagedObject;
import java.lang.reflect.*;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
@@ -212,18 +213,17 @@ public final class RamUsageEstimator {
// regardless of the architecture).
int objectAlignment = 8;
try {
- final Class<?> beanClazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
- final Object hotSpotBean = ManagementFactory.newPlatformMXBeanProxy(
- ManagementFactory.getPlatformMBeanServer(),
- "com.sun.management:type=HotSpotDiagnostic",
- beanClazz
- );
- final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);
- final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes");
- objectAlignment = Integer.parseInt(
- vmOption.getClass().getMethod("getValue").invoke(vmOption).toString()
- );
- supportedFeatures.add(JvmFeature.OBJECT_ALIGNMENT);
+ final Class<? extends PlatformManagedObject> beanClazz =
+ Class.forName("com.sun.management.HotSpotDiagnosticMXBean").asSubclass(PlatformManagedObject.class);
+ final Object hotSpotBean = ManagementFactory.getPlatformMXBean(beanClazz);
+ if (hotSpotBean != null) {
+ final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);
+ final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes");
+ objectAlignment = Integer.parseInt(
+ vmOption.getClass().getMethod("getValue").invoke(vmOption).toString()
+ );
+ supportedFeatures.add(JvmFeature.OBJECT_ALIGNMENT);
+ }
} catch (Exception e) {
// Ignore.
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java Tue Jul 16 17:00:05 2013
@@ -43,4 +43,14 @@ public final class ToStringUtils {
}
}
+ private final static char [] HEX = "0123456789abcdef".toCharArray();
+
+ public static String longHex(long x) {
+ char [] asHex = new char [16];
+ for (int i = 16; --i >= 0; x >>>= 4) {
+ asHex[i] = HEX[(int) x & 0x0F];
+ }
+ return "0x" + new String(asHex);
+ }
+
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/Version.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/Version.java Tue Jul 16 17:00:05 2013
@@ -68,6 +68,13 @@ public enum Version {
@Deprecated
LUCENE_44,
+ /**
+ * Match settings and bugs in Lucene's 4.5 release.
+ * @deprecated (5.0) Use latest
+ */
+ @Deprecated
+ LUCENE_45,
+
/** Match settings and bugs in Lucene's 5.0 release.
* <p>
* Use this to get the latest & greatest settings, bug
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/automaton/SpecialOperations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/automaton/SpecialOperations.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/automaton/SpecialOperations.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/automaton/SpecialOperations.java Tue Jul 16 17:00:05 2013
@@ -219,7 +219,7 @@ final public class SpecialOperations {
/**
* Returns the set of accepted strings, assuming that at most
* <code>limit</code> strings are accepted. If more than <code>limit</code>
- * strings are accepted, null is returned. If <code>limit</code><0, then
+ * strings are accepted, the first limit strings found are returned. If <code>limit</code><0, then
* the limit is infinite.
*/
public static Set<IntsRef> getFiniteStrings(Automaton a, int limit) {
@@ -227,11 +227,9 @@ final public class SpecialOperations {
if (a.isSingleton()) {
if (limit > 0) {
strings.add(Util.toUTF32(a.singleton, new IntsRef()));
- } else {
- return null;
}
} else if (!getFiniteStrings(a.initial, new HashSet<State>(), strings, new IntsRef(), limit)) {
- return null;
+ return strings;
}
return strings;
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java Tue Jul 16 17:00:05 2013
@@ -37,7 +37,7 @@ abstract class AbstractAppendingLongBuff
PackedInts.Reader[] deltas;
private long deltasBytes;
int valuesOff;
- final long[] pending;
+ long[] pending;
int pendingOff;
AbstractAppendingLongBuffer(int initialBlockCount, int pageSize) {
@@ -50,13 +50,27 @@ abstract class AbstractAppendingLongBuff
pendingOff = 0;
}
+ final int pageSize() {
+ return pageMask + 1;
+ }
+
/** Get the number of values that have been added to the buffer. */
public final long size() {
- return valuesOff * (long) pending.length + pendingOff;
+ long size = pendingOff;
+ if (valuesOff > 0) {
+ size += deltas[valuesOff - 1].size();
+ }
+ if (valuesOff > 1) {
+ size += (long) (valuesOff - 1) * pageSize();
+ }
+ return size;
}
/** Append a value to this buffer. */
public final void add(long l) {
+ if (pending == null) {
+ throw new IllegalStateException("This buffer is frozen");
+ }
if (pendingOff == pending.length) {
// check size
if (deltas.length == valuesOff) {
@@ -64,9 +78,7 @@ abstract class AbstractAppendingLongBuff
grow(newLength);
}
packPendingValues();
- if (deltas[valuesOff] != null) {
- deltasBytes += deltas[valuesOff].ramBytesUsed();
- }
+ deltasBytes += deltas[valuesOff].ramBytesUsed();
++valuesOff;
// reset pending buffer
pendingOff = 0;
@@ -83,9 +95,7 @@ abstract class AbstractAppendingLongBuff
/** Get a value from this buffer. */
public final long get(long index) {
- if (index < 0 || index >= size()) {
- throw new IndexOutOfBoundsException("" + index);
- }
+ assert index >= 0 && index < size();
final int block = (int) (index >> pageShift);
final int element = (int) (index & pageMask);
return get(block, element);
@@ -99,13 +109,15 @@ abstract class AbstractAppendingLongBuff
long[] currentValues;
int vOff, pOff;
+ int currentCount; // number of entries of the current page
Iterator() {
vOff = pOff = 0;
if (valuesOff == 0) {
currentValues = pending;
+ currentCount = pendingOff;
} else {
- currentValues = new long[pending.length];
+ currentValues = new long[deltas[0].size()];
fillValues();
}
}
@@ -114,18 +126,20 @@ abstract class AbstractAppendingLongBuff
/** Whether or not there are remaining values. */
public final boolean hasNext() {
- return vOff < valuesOff || (vOff == valuesOff && pOff < pendingOff);
+ return pOff < currentCount;
}
/** Return the next long in the buffer. */
public final long next() {
assert hasNext();
long result = currentValues[pOff++];
- if (pOff == pending.length) {
+ if (pOff == currentCount) {
vOff += 1;
pOff = 0;
if (vOff <= valuesOff) {
fillValues();
+ } else {
+ currentCount = 0;
}
}
return result;
@@ -136,7 +150,9 @@ abstract class AbstractAppendingLongBuff
long baseRamBytesUsed() {
return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
+ 3 * RamUsageEstimator.NUM_BYTES_OBJECT_REF // the 3 arrays
- + 2 * RamUsageEstimator.NUM_BYTES_INT; // the 2 offsets
+ + 2 * RamUsageEstimator.NUM_BYTES_INT // the 2 offsets
+ + 2 * RamUsageEstimator.NUM_BYTES_INT // pageShift, pageMask
+ + RamUsageEstimator.NUM_BYTES_LONG; // deltasBytes
}
/**
@@ -145,13 +161,25 @@ abstract class AbstractAppendingLongBuff
public long ramBytesUsed() {
// TODO: this is called per-doc-per-norms/dv-field, can we optimize this?
long bytesUsed = RamUsageEstimator.alignObjectSize(baseRamBytesUsed())
- + 2 * RamUsageEstimator.NUM_BYTES_INT // pageShift, pageMask
- + RamUsageEstimator.NUM_BYTES_LONG // valuesBytes
- + RamUsageEstimator.sizeOf(pending)
+ + (pending != null ? RamUsageEstimator.sizeOf(pending) : 0L)
+ RamUsageEstimator.sizeOf(minValues)
+ RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * deltas.length); // values
return bytesUsed + deltasBytes;
}
+ /** Pack all pending values in this buffer. Subsequent calls to {@link #add(long)} will fail. */
+ public void freeze() {
+ if (pendingOff > 0) {
+ if (deltas.length == valuesOff) {
+ grow(valuesOff + 1); // don't oversize!
+ }
+ packPendingValues();
+ deltasBytes += deltas[valuesOff].ramBytesUsed();
+ ++valuesOff;
+ pendingOff = 0;
+ }
+ pending = null;
+ }
+
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java Tue Jul 16 17:00:05 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.util.packed;
* limitations under the License.
*/
-import java.util.Arrays;
/**
* Utility class to buffer a list of signed longs in memory. This class only
@@ -52,8 +51,6 @@ public final class AppendingLongBuffer e
@Override
void packPendingValues() {
- assert pendingOff == pending.length;
-
// compute max delta
long minValue = pending[0];
long maxValue = pending[0];
@@ -64,7 +61,9 @@ public final class AppendingLongBuffer e
final long delta = maxValue - minValue;
minValues[valuesOff] = minValue;
- if (delta != 0) {
+ if (delta == 0) {
+ deltas[valuesOff] = new PackedInts.NullReader(pendingOff);
+ } else {
// build a new packed reader
final int bitsRequired = delta < 0 ? 64 : PackedInts.bitsRequired(delta);
for (int i = 0; i < pendingOff; ++i) {
@@ -95,13 +94,13 @@ public final class AppendingLongBuffer e
void fillValues() {
if (vOff == valuesOff) {
currentValues = pending;
- } else if (deltas[vOff] == null) {
- Arrays.fill(currentValues, minValues[vOff]);
+ currentCount = pendingOff;
} else {
- for (int k = 0; k < pending.length; ) {
- k += deltas[vOff].get(k, currentValues, k, pending.length - k);
+ currentCount = deltas[vOff].size();
+ for (int k = 0; k < currentCount; ) {
+ k += deltas[vOff].get(k, currentValues, k, currentCount - k);
}
- for (int k = 0; k < pending.length; ++k) {
+ for (int k = 0; k < currentCount; ++k) {
currentValues[k] += minValues[vOff];
}
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java Tue Jul 16 17:00:05 2013
@@ -25,20 +25,30 @@ import org.apache.lucene.util.RamUsageEs
/**
* Implements {@link PackedInts.Mutable}, but grows the
* bit count of the underlying packed ints on-demand.
+ * <p>Beware that this class will accept to set negative values but in order
+ * to do this, it will grow the number of bits per value to 64.
*
* <p>@lucene.internal</p>
*/
-
public class GrowableWriter implements PackedInts.Mutable {
- private long currentMaxValue;
+ private long currentMask;
private PackedInts.Mutable current;
private final float acceptableOverheadRatio;
+ /**
+ * @param startBitsPerValue the initial number of bits per value, may grow depending on the data
+ * @param valueCount the number of values
+ * @param acceptableOverheadRatio an acceptable overhead ratio
+ */
public GrowableWriter(int startBitsPerValue, int valueCount, float acceptableOverheadRatio) {
this.acceptableOverheadRatio = acceptableOverheadRatio;
current = PackedInts.getMutable(valueCount, startBitsPerValue, this.acceptableOverheadRatio);
- currentMaxValue = PackedInts.maxValue(current.getBitsPerValue());
+ currentMask = mask(current.getBitsPerValue());
+ }
+
+ private static long mask(int bitsPerValue) {
+ return bitsPerValue == 64 ? ~0L : PackedInts.maxValue(bitsPerValue);
}
@Override
@@ -71,16 +81,16 @@ public class GrowableWriter implements P
}
private void ensureCapacity(long value) {
- assert value >= 0;
- if (value <= currentMaxValue) {
+ if ((value & currentMask) == value) {
return;
}
- final int bitsRequired = PackedInts.bitsRequired(value);
+ final int bitsRequired = value < 0 ? 64 : PackedInts.bitsRequired(value);
+ assert bitsRequired > current.getBitsPerValue();
final int valueCount = size();
PackedInts.Mutable next = PackedInts.getMutable(valueCount, bitsRequired, acceptableOverheadRatio);
PackedInts.copy(current, 0, next, 0, valueCount, PackedInts.DEFAULT_BUFFER_SIZE);
current = next;
- currentMaxValue = PackedInts.maxValue(current.getBitsPerValue());
+ currentMask = mask(current.getBitsPerValue());
}
@Override
@@ -110,6 +120,10 @@ public class GrowableWriter implements P
public int set(int index, long[] arr, int off, int len) {
long max = 0;
for (int i = off, end = off + len; i < end; ++i) {
+ // bitwise or is nice because either all values are positive and the
+ // or-ed result will require as many bits per value as the max of the
+ // values, or one of them is negative and the result will be negative,
+ // forcing GrowableWriter to use 64 bits per value
max |= arr[i];
}
ensureCapacity(max);
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java Tue Jul 16 17:00:05 2013
@@ -43,7 +43,7 @@ public final class MonotonicAppendingLon
* @param pageSize the size of a single page */
public MonotonicAppendingLongBuffer(int initialPageCount, int pageSize) {
super(initialPageCount, pageSize);
- averages = new float[pending.length];
+ averages = new float[pageSize];
}
/** Create an {@link MonotonicAppendingLongBuffer} with initialPageCount=16
@@ -74,16 +74,15 @@ public final class MonotonicAppendingLon
@Override
void packPendingValues() {
- assert pendingOff == pending.length;
-
+ assert pendingOff > 0;
minValues[valuesOff] = pending[0];
- averages[valuesOff] = (float) (pending[pending.length - 1] - pending[0]) / (pending.length - 1);
+ averages[valuesOff] = pendingOff == 1 ? 0 : (float) (pending[pendingOff - 1] - pending[0]) / (pendingOff - 1);
- for (int i = 0; i < pending.length; ++i) {
+ for (int i = 0; i < pendingOff; ++i) {
pending[i] = zigZagEncode(pending[i] - minValues[valuesOff] - (long) (averages[valuesOff] * (long) i));
}
long maxDelta = 0;
- for (int i = 0; i < pending.length; ++i) {
+ for (int i = 0; i < pendingOff; ++i) {
if (pending[i] < 0) {
maxDelta = -1;
break;
@@ -91,7 +90,9 @@ public final class MonotonicAppendingLon
maxDelta = Math.max(maxDelta, pending[i]);
}
}
- if (maxDelta != 0) {
+ if (maxDelta == 0) {
+ deltas[valuesOff] = new PackedInts.NullReader(pendingOff);
+ } else {
final int bitsRequired = maxDelta < 0 ? 64 : PackedInts.bitsRequired(maxDelta);
final PackedInts.Mutable mutable = PackedInts.getMutable(pendingOff, bitsRequired, PackedInts.COMPACT);
for (int i = 0; i < pendingOff; ) {
@@ -118,15 +119,13 @@ public final class MonotonicAppendingLon
void fillValues() {
if (vOff == valuesOff) {
currentValues = pending;
- } else if (deltas[vOff] == null) {
- for (int k = 0; k < pending.length; ++k) {
- currentValues[k] = minValues[vOff] + (long) (averages[vOff] * (long) k);
- }
+ currentCount = pendingOff;
} else {
- for (int k = 0; k < pending.length; ) {
- k += deltas[vOff].get(k, currentValues, k, pending.length - k);
+ currentCount = deltas[vOff].size();
+ for (int k = 0; k < currentCount; ) {
+ k += deltas[vOff].get(k, currentValues, k, currentCount - k);
}
- for (int k = 0; k < pending.length; ++k) {
+ for (int k = 0; k < currentCount; ++k) {
currentValues[k] = minValues[vOff] + (long) (averages[vOff] * (long) k) + zigZagDecode(currentValues[k]);
}
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Tue Jul 16 17:00:05 2013
@@ -18,12 +18,14 @@ package org.apache.lucene.util.packed;
*/
import java.io.IOException;
+import java.util.Arrays;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.LongsRef;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* Simplistic compression for array of unsigned long values.
@@ -703,7 +705,8 @@ public class PackedInts {
@Override
public int get(int index, long[] arr, int off, int len) {
- return 0;
+ Arrays.fill(arr, off, off + len, 0);
+ return len;
}
@Override
@@ -718,7 +721,7 @@ public class PackedInts {
@Override
public long ramBytesUsed() {
- return 0;
+ return RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT);
}
@Override
@@ -1041,14 +1044,21 @@ public class PackedInts {
*/
public static Mutable getMutable(int valueCount,
int bitsPerValue, float acceptableOverheadRatio) {
- assert valueCount >= 0;
-
final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio);
- switch (formatAndBits.format) {
+ return getMutable(valueCount, formatAndBits.bitsPerValue, formatAndBits.format);
+ }
+
+ /** Same as {@link #getMutable(int, int, float)} with a pre-computed number
+ * of bits per value and format.
+ * @lucene.internal */
+ public static Mutable getMutable(int valueCount,
+ int bitsPerValue, PackedInts.Format format) {
+ assert valueCount >= 0;
+ switch (format) {
case PACKED_SINGLE_BLOCK:
- return Packed64SingleBlock.create(valueCount, formatAndBits.bitsPerValue);
+ return Packed64SingleBlock.create(valueCount, bitsPerValue);
case PACKED:
- switch (formatAndBits.bitsPerValue) {
+ switch (bitsPerValue) {
case 8:
return new Direct8(valueCount);
case 16:
@@ -1068,7 +1078,7 @@ public class PackedInts {
}
break;
}
- return new Packed64(valueCount, formatAndBits.bitsPerValue);
+ return new Packed64(valueCount, bitsPerValue);
default:
throw new AssertionError();
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java Tue Jul 16 17:00:05 2013
@@ -17,10 +17,8 @@ package org.apache.lucene.util.packed;
* limitations under the License.
*/
-import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
-import static org.apache.lucene.util.packed.PackedInts.numBlocks;
-
import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.packed.PackedInts.Mutable;
/**
* A {@link PagedGrowableWriter}. This class slices data into fixed-size blocks
@@ -30,16 +28,8 @@ import org.apache.lucene.util.RamUsageEs
* less memory-efficient.
* @lucene.internal
*/
-public final class PagedGrowableWriter {
-
- static final int MIN_BLOCK_SIZE = 1 << 6;
- static final int MAX_BLOCK_SIZE = 1 << 30;
+public final class PagedGrowableWriter extends AbstractPagedMutable<PagedGrowableWriter> {
- final long size;
- final int pageShift;
- final int pageMask;
- final GrowableWriter[] subWriters;
- final int startBitsPerValue;
final float acceptableOverheadRatio;
/**
@@ -56,98 +46,26 @@ public final class PagedGrowableWriter {
}
PagedGrowableWriter(long size, int pageSize,int startBitsPerValue, float acceptableOverheadRatio, boolean fillPages) {
- this.size = size;
- this.startBitsPerValue = startBitsPerValue;
+ super(startBitsPerValue, size, pageSize);
this.acceptableOverheadRatio = acceptableOverheadRatio;
- pageShift = checkBlockSize(pageSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
- pageMask = pageSize - 1;
- final int numPages = numBlocks(size, pageSize);
- subWriters = new GrowableWriter[numPages];
if (fillPages) {
- for (int i = 0; i < numPages; ++i) {
- // do not allocate for more entries than necessary on the last page
- final int valueCount = i == numPages - 1 ? lastPageSize(size) : pageSize;
- subWriters[i] = new GrowableWriter(startBitsPerValue, valueCount, acceptableOverheadRatio);
- }
+ fillPages();
}
}
- private int lastPageSize(long size) {
- final int sz = indexInPage(size);
- return sz == 0 ? pageSize() : sz;
- }
-
- private int pageSize() {
- return pageMask + 1;
- }
-
- /** The number of values. */
- public long size() {
- return size;
- }
-
- int pageIndex(long index) {
- return (int) (index >>> pageShift);
- }
-
- int indexInPage(long index) {
- return (int) index & pageMask;
- }
-
- /** Get value at <code>index</code>. */
- public long get(long index) {
- assert index >= 0 && index < size;
- final int pageIndex = pageIndex(index);
- final int indexInPage = indexInPage(index);
- return subWriters[pageIndex].get(indexInPage);
- }
-
- /** Set value at <code>index</code>. */
- public void set(long index, long value) {
- assert index >= 0 && index < size;
- final int pageIndex = pageIndex(index);
- final int indexInPage = indexInPage(index);
- subWriters[pageIndex].set(indexInPage, value);
- }
-
- /** Create a new {@link PagedGrowableWriter} of size <code>newSize</code>
- * based on the content of this buffer. This method is much more efficient
- * than creating a new {@link PagedGrowableWriter} and copying values one by
- * one. */
- public PagedGrowableWriter resize(long newSize) {
- final PagedGrowableWriter newWriter = new PagedGrowableWriter(newSize, pageSize(), startBitsPerValue, acceptableOverheadRatio, false);
- final int numCommonPages = Math.min(newWriter.subWriters.length, subWriters.length);
- final long[] copyBuffer = new long[1024];
- for (int i = 0; i < newWriter.subWriters.length; ++i) {
- final int valueCount = i == newWriter.subWriters.length - 1 ? lastPageSize(newSize) : pageSize();
- final int bpv = i < numCommonPages ? subWriters[i].getBitsPerValue() : startBitsPerValue;
- newWriter.subWriters[i] = new GrowableWriter(bpv, valueCount, acceptableOverheadRatio);
- if (i < numCommonPages) {
- final int copyLength = Math.min(valueCount, subWriters[i].size());
- PackedInts.copy(subWriters[i], 0, newWriter.subWriters[i].getMutable(), 0, copyLength, copyBuffer);
- }
- }
- return newWriter;
+ @Override
+ protected Mutable newMutable(int valueCount, int bitsPerValue) {
+ return new GrowableWriter(bitsPerValue, valueCount, acceptableOverheadRatio);
}
- /** Return the number of bytes used by this object. */
- public long ramBytesUsed() {
- long bytesUsed = RamUsageEstimator.alignObjectSize(
- RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
- + RamUsageEstimator.NUM_BYTES_OBJECT_REF
- + RamUsageEstimator.NUM_BYTES_LONG
- + 3 * RamUsageEstimator.NUM_BYTES_INT
- + RamUsageEstimator.NUM_BYTES_FLOAT);
- bytesUsed += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * subWriters.length);
- for (GrowableWriter gw : subWriters) {
- bytesUsed += gw.ramBytesUsed();
- }
- return bytesUsed;
+ @Override
+ protected PagedGrowableWriter newUnfilledCopy(long newSize) {
+ return new PagedGrowableWriter(newSize, pageSize(), bitsPerValue, acceptableOverheadRatio, false);
}
@Override
- public String toString() {
- return getClass().getSimpleName() + "(size=" + size() + ",pageSize=" + pageSize() + ")";
+ protected long baseRamBytesUsed() {
+ return super.baseRamBytesUsed() + RamUsageEstimator.NUM_BYTES_FLOAT;
}
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestSearch.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestSearch.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestSearch.java Tue Jul 16 17:00:05 2013
@@ -127,7 +127,7 @@ public class TestSearch extends LuceneTe
for (int j = 0; j < docs.length; j++) {
Document d = new Document();
d.add(newTextField("contents", docs[j], Field.Store.YES));
- d.add(newStringField("id", ""+j, Field.Store.NO));
+ d.add(new IntField("id", j, Field.Store.NO));
writer.addDocument(d);
}
writer.close();
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java Tue Jul 16 17:00:05 2013
@@ -81,7 +81,7 @@ public class TestSearchForDuplicates ext
for (int j = 0; j < MAX_DOCS; j++) {
Document d = new Document();
d.add(newTextField(PRIORITY_FIELD, HIGH_PRIORITY, Field.Store.YES));
- d.add(newTextField(ID_FIELD, Integer.toString(j), Field.Store.YES));
+ d.add(new IntField(ID_FIELD, j, Field.Store.YES));
writer.addDocument(d);
}
writer.close();
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Tue Jul 16 17:00:05 2013
@@ -96,7 +96,7 @@ public class TestMockAnalyzer extends Ba
String testString = "t";
Analyzer analyzer = new MockAnalyzer(random());
- TokenStream stream = analyzer.tokenStream("dummy", new StringReader(testString));
+ TokenStream stream = analyzer.tokenStream("dummy", testString);
stream.reset();
while (stream.incrementToken()) {
// consume
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/document/TestField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/document/TestField.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/document/TestField.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/document/TestField.java Tue Jul 16 17:00:05 2013
@@ -18,11 +18,8 @@ package org.apache.lucene.document;
*/
import java.io.StringReader;
-import java.nio.CharBuffer;
-
import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.Token;
-import org.apache.lucene.document.Field.ReusableStringReader;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -518,39 +515,4 @@ public class TestField extends LuceneTes
}
}
- public void testReusableStringReader() throws Exception {
- ReusableStringReader reader = new ReusableStringReader();
- assertEquals(-1, reader.read());
- assertEquals(-1, reader.read(new char[1]));
- assertEquals(-1, reader.read(new char[2], 1, 1));
- assertEquals(-1, reader.read(CharBuffer.wrap(new char[2])));
-
- reader.setValue("foobar");
- char[] buf = new char[4];
- assertEquals(4, reader.read(buf));
- assertEquals("foob", new String(buf));
- assertEquals(2, reader.read(buf));
- assertEquals("ar", new String(buf, 0, 2));
- assertEquals(-1, reader.read(buf));
- reader.close();
-
- reader.setValue("foobar");
- assertEquals(0, reader.read(buf, 1, 0));
- assertEquals(3, reader.read(buf, 1, 3));
- assertEquals("foo", new String(buf, 1, 3));
- assertEquals(2, reader.read(CharBuffer.wrap(buf, 2, 2)));
- assertEquals("ba", new String(buf, 2, 2));
- assertEquals('r', (char) reader.read());
- assertEquals(-1, reader.read(buf));
- reader.close();
-
- reader.setValue("foobar");
- StringBuilder sb = new StringBuilder();
- int ch;
- while ((ch = reader.read()) != -1) {
- sb.append((char) ch);
- }
- reader.close();
- assertEquals("foobar", sb.toString());
- }
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Tue Jul 16 17:00:05 2013
@@ -305,11 +305,7 @@ public class TestConcurrentMergeSchedule
}
}
};
- if (maxMergeThreads > cms.getMaxMergeCount()) {
- cms.setMaxMergeCount(maxMergeCount);
- }
- cms.setMaxThreadCount(maxMergeThreads);
- cms.setMaxMergeCount(maxMergeCount);
+ cms.setMaxMergesAndThreads(maxMergeCount, maxMergeThreads);
iwc.setMergeScheduler(cms);
iwc.setMaxBufferedDocs(2);
@@ -335,8 +331,7 @@ public class TestConcurrentMergeSchedule
long totMergedBytes;
public TrackingCMS() {
- setMaxMergeCount(5);
- setMaxThreadCount(5);
+ setMaxMergesAndThreads(5, 5);
}
@Override
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Tue Jul 16 17:00:05 2013
@@ -33,6 +33,7 @@ import org.apache.lucene.codecs.lucene41
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.IntField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
@@ -764,7 +765,7 @@ public void testFilesOpenClose() throws
setMergePolicy(newLogMergePolicy(10))
);
Document doc = new Document();
- doc.add(newStringField("number", "17", Field.Store.NO));
+ doc.add(new IntField("number", 17, Field.Store.NO));
writer.addDocument(doc);
writer.commit();
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java Tue Jul 16 17:00:05 2013
@@ -87,23 +87,17 @@ public class TestDocValuesWithThreads ex
BytesRef scratch2 = new BytesRef();
for(int iter=0;iter<iters;iter++) {
int docID = threadRandom.nextInt(numDocs);
- switch(threadRandom.nextInt(6)) {
+ switch(threadRandom.nextInt(4)) {
case 0:
- assertEquals((byte) numbers.get(docID).longValue(), FieldCache.DEFAULT.getBytes(ar, "number", false).get(docID));
- break;
- case 1:
- assertEquals((short) numbers.get(docID).longValue(), FieldCache.DEFAULT.getShorts(ar, "number", false).get(docID));
- break;
- case 2:
assertEquals((int) numbers.get(docID).longValue(), FieldCache.DEFAULT.getInts(ar, "number", false).get(docID));
break;
- case 3:
+ case 1:
assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getLongs(ar, "number", false).get(docID));
break;
- case 4:
+ case 2:
assertEquals(Float.intBitsToFloat((int) numbers.get(docID).longValue()), FieldCache.DEFAULT.getFloats(ar, "number", false).get(docID), 0.0f);
break;
- case 5:
+ case 3:
assertEquals(Double.longBitsToDouble(numbers.get(docID).longValue()), FieldCache.DEFAULT.getDoubles(ar, "number", false).get(docID), 0.0);
break;
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java Tue Jul 16 17:00:05 2013
@@ -170,12 +170,11 @@ public class TestFieldsReader extends Lu
@Override
public void readInternal(byte[] b, int offset, int length) throws IOException {
simOutage();
+ delegate.seek(getFilePointer());
delegate.readBytes(b, offset, length);
}
@Override
public void seekInternal(long pos) throws IOException {
- //simOutage();
- delegate.seek(pos);
}
@Override
public long length() {
@@ -187,7 +186,14 @@ public class TestFieldsReader extends Lu
}
@Override
public FaultyIndexInput clone() {
- return new FaultyIndexInput(delegate.clone());
+ FaultyIndexInput i = new FaultyIndexInput(delegate.clone());
+ // seek the clone to our current position
+ try {
+ i.seek(getFilePointer());
+ } catch (IOException e) {
+ throw new RuntimeException();
+ }
+ return i;
}
}
@@ -197,8 +203,9 @@ public class TestFieldsReader extends Lu
try {
Directory dir = new FaultyFSDirectory(indexDir);
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE));
+ IndexWriterConfig iwc = newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE);
+ IndexWriter writer = new IndexWriter(dir, iwc);
for(int i=0;i<2;i++)
writer.addDocument(testDoc);
writer.forceMerge(1);
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Jul 16 17:00:05 2013
@@ -2209,4 +2209,27 @@ public class TestIndexWriter extends Luc
dir.close();
}
}
+
+ public void testHasUncommittedChanges() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ assertTrue(writer.hasUncommittedChanges()); // this will be true because a commit will create an empty index
+ Document doc = new Document();
+ doc.add(newTextField("myfield", "a b c", Field.Store.NO));
+ writer.addDocument(doc);
+ assertTrue(writer.hasUncommittedChanges());
+ writer.commit();
+ assertFalse(writer.hasUncommittedChanges());
+ writer.addDocument(doc);
+ assertTrue(writer.hasUncommittedChanges());
+ writer.close();
+
+ writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ assertFalse(writer.hasUncommittedChanges());
+ writer.addDocument(doc);
+ assertTrue(writer.hasUncommittedChanges());
+
+ writer.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java Tue Jul 16 17:00:05 2013
@@ -18,8 +18,6 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.io.StringReader;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -49,7 +47,7 @@ public class TestLongPostings extends Lu
if (other != null && s.equals(other)) {
continue;
}
- final TokenStream ts = a.tokenStream("foo", new StringReader(s));
+ final TokenStream ts = a.tokenStream("foo", s);
final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
final BytesRef termBytes = termAtt.getBytesRef();
ts.reset();
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java Tue Jul 16 17:00:05 2013
@@ -26,8 +26,9 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -97,21 +98,28 @@ public class TestMaxTermFrequency extend
/**
* Simple similarity that encodes maxTermFrequency directly as a byte
*/
- class TestSimilarity extends DefaultSimilarity {
+ class TestSimilarity extends TFIDFSimilarity {
@Override
- public byte encodeNormValue(float f) {
- return (byte) f;
+ public float lengthNorm(FieldInvertState state) {
+ return state.getMaxTermFrequency();
}
-
+
@Override
- public float decodeNormValue(byte b) {
- return (float) b;
+ public long encodeNormValue(float f) {
+ return (byte) f;
}
@Override
- public float lengthNorm(FieldInvertState state) {
- return state.getMaxTermFrequency();
+ public float decodeNormValue(long norm) {
+ return norm;
}
+
+ @Override public float coord(int overlap, int maxOverlap) { return 0; }
+ @Override public float queryNorm(float sumOfSquaredWeights) { return 0; }
+ @Override public float tf(float freq) { return 0; }
+ @Override public float idf(long docFreq, long numDocs) { return 0; }
+ @Override public float sloppyFreq(int distance) { return 0; }
+ @Override public float scorePayload(int doc, int start, int end, BytesRef payload) { return 0; }
}
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestNorms.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestNorms.java Tue Jul 16 17:00:05 2013
@@ -29,7 +29,9 @@ import org.apache.lucene.search.TermStat
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -45,21 +47,29 @@ import org.apache.lucene.util._TestUtil;
public class TestNorms extends LuceneTestCase {
final String byteTestField = "normsTestByte";
- class CustomNormEncodingSimilarity extends DefaultSimilarity {
+ class CustomNormEncodingSimilarity extends TFIDFSimilarity {
+
@Override
- public byte encodeNormValue(float f) {
- return (byte) f;
+ public long encodeNormValue(float f) {
+ return (long) f;
}
@Override
- public float decodeNormValue(byte b) {
- return (float) b;
+ public float decodeNormValue(long norm) {
+ return norm;
}
@Override
public float lengthNorm(FieldInvertState state) {
return state.getLength();
}
+
+ @Override public float coord(int overlap, int maxOverlap) { return 0; }
+ @Override public float queryNorm(float sumOfSquaredWeights) { return 0; }
+ @Override public float tf(float freq) { return 0; }
+ @Override public float idf(long docFreq, long numDocs) { return 0; }
+ @Override public float sloppyFreq(int distance) { return 0; }
+ @Override public float scorePayload(int doc, int start, int end, BytesRef payload) { return 0; }
}
// LUCENE-1260
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java Tue Jul 16 17:00:05 2013
@@ -18,26 +18,35 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.*;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
public class TestOmitTf extends LuceneTestCase {
public static class SimpleSimilarity extends TFIDFSimilarity {
+ @Override public float decodeNormValue(long norm) { return norm; }
+ @Override public long encodeNormValue(float f) { return (long) f; }
@Override
public float queryNorm(float sumOfSquaredWeights) { return 1.0f; }
@Override
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java Tue Jul 16 17:00:05 2013
@@ -59,7 +59,7 @@ public class TestRollingUpdates extends
// TODO: sometimes update ids not in order...
for(int docIter=0;docIter<numUpdates;docIter++) {
final Document doc = docs.nextDoc();
- final String myID = ""+id;
+ final String myID = Integer.toString(id);
if (id == SIZE-1) {
id = 0;
} else {
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java Tue Jul 16 17:00:05 2013
@@ -18,8 +18,6 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.io.StringReader;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -176,7 +174,7 @@ public class TestTermVectorsWriter exten
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
Document doc = new Document();
- TokenStream stream = analyzer.tokenStream("field", new StringReader("abcd "));
+ TokenStream stream = analyzer.tokenStream("field", "abcd ");
stream.reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
stream = new CachingTokenFilter(stream);
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/search/BaseTestRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/search/BaseTestRangeFilter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/search/BaseTestRangeFilter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/search/BaseTestRangeFilter.java Tue Jul 16 17:00:05 2013
@@ -22,7 +22,12 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.RandomIndexWriter;
@@ -115,9 +120,17 @@ public class BaseTestRangeFilter extends
Document doc = new Document();
Field idField = newStringField(random, "id", "", Field.Store.YES);
+ Field intIdField = new IntField("id_int", 0, Store.YES);
+ Field floatIdField = new FloatField("id_float", 0, Store.YES);
+ Field longIdField = new LongField("id_long", 0, Store.YES);
+ Field doubleIdField = new DoubleField("id_double", 0, Store.YES);
Field randField = newStringField(random, "rand", "", Field.Store.YES);
Field bodyField = newStringField(random, "body", "", Field.Store.NO);
doc.add(idField);
+ doc.add(intIdField);
+ doc.add(floatIdField);
+ doc.add(longIdField);
+ doc.add(doubleIdField);
doc.add(randField);
doc.add(bodyField);
@@ -133,6 +146,10 @@ public class BaseTestRangeFilter extends
for (int d = minId; d <= maxId; d++) {
idField.setStringValue(pad(d));
+ intIdField.setIntValue(d);
+ floatIdField.setFloatValue(d);
+ longIdField.setLongValue(d);
+ doubleIdField.setDoubleValue(d);
int r = index.allowNegativeRandomInts ? random.nextInt() : random
.nextInt(Integer.MAX_VALUE);
if (index.maxR < r) {