You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by rm...@apache.org on 2010/02/26 14:10:08 UTC
svn commit: r916666 [10/16] - in /lucene/java/branches/flex_1458: ./
contrib/ contrib/analyzers/common/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/
contrib/analyzers/...
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ArrayUtil.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ArrayUtil.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ArrayUtil.java Fri Feb 26 13:09:54 2010
@@ -7,9 +7,9 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,8 +19,20 @@
/**
* Methods for manipulating arrays.
+ *
+ * @lucene.internal
*/
public final class ArrayUtil {
+
+ /**
+ * @deprecated This constructor was not intended to be public and should not be used.
+ * This class contains solely a static utility methods.
+ * It will be made private in Lucene 4.0
+ */
+ // make private in 4.0!
+ @Deprecated
+ public ArrayUtil() {} // no instance
+
/*
Begin Apache Harmony code
@@ -122,20 +134,95 @@
END APACHE HARMONY CODE
*/
+ /** Returns an array size >= minTargetSize, generally
+ * over-allocating exponentially to achieve amortized
+ * linear-time cost as the array grows.
+ *
+ * NOTE: this was originally borrowed from Python 2.4.2
+ * listobject.c sources (attribution in LICENSE.txt), but
+ * has now been substantially changed based on
+ * discussions from java-dev thread with subject "Dynamic
+ * array reallocation algorithms", started on Jan 12
+ * 2010.
+ *
+ * @param minTargetSize Minimum required value to be returned.
+ * @param bytesPerElement Bytes used by each element of
+ * the array. See constants in {@link RamUsageEstimator}.
+ *
+ * @lucene.internal
+ */
+
+ public static int oversize(int minTargetSize, int bytesPerElement) {
+
+ if (minTargetSize < 0) {
+ // catch usage that accidentally overflows int
+ throw new IllegalArgumentException("invalid array size " + minTargetSize);
+ }
+
+ if (minTargetSize == 0) {
+ // wait until at least one element is requested
+ return 0;
+ }
+
+ // asymptotic exponential growth by 1/8th, favors
+ // spending a bit more CPU to not tye up too much wasted
+ // RAM:
+ int extra = minTargetSize >> 3;
+
+ if (extra < 3) {
+ // for very small arrays, where constant overhead of
+ // realloc is presumably relatively high, we grow
+ // faster
+ extra = 3;
+ }
- public static int getNextSize(int targetSize) {
- /* This over-allocates proportional to the list size, making room
- * for additional growth. The over-allocation is mild, but is
- * enough to give linear-time amortized behavior over a long
- * sequence of appends() in the presence of a poorly-performing
- * system realloc().
- * The growth pattern is: 0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
- */
- return (targetSize >> 3) + (targetSize < 9 ? 3 : 6) + targetSize;
+ int newSize = minTargetSize + extra;
+
+ // add 7 to allow for worst case byte alignment addition below:
+ if (newSize+7 < 0) {
+ // int overflowed -- return max allowed array size
+ return Integer.MAX_VALUE;
+ }
+
+ if (Constants.JRE_IS_64BIT) {
+ // round up to 8 byte alignment in 64bit env
+ switch(bytesPerElement) {
+ case 4:
+ // round up to multiple of 2
+ return (newSize + 1) & 0x7ffffffe;
+ case 2:
+ // round up to multiple of 4
+ return (newSize + 3) & 0x7ffffffc;
+ case 1:
+ // round up to multiple of 8
+ return (newSize + 7) & 0x7ffffff8;
+ case 8:
+ // no rounding
+ default:
+ // odd (invalid?) size
+ return newSize;
+ }
+ } else {
+ // round up to 4 byte alignment in 64bit env
+ switch(bytesPerElement) {
+ case 2:
+ // round up to multiple of 2
+ return (newSize + 1) & 0x7ffffffe;
+ case 1:
+ // round up to multiple of 4
+ return (newSize + 3) & 0x7ffffffc;
+ case 4:
+ case 8:
+ // no rounding
+ default:
+ // odd (invalid?) size
+ return newSize;
+ }
+ }
}
- public static int getShrinkSize(int currentSize, int targetSize) {
- final int newSize = getNextSize(targetSize);
+ public static int getShrinkSize(int currentSize, int targetSize, int bytesPerElement) {
+ final int newSize = oversize(targetSize, bytesPerElement);
// Only reallocate if we are "substantially" smaller.
// This saves us from "running hot" (constantly making a
// bit bigger then a bit smaller, over and over):
@@ -147,7 +234,7 @@
public static int[] grow(int[] array, int minSize) {
if (array.length < minSize) {
- int[] newArray = new int[getNextSize(minSize)];
+ int[] newArray = new int[oversize(minSize, RamUsageEstimator.NUM_BYTES_INT)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else
@@ -159,7 +246,7 @@
}
public static int[] shrink(int[] array, int targetSize) {
- final int newSize = getShrinkSize(array.length, targetSize);
+ final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_INT);
if (newSize != array.length) {
int[] newArray = new int[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
@@ -170,7 +257,7 @@
public static long[] grow(long[] array, int minSize) {
if (array.length < minSize) {
- long[] newArray = new long[getNextSize(minSize)];
+ long[] newArray = new long[oversize(minSize, RamUsageEstimator.NUM_BYTES_LONG)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else
@@ -182,7 +269,7 @@
}
public static long[] shrink(long[] array, int targetSize) {
- final int newSize = getShrinkSize(array.length, targetSize);
+ final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_LONG);
if (newSize != array.length) {
long[] newArray = new long[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
@@ -193,7 +280,7 @@
public static byte[] grow(byte[] array, int minSize) {
if (array.length < minSize) {
- byte[] newArray = new byte[getNextSize(minSize)];
+ byte[] newArray = new byte[oversize(minSize, 1)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else
@@ -204,10 +291,10 @@
return grow(array, 1 + array.length);
}
- public static char[] shrink(char[] array, int targetSize) {
- final int newSize = getShrinkSize(array.length, targetSize);
+ public static byte[] shrink(byte[] array, int targetSize) {
+ final int newSize = getShrinkSize(array.length, targetSize, 1);
if (newSize != array.length) {
- char[] newArray = new char[newSize];
+ byte[] newArray = new byte[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
@@ -216,7 +303,7 @@
public static char[] grow(char[] array, int minSize) {
if (array.length < minSize) {
- char[] newArray = new char[getNextSize(minSize)];
+ char[] newArray = new char[oversize(minSize, RamUsageEstimator.NUM_BYTES_CHAR)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else
@@ -227,10 +314,10 @@
return grow(array, 1 + array.length);
}
- public static byte[] shrink(byte[] array, int targetSize) {
- final int newSize = getShrinkSize(array.length, targetSize);
+ public static char[] shrink(char[] array, int targetSize) {
+ final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_CHAR);
if (newSize != array.length) {
- byte[] newArray = new byte[newSize];
+ char[] newArray = new char[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/AttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/AttributeSource.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/AttributeSource.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/AttributeSource.java Fri Feb 26 13:09:54 2010
@@ -17,11 +17,12 @@
* limitations under the License.
*/
+import java.lang.ref.WeakReference;
import java.util.Collections;
import java.util.NoSuchElementException;
import java.util.Iterator;
import java.util.LinkedHashMap;
-import java.util.IdentityHashMap;
+import java.util.WeakHashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.Map.Entry;
@@ -54,8 +55,8 @@
public static final AttributeFactory DEFAULT_ATTRIBUTE_FACTORY = new DefaultAttributeFactory();
private static final class DefaultAttributeFactory extends AttributeFactory {
- private static final IdentityHashMap<Class<? extends Attribute>, Class<? extends AttributeImpl>> attClassImplMap =
- new IdentityHashMap<Class<? extends Attribute>, Class<? extends AttributeImpl>>();
+ private static final WeakHashMap<Class<? extends Attribute>, WeakReference<Class<? extends AttributeImpl>>> attClassImplMap =
+ new WeakHashMap<Class<? extends Attribute>, WeakReference<Class<? extends AttributeImpl>>>();
private DefaultAttributeFactory() {}
@@ -72,12 +73,15 @@
private static Class<? extends AttributeImpl> getClassForInterface(Class<? extends Attribute> attClass) {
synchronized(attClassImplMap) {
- Class<? extends AttributeImpl> clazz = attClassImplMap.get(attClass);
+ final WeakReference<Class<? extends AttributeImpl>> ref = attClassImplMap.get(attClass);
+ Class<? extends AttributeImpl> clazz = (ref == null) ? null : ref.get();
if (clazz == null) {
try {
attClassImplMap.put(attClass,
- clazz = Class.forName(attClass.getName() + "Impl", true, attClass.getClassLoader())
- .asSubclass(AttributeImpl.class)
+ new WeakReference<Class<? extends AttributeImpl>>(
+ clazz = Class.forName(attClass.getName() + "Impl", true, attClass.getClassLoader())
+ .asSubclass(AttributeImpl.class)
+ )
);
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException("Could not find implementing class for " + attClass.getName());
@@ -173,8 +177,8 @@
}
/** a cache that stores all interfaces for known implementation classes for performance (slow reflection) */
- private static final IdentityHashMap<Class<? extends AttributeImpl>,LinkedList<Class<? extends Attribute>>> knownImplClasses =
- new IdentityHashMap<Class<? extends AttributeImpl>,LinkedList<Class<? extends Attribute>>>();
+ private static final WeakHashMap<Class<? extends AttributeImpl>,LinkedList<WeakReference<Class<? extends Attribute>>>> knownImplClasses =
+ new WeakHashMap<Class<? extends AttributeImpl>,LinkedList<WeakReference<Class<? extends Attribute>>>>();
/** <b>Expert:</b> Adds a custom AttributeImpl instance with one or more Attribute interfaces.
* <p><font color="red"><b>Please note:</b> It is not guaranteed, that <code>att</code> is added to
@@ -187,18 +191,20 @@
public void addAttributeImpl(final AttributeImpl att) {
final Class<? extends AttributeImpl> clazz = att.getClass();
if (attributeImpls.containsKey(clazz)) return;
- LinkedList<Class<? extends Attribute>> foundInterfaces;
+ LinkedList<WeakReference<Class<? extends Attribute>>> foundInterfaces;
synchronized(knownImplClasses) {
foundInterfaces = knownImplClasses.get(clazz);
if (foundInterfaces == null) {
- knownImplClasses.put(clazz, foundInterfaces = new LinkedList<Class<? extends Attribute>>());
+ // we have a strong reference to the class instance holding all interfaces in the list (parameter "att"),
+ // so all WeakReferences are never evicted by GC
+ knownImplClasses.put(clazz, foundInterfaces = new LinkedList<WeakReference<Class<? extends Attribute>>>());
// find all interfaces that this attribute instance implements
// and that extend the Attribute interface
Class<?> actClazz = clazz;
do {
for (Class<?> curInterface : actClazz.getInterfaces()) {
if (curInterface != Attribute.class && Attribute.class.isAssignableFrom(curInterface)) {
- foundInterfaces.add(curInterface.asSubclass(Attribute.class));
+ foundInterfaces.add(new WeakReference<Class<? extends Attribute>>(curInterface.asSubclass(Attribute.class)));
}
}
actClazz = actClazz.getSuperclass();
@@ -207,7 +213,10 @@
}
// add all interfaces of this AttributeImpl to the maps
- for (Class<? extends Attribute> curInterface : foundInterfaces) {
+ for (WeakReference<Class<? extends Attribute>> curInterfaceRef : foundInterfaces) {
+ final Class<? extends Attribute> curInterface = curInterfaceRef.get();
+ assert (curInterface != null) :
+ "We have a strong reference on the class holding the interfaces, so they should never get evicted";
// Attribute is a superclass of this interface
if (!attributes.containsKey(curInterface)) {
// invalidate state to force recomputation in captureState()
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BitUtil.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BitUtil.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BitUtil.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BitUtil.java Fri Feb 26 13:09:54 2010
@@ -18,8 +18,7 @@
package org.apache.lucene.util; // from org.apache.solr.util rev 555343
/** A variety of high efficiency bit twiddling routines.
- *
- * @version $Id$
+ * @lucene.internal
*/
public class BitUtil {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BitVector.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BitVector.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BitVector.java Fri Feb 26 13:09:54 2010
@@ -63,7 +63,7 @@
/** Sets the value of <code>bit</code> to one. */
public final void set(int bit) {
if (bit >= size) {
- throw new ArrayIndexOutOfBoundsException(bit);
+ throw new ArrayIndexOutOfBoundsException("bit=" + bit + " size=" + size);
}
bits[bit >> 3] |= 1 << (bit & 7);
count = -1;
@@ -73,7 +73,7 @@
* returns true if bit was already set */
public final boolean getAndSet(int bit) {
if (bit >= size) {
- throw new ArrayIndexOutOfBoundsException(bit);
+ throw new ArrayIndexOutOfBoundsException("bit=" + bit + " size=" + size);
}
final int pos = bit >> 3;
final int v = bits[pos];
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/CharacterUtils.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/CharacterUtils.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/CharacterUtils.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/CharacterUtils.java Fri Feb 26 13:09:54 2010
@@ -1,5 +1,8 @@
package org.apache.lucene.util;
+import java.io.IOException;
+import java.io.Reader;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -21,6 +24,8 @@
* {@link CharacterUtils} provides a unified interface to Character-related
* operations to implement backwards compatible character operations based on a
* {@link Version} instance.
+ *
+ * @lucene.internal
*/
public abstract class CharacterUtils {
private static final Java4CharacterUtils JAVA_4 = new Java4CharacterUtils();
@@ -104,10 +109,52 @@
* the char array.
*/
public abstract int codePointAt(final char[] chars, final int offset, final int limit);
+
+ /**
+ * Creates a new {@link CharacterBuffer} and allocates a <code>char[]</code>
+ * of the given bufferSize.
+ *
+ * @param bufferSize
+ * the internal char buffer size, must be <code>>= 2</code>
+ * @return a new {@link CharacterBuffer} instance.
+ */
+ public static CharacterBuffer newCharacterBuffer(final int bufferSize) {
+ if(bufferSize < 2)
+ throw new IllegalArgumentException("buffersize must be >= 2");
+ return new CharacterBuffer(new char[bufferSize], 0, 0);
+ }
+
+ /**
+ * Fills the {@link CharacterBuffer} with characters read from the given
+ * reader {@link Reader}. This method tries to read as many characters into
+ * the {@link CharacterBuffer} as possible, each call to fill will start
+ * filling the buffer from offset <code>0</code> up to the length of the size
+ * of the internal character array.
+ * <p>
+ * Depending on the {@link Version} passed to
+ * {@link CharacterUtils#getInstance(Version)} this method implements
+ * supplementary character awareness when filling the given buffer. For all
+ * {@link Version} > 3.0 {@link #fill(CharacterBuffer, Reader)} guarantees
+ * that the given {@link CharacterBuffer} will never contain a high surrogate
+ * character as the last element in the buffer unless it is the last available
+ * character in the reader. In other words, high and low surrogate pairs will
+ * always be preserved across buffer boarders.
+ * </p>
+ *
+ * @param buffer
+ * the buffer to fill.
+ * @param reader
+ * the reader to read characters from.
+ * @return <code>true</code> if and only if no more characters are available
+ * in the reader, otherwise <code>false</code>.
+ * @throws IOException
+ * if the reader throws an {@link IOException}.
+ */
+ public abstract boolean fill(CharacterBuffer buffer, Reader reader) throws IOException;
private static final class Java5CharacterUtils extends CharacterUtils {
Java5CharacterUtils() {
- };
+ }
@Override
public final int codePointAt(final char[] chars, final int offset) {
@@ -124,12 +171,32 @@
return Character.codePointAt(chars, offset, limit);
}
-
+ @Override
+ public boolean fill(final CharacterBuffer buffer, final Reader reader) throws IOException {
+ final char[] charBuffer = buffer.buffer;
+ buffer.offset = 0;
+ charBuffer[0] = buffer.lastTrailingHighSurrogate;
+ final int offset = buffer.lastTrailingHighSurrogate == 0 ? 0 : 1;
+ buffer.lastTrailingHighSurrogate = 0;
+ final int read = reader.read(charBuffer, offset, charBuffer.length
+ - offset);
+ if (read == -1) {
+ buffer.length = offset;
+ return offset != 0;
+ }
+ buffer.length = read + offset;
+ // special case if the read returns 0 and the lastTrailingHighSurrogate was set
+ if (buffer.length > 1
+ && Character.isHighSurrogate(charBuffer[buffer.length - 1])) {
+ buffer.lastTrailingHighSurrogate = charBuffer[--buffer.length];
+ }
+ return true;
+ }
}
private static final class Java4CharacterUtils extends CharacterUtils {
Java4CharacterUtils() {
- };
+ }
@Override
public final int codePointAt(final char[] chars, final int offset) {
@@ -148,6 +215,72 @@
return chars[offset];
}
+ @Override
+ public boolean fill(final CharacterBuffer buffer, final Reader reader) throws IOException {
+ buffer.offset = 0;
+ final int read = reader.read(buffer.buffer);
+ if(read == -1)
+ return false;
+ buffer.length = read;
+ return true;
+ }
+
+ }
+
+ /**
+ * A simple IO buffer to use with
+ * {@link CharacterUtils#fill(CharacterBuffer, Reader)}.
+ */
+ public static final class CharacterBuffer {
+
+ private final char[] buffer;
+ private int offset;
+ private int length;
+ private char lastTrailingHighSurrogate = 0;
+
+ CharacterBuffer(char[] buffer, int offset, int length) {
+ this.buffer = buffer;
+ this.offset = offset;
+ this.length = length;
+ }
+
+ /**
+ * Returns the internal buffer
+ *
+ * @return the buffer
+ */
+ public char[] getBuffer() {
+ return buffer;
+ }
+
+ /**
+ * Returns the data offset in the internal buffer.
+ *
+ * @return the offset
+ */
+ public int getOffset() {
+ return offset;
+ }
+
+ /**
+ * Return the length of the data in the internal buffer starting at
+ * {@link #getOffset()}
+ *
+ * @return the length
+ */
+ public int getLength() {
+ return length;
+ }
+
+ /**
+ * Resets the CharacterBuffer. All internals are reset to its default
+ * values.
+ */
+ public void reset() {
+ offset = 0;
+ length = 0;
+ lastTrailingHighSurrogate = 0;
+ }
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/CloseableThreadLocal.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/CloseableThreadLocal.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/CloseableThreadLocal.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/CloseableThreadLocal.java Fri Feb 26 13:09:54 2010
@@ -48,6 +48,8 @@
* {@link #close} takes care of all
* threads. You should not call {@link #close} until all
* threads are done using the instance.
+ *
+ * @lucene.internal
*/
public class CloseableThreadLocal<T> implements Closeable {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/DummyConcurrentLock.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/DummyConcurrentLock.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/DummyConcurrentLock.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/DummyConcurrentLock.java Fri Feb 26 13:09:54 2010
@@ -24,6 +24,7 @@
/**
* A dummy lock as a replacement for {@link ReentrantLock} to disable locking
+ * @lucene.internal
*/
public final class DummyConcurrentLock implements Lock {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java Fri Feb 26 13:09:54 2010
@@ -44,11 +44,7 @@
* Unit tests) to check at run time if the FieldCache contains "insane"
* usages of the FieldCache.
* </p>
- * <p>
- * <b>EXPERIMENTAL API:</b> This API is considered extremely advanced and
- * experimental. It may be removed or altered w/o warning in future releases
- * of Lucene.
- * </p>
+ * @lucene.experimental
* @see FieldCache
* @see FieldCacheSanityChecker.Insanity
* @see FieldCacheSanityChecker.InsanityType
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java Fri Feb 26 13:09:54 2010
@@ -50,6 +50,8 @@
* WARNING: This means that the deprecated Buffer-based methods
* only work correctly with buffers that have an offset of 0. For example, they
* will not correctly interpret buffers returned by {@link ByteBuffer#slice}.
+ *
+ * @lucene.experimental
*/
public class IndexableBinaryStringTools {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/MapOfSets.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/MapOfSets.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/MapOfSets.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/MapOfSets.java Fri Feb 26 13:09:54 2010
@@ -25,6 +25,7 @@
/**
* Helper class for keeping Lists of Objects associated with keys. <b>WARNING: THIS CLASS IS NOT THREAD SAFE</b>
+ * @lucene.internal
*/
public class MapOfSets<K, V> {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/NumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/NumericUtils.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/NumericUtils.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/NumericUtils.java Fri Feb 26 13:09:54 2010
@@ -55,8 +55,7 @@
* {@link String#compareTo(String)}) representations of numeric data types for other
* usages (e.g. sorting).
*
- * <p><font color="red"><b>NOTE:</b> This API is experimental and
- * might change in incompatible ways in the next release.</font>
+ * @lucene.experimental
*
* @since 2.9
*/
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/OpenBitSet.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/OpenBitSet.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/OpenBitSet.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/OpenBitSet.java Fri Feb 26 13:09:54 2010
@@ -73,8 +73,6 @@
<th>1% full</th> <td>2.51</td> <td>3.49</td> <td> </td> <td>1.00</td> <td> </td> <td>1.02</td>
</tr>
</table>
-
- * @version $Id$
*/
public class OpenBitSet extends DocIdSet implements Bits, Cloneable, Serializable {
@@ -809,12 +807,16 @@
@Override
public int hashCode() {
- long h = 0x98761234; // something non-zero for length==0
- for (int i = bits.length; --i>=0;) {
+ // Start with a zero hash and use a mix that results in zero if the input is zero.
+ // This effectively truncates trailing zeros without an explicit check.
+ long h = 0;
+ for (int i = bits.length; --i>=0;) {
h ^= bits[i];
h = (h << 1) | (h >>> 63); // rotate left
}
- return (int)((h>>32) ^ h); // fold leftmost bits into right
+ // fold leftmost bits into right and add a constant to prevent
+ // empty sets from returning 0, which is too common.
+ return (int)((h>>32) ^ h) + 0x98761234;
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/OpenBitSetIterator.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/OpenBitSetIterator.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/OpenBitSetIterator.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/OpenBitSetIterator.java Fri Feb 26 13:09:54 2010
@@ -22,8 +22,6 @@
/** An iterator to iterate over set bits in an OpenBitSet.
* This is faster than nextSetBit() for iterating over the complete set of bits,
* especially when the density of the bits set is high.
- *
- * @version $Id$
*/
public class OpenBitSetIterator extends DocIdSetIterator {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Parameter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Parameter.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Parameter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Parameter.java Fri Feb 26 13:09:54 2010
@@ -26,6 +26,7 @@
/**
* A serializable Enum class.
* @deprecated Use Java 5 enum, will be removed in a later Lucene 3.x release.
+ * @lucene.internal
*/
@Deprecated
@SuppressWarnings("serial")
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/PriorityQueue.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/PriorityQueue.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/PriorityQueue.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/PriorityQueue.java Fri Feb 26 13:09:54 2010
@@ -23,7 +23,8 @@
*
* <p><b>NOTE</b>: This class pre-allocates a full array of
* length <code>maxSize+1</code>, in {@link #initialize}.
- *
+ *
+ * @lucene.internal
*/
public abstract class PriorityQueue<T> {
private int size;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/RamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/RamUsageEstimator.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/RamUsageEstimator.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/RamUsageEstimator.java Fri Feb 26 13:09:54 2010
@@ -30,9 +30,11 @@
* Internally uses a Map to temporally hold a reference to every
* object seen.
*
- * If checkIntered, all Strings checked will be interned, but those
+ * If checkInterned, all Strings checked will be interned, but those
* that were not already interned will be released for GC when the
* estimate is complete.
+ *
+ * @lucene.internal
*/
public final class RamUsageEstimator {
private MemoryModel memoryModel;
@@ -43,6 +45,14 @@
private int arraySize;
private int classSize;
+ public final static int NUM_BYTES_OBJECT_REF = Constants.JRE_IS_64BIT ? 8 : 4;
+ public final static int NUM_BYTES_CHAR = 2;
+ public final static int NUM_BYTES_SHORT = 2;
+ public final static int NUM_BYTES_INT = 4;
+ public final static int NUM_BYTES_LONG = 8;
+ public final static int NUM_BYTES_FLOAT = 4;
+ public final static int NUM_BYTES_DOUBLE = 8;
+
private boolean checkInterned;
/**
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ReaderUtil.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ReaderUtil.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ReaderUtil.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ReaderUtil.java Fri Feb 26 13:09:54 2010
@@ -27,6 +27,7 @@
/**
* Common util methods for dealing with {@link IndexReader}s.
*
+ * @lucene.internal
*/
public class ReaderUtil {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ScorerDocQueue.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ScorerDocQueue.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ScorerDocQueue.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ScorerDocQueue.java Fri Feb 26 13:09:54 2010
@@ -27,6 +27,8 @@
/** A ScorerDocQueue maintains a partial ordering of its Scorers such that the
least Scorer can always be found in constant time. Put()'s and pop()'s
require log(size) time. The ordering is by Scorer.doc().
+ *
+ * @lucene.internal
*/
public class ScorerDocQueue { // later: SpansQueue for spans with doc and term positions
private final HeapedScorerDoc[] heap;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SimpleStringInterner.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SimpleStringInterner.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SimpleStringInterner.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SimpleStringInterner.java Fri Feb 26 13:09:54 2010
@@ -19,7 +19,10 @@
/**
* Simple lockless and memory barrier free String intern cache that is guaranteed
- * to return the same String instance as String.intern() does.
+ * to return the same String instance as String.intern()
+ * does.
+ *
+ * @lucene.internal
*/
public class SimpleStringInterner extends StringInterner {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SmallFloat.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SmallFloat.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SmallFloat.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SmallFloat.java Fri Feb 26 13:09:54 2010
@@ -18,7 +18,7 @@
/** Floating point numbers smaller than 32 bits.
*
- * @version $Id$
+ * @lucene.internal
*/
public class SmallFloat {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SortedVIntList.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SortedVIntList.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SortedVIntList.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/SortedVIntList.java Fri Feb 26 13:09:54 2010
@@ -128,8 +128,8 @@
}
if ((lastBytePos + MAX_BYTES_PER_INT) > bytes.length) {
- // biggest possible int does not fit
- resizeBytes((bytes.length * 2) + MAX_BYTES_PER_INT);
+ // Biggest possible int does not fit.
+ resizeBytes(ArrayUtil.oversize(lastBytePos + MAX_BYTES_PER_INT, 1));
}
// See org.apache.lucene.store.IndexOutput.writeVInt()
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/StringHelper.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/StringHelper.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/StringHelper.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/StringHelper.java Fri Feb 26 13:09:54 2010
@@ -20,6 +20,8 @@
/**
* Methods for manipulating strings.
+ *
+ * @lucene.internal
*/
public abstract class StringHelper {
/**
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/UnicodeUtil.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/UnicodeUtil.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/UnicodeUtil.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/UnicodeUtil.java Fri Feb 26 13:09:54 2010
@@ -54,8 +54,7 @@
* without always allocating a new byte[] as
* String.getBytes("UTF-8") does.
*
- * <p><b>WARNING</b>: This API is a new and experimental and
- * may suddenly change. </p>
+ * @lucene.internal
*/
final public class UnicodeUtil {
@@ -72,6 +71,9 @@
private static final long HALF_SHIFT = 10;
private static final long HALF_MASK = 0x3FFL;
+ /**
+ * @lucene.internal
+ */
public static final class UTF16Result {
public char[] result = new char[10];
public int[] offsets = new int[10];
@@ -113,7 +115,7 @@
// Pre-allocate for worst case 4-for-1
final int maxLen = length * 4;
if (out.length < maxLen)
- out = result.bytes = new byte[maxLen];
+ out = result.bytes = new byte[ArrayUtil.oversize(maxLen, 1)];
result.offset = 0;
while(i < end) {
@@ -169,7 +171,7 @@
// Pre-allocate for worst case 4-for-1
final int maxLen = length * 4;
if (out.length < maxLen)
- out = result.bytes = new byte[maxLen];
+ out = result.bytes = new byte[ArrayUtil.oversize(maxLen, 1)];
result.offset = 0;
while(i < end) {
@@ -223,7 +225,7 @@
// Pre-allocate for worst case 4-for-1
final int maxLen = length * 4;
if (out.length < maxLen)
- out = result.bytes = new byte[maxLen];
+ out = result.bytes = new byte[ArrayUtil.oversize(maxLen, 1)];
int upto = 0;
for(int i=offset;i<end;i++) {
@@ -274,9 +276,7 @@
final int end = offset + length;
char[] out = result.result;
if (result.offsets.length <= end) {
- int[] newOffsets = new int[2*end];
- System.arraycopy(result.offsets, 0, newOffsets, 0, result.offsets.length);
- result.offsets = newOffsets;
+ result.offsets = ArrayUtil.grow(result.offsets, end+1);
}
final int[] offsets = result.offsets;
@@ -290,9 +290,7 @@
// Pre-allocate for worst case 1-for-1
if (outUpto+length >= out.length) {
- char[] newOut = new char[2*(outUpto+length)];
- System.arraycopy(out, 0, newOut, 0, outUpto);
- result.result = out = newOut;
+ out = result.result = ArrayUtil.grow(out, outUpto+length+1);
}
while (upto < end) {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Version.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Version.java Fri Feb 26 13:09:54 2010
@@ -50,20 +50,32 @@
/** Match settings and bugs in Lucene's 3.0 release. */
LUCENE_30,
- /** Match settings and bugs in Lucene's 3.1 release. */
+ /** Match settings and bugs in Lucene's 3.1 release.
+ * <p>
+ * Use this to get the latest & greatest settings, bug
+ * fixes, etc, for Lucene.
+ */
LUCENE_31,
/* Add new constants for later versions **here** to respect order! */
- /** Use this to get the latest & greatest settings, bug
- * fixes, etc, for Lucene.
- *
+ /**
* <p><b>WARNING</b>: if you use this setting, and then
* upgrade to a newer release of Lucene, sizable changes
- * may happen. If precise back compatibility is important
+ * may happen. If backwards compatibility is important
* then you should instead explicitly specify an actual
* version.
+ * <p>
+ * If you use this constant then you may need to
+ * <b>re-index all of your documents</b> when upgrading
+ * Lucene, as the way text is indexed may have changed.
+ * Additionally, you may need to <b>re-test your entire
+ * application</b> to ensure it behaves as expected, as
+ * some defaults may have changed and may break functionality
+ * in your application.
+ * @deprecated Use an actual version instead.
*/
+ @Deprecated
LUCENE_CURRENT;
public boolean onOrAfter(Version other) {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/VirtualMethod.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/VirtualMethod.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/VirtualMethod.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/VirtualMethod.java Fri Feb 26 13:09:54 2010
@@ -20,7 +20,7 @@
import java.lang.reflect.Method;
import java.util.Collections;
import java.util.HashSet;
-import java.util.IdentityHashMap;
+import java.util.WeakHashMap;
import java.util.Set;
/**
@@ -54,6 +54,8 @@
* The one with the larger distance should be used preferable.
* This way also more complicated method rename scenarios can be handled
* (think of 2.9 {@code TokenStream} deprecations).</p>
+ *
+ * @lucene.internal
*/
public final class VirtualMethod<C> {
@@ -62,8 +64,8 @@
private final Class<C> baseClass;
private final String method;
private final Class<?>[] parameters;
- private final IdentityHashMap<Class<? extends C>, Integer> cache =
- new IdentityHashMap<Class<? extends C>, Integer>();
+ private final WeakHashMap<Class<? extends C>, Integer> cache =
+ new WeakHashMap<Class<? extends C>, Integer>();
/**
* Creates a new instance for the given {@code baseClass} and method declaration.
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/Cache.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/Cache.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/Cache.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/Cache.java Fri Feb 26 13:09:54 2010
@@ -21,6 +21,8 @@
/**
* Base class for cache implementations.
+ *
+ * @lucene.internal
*/
public abstract class Cache<K,V> implements Closeable {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/DoubleBarrelLRUCache.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/DoubleBarrelLRUCache.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/DoubleBarrelLRUCache.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/DoubleBarrelLRUCache.java Fri Feb 26 13:09:54 2010
@@ -38,11 +38,7 @@
* not grow unbounded in size when under hideously high
* load.</p>
*
- * <p>NOTE: this class is meant only to be used internally
- * by Lucene; it's only public so it can be shared across
- * packages. This means the API is freely subject to
- * change, and, the class could be removed entirely, in any
- * Lucene release. Use directly at your own risk!
+ * @lucene.internal
*/
final public class DoubleBarrelLRUCache<K,V> extends Cache<K,V> {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/SimpleLRUCache.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/SimpleLRUCache.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/SimpleLRUCache.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/SimpleLRUCache.java Fri Feb 26 13:09:54 2010
@@ -27,6 +27,8 @@
*
* @deprecated Lucene's internal use of this class has now
* switched to {@link DoubleBarrelLRUCache}.
+ *
+ * @lucene.internal
*/
@Deprecated
public class SimpleLRUCache<K,V> extends SimpleMapCache<K,V> {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/SimpleMapCache.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/SimpleMapCache.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/SimpleMapCache.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/SimpleMapCache.java Fri Feb 26 13:09:54 2010
@@ -28,6 +28,8 @@
*
* @deprecated Lucene's internal use of this class has now
* switched to {@link DoubleBarrelLRUCache}.
+ *
+ * @lucene.internal
*/
@Deprecated
public class SimpleMapCache<K,V> extends Cache<K,V> {
Modified: lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/fileformats.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/fileformats.xml?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/fileformats.xml (original)
+++ lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/fileformats.xml Fri Feb 26 13:09:54 2010
@@ -1846,11 +1846,12 @@
<p>
When referring to term numbers, Lucene's current
- implementation uses a Java <code>int</code>, which means
- the maximum number of unique terms in any single index
- segment is 2,147,483,648. This is technically not a
- limitation of the index file format, just of Lucene's
- current implementation.
+ implementation uses a Java <code>int</code> to hold the
+ term index, which means the maximum number of unique
+ terms in any single index segment is ~2.1 billion times
+ the term index interval (default 128) = ~274 billion.
+ This is technically not a limitation of the index file
+ format, just of Lucene's current implementation.
</p>
<p>
Similarly, Lucene uses a Java <code>int</code> to refer
Modified: lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml (original)
+++ lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml Fri Feb 26 13:09:54 2010
@@ -120,12 +120,7 @@
<section id="remote"><title>remote</title>
<p>Classes to help use Lucene with RMI.</p>
<p>See <a href="../api/contrib-remote/index.html">remote javadoc</a></p>
- </section>
-
- <section id="snowball"><title>snowball</title>
- <p>Pre-compiled versions of the Snowball stemmers for Lucene.</p>
- <p>See <a href="../api/contrib-snowball/index.html">snowball javadoc</a></p>
- </section>
+ </section>
<section id="spatial"><title>spatial</title>
<p>Classes to help with efficient distance based sorting.</p>
Modified: lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/site.xml?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/site.xml (original)
+++ lucene/java/branches/flex_1458/src/site/src/documentation/content/xdocs/site.xml Fri Feb 26 13:09:54 2010
@@ -72,7 +72,6 @@
<javadoc-contrib-similarity label="Similarity" href="ext:javadocs-contrib-similarity"/>
-->
<javadoc-contrib-remote label="Remote" href="ext:javadocs-contrib-remote"/>
- <javadoc-contrib-snowball label="Snowball" href="ext:javadocs-contrib-snowball"/>
<javadoc-contrib-spatial label="Spatial" href="ext:javadocs-contrib-spatial"/>
<javadoc-contrib-spellchecker label="Spellchecker" href="ext:javadocs-contrib-spellchecker"/>
<javadoc-contrib-surround label="Surround" href="ext:javadocs-contrib-surround"/>
@@ -125,7 +124,6 @@
<javadocs-contrib-regex href="api/contrib-regex/index.html"/>
<javadocs-contrib-remote href="api/contrib-remote/index.html"/>
<javadocs-contrib-similarity href="api/contrib-similarity/index.html"/>
- <javadocs-contrib-snowball href="api/contrib-snowball/index.html"/>
<javadocs-contrib-spatial href="api/contrib-spatial/index.html"/>
<javadocs-contrib-spellchecker href="api/contrib-spellchecker/index.html"/>
<javadocs-contrib-surround href="api/contrib-surround/index.html"/>
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestDemo.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestDemo.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestDemo.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestDemo.java Fri Feb 26 13:09:54 2010
@@ -34,7 +34,6 @@
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.Version;
/**
* A very simple demo used in the API documentation (src/java/overview.html).
@@ -46,7 +45,7 @@
public void testDemo() throws IOException, ParseException {
- Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
+ Analyzer analyzer = new StandardAnalyzer(TEST_VERSION_CURRENT);
// Store the index in memory:
Directory directory = new RAMDirectory();
@@ -67,7 +66,7 @@
assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits);
// Parse a simple query that searches for "text":
- QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, "fieldname", analyzer);
+ QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, "fieldname", analyzer);
Query query = parser.parse("text");
ScoreDoc[] hits = isearcher.search(query, null, 1).scoreDocs;
assertEquals(1, hits.length);
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestMergeSchedulerExternal.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestMergeSchedulerExternal.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestMergeSchedulerExternal.java Fri Feb 26 13:09:54 2010
@@ -95,7 +95,7 @@
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.add(idField);
- IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
+ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(TEST_VERSION_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
MyMergeScheduler ms = new MyMergeScheduler();
writer.setMergeScheduler(ms);
writer.setMaxBufferedDocs(2);
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSearch.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSearch.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSearch.java Fri Feb 26 13:09:54 2010
@@ -22,7 +22,6 @@
import java.io.StringWriter;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.Version;
import junit.framework.TestSuite;
import junit.textui.TestRunner;
@@ -74,7 +73,7 @@
throws Exception
{
Directory directory = new RAMDirectory();
- Analyzer analyzer = new SimpleAnalyzer();
+ Analyzer analyzer = new SimpleAnalyzer(TEST_VERSION_CURRENT);
IndexWriter writer = new IndexWriter(directory, analyzer, true,
IndexWriter.MaxFieldLength.LIMITED);
@@ -108,7 +107,7 @@
};
ScoreDoc[] hits = null;
- QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, "contents", analyzer);
+ QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, "contents", analyzer);
parser.setPhraseSlop(4);
for (int j = 0; j < queries.length; j++) {
Query query = parser.parse(queries[j]);
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSearchForDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSearchForDuplicates.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSearchForDuplicates.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSearchForDuplicates.java Fri Feb 26 13:09:54 2010
@@ -27,8 +27,6 @@
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.queryParser.*;
-import org.apache.lucene.util.Version;
-
import org.apache.lucene.util.LuceneTestCase;
import junit.framework.TestSuite;
import junit.textui.TestRunner;
@@ -79,7 +77,7 @@
private void doTest(PrintWriter out, boolean useCompoundFiles) throws Exception {
Directory directory = new RAMDirectory();
- Analyzer analyzer = new SimpleAnalyzer();
+ Analyzer analyzer = new SimpleAnalyzer(TEST_VERSION_CURRENT);
IndexWriter writer = new IndexWriter(directory, analyzer, true,
IndexWriter.MaxFieldLength.LIMITED);
@@ -101,7 +99,7 @@
// try a search without OR
Searcher searcher = new IndexSearcher(directory, true);
- QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, PRIORITY_FIELD, analyzer);
+ QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, PRIORITY_FIELD, analyzer);
Query query = parser.parse(HIGH_PRIORITY);
out.println("Query: " + query.toString(PRIORITY_FIELD));
@@ -116,7 +114,7 @@
searcher = new IndexSearcher(directory, true);
hits = null;
- parser = new QueryParser(Version.LUCENE_CURRENT, PRIORITY_FIELD, analyzer);
+ parser = new QueryParser(TEST_VERSION_CURRENT, PRIORITY_FIELD, analyzer);
query = parser.parse(HIGH_PRIORITY + " OR " + MED_PRIORITY);
out.println("Query: " + query.toString(PRIORITY_FIELD));
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSnapshotDeletionPolicy.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSnapshotDeletionPolicy.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSnapshotDeletionPolicy.java Fri Feb 26 13:09:54 2010
@@ -67,7 +67,7 @@
Directory dir = new MockRAMDirectory();
SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
- IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
+ IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(TEST_VERSION_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
// Force frequent flushes
writer.setMaxBufferedDocs(2);
Document doc = new Document();
@@ -83,7 +83,7 @@
writer.close();
copyFiles(dir, cp);
- writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
+ writer = new IndexWriter(dir, new StandardAnalyzer(TEST_VERSION_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
copyFiles(dir, cp);
for(int i=0;i<7;i++) {
writer.addDocument(doc);
@@ -95,7 +95,7 @@
writer.close();
copyFiles(dir, cp);
dp.release();
- writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
+ writer = new IndexWriter(dir, new StandardAnalyzer(TEST_VERSION_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
writer.close();
try {
copyFiles(dir, cp);
@@ -111,7 +111,7 @@
final long stopTime = System.currentTimeMillis() + 1000;
SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
- final IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
+ final IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(TEST_VERSION_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
// Force frequent flushes
writer.setMaxBufferedDocs(2);
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Fri Feb 26 13:09:54 2010
@@ -21,6 +21,8 @@
import java.io.IOException;
import org.apache.lucene.analysis.tokenattributes.*;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.LuceneTestCase;
/**
@@ -38,26 +40,67 @@
// some helpers to test Analyzers and TokenStreams:
- public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException {
+ public static interface CheckClearAttributesAttribute extends Attribute {
+ boolean getAndResetClearCalled();
+ }
+
+ public static final class CheckClearAttributesAttributeImpl extends AttributeImpl implements CheckClearAttributesAttribute {
+ private boolean clearCalled = false;
+
+ public boolean getAndResetClearCalled() {
+ try {
+ return clearCalled;
+ } finally {
+ clearCalled = false;
+ }
+ }
+
+ @Override
+ public void clear() {
+ clearCalled = true;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return (
+ other instanceof CheckClearAttributesAttributeImpl &&
+ ((CheckClearAttributesAttributeImpl) other).clearCalled == this.clearCalled
+ );
+ }
+
+ @Override
+ public int hashCode() {
+ return 76137213 ^ Boolean.valueOf(clearCalled).hashCode();
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ ((CheckClearAttributesAttributeImpl) target).clear();
+ }
+ }
+
+ public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], Integer finalOffset) throws IOException {
assertNotNull(output);
- assertTrue("has TermAttribute", ts.hasAttribute(TermAttribute.class));
+ CheckClearAttributesAttribute checkClearAtt = ts.addAttribute(CheckClearAttributesAttribute.class);
+
+ assertTrue("has no TermAttribute", ts.hasAttribute(TermAttribute.class));
TermAttribute termAtt = ts.getAttribute(TermAttribute.class);
OffsetAttribute offsetAtt = null;
- if (startOffsets != null || endOffsets != null) {
- assertTrue("has OffsetAttribute", ts.hasAttribute(OffsetAttribute.class));
+ if (startOffsets != null || endOffsets != null || finalOffset != null) {
+ assertTrue("has no OffsetAttribute", ts.hasAttribute(OffsetAttribute.class));
offsetAtt = ts.getAttribute(OffsetAttribute.class);
}
TypeAttribute typeAtt = null;
if (types != null) {
- assertTrue("has TypeAttribute", ts.hasAttribute(TypeAttribute.class));
+ assertTrue("has no TypeAttribute", ts.hasAttribute(TypeAttribute.class));
typeAtt = ts.getAttribute(TypeAttribute.class);
}
PositionIncrementAttribute posIncrAtt = null;
if (posIncrements != null) {
- assertTrue("has PositionIncrementAttribute", ts.hasAttribute(PositionIncrementAttribute.class));
+ assertTrue("has no PositionIncrementAttribute", ts.hasAttribute(PositionIncrementAttribute.class));
posIncrAtt = ts.getAttribute(PositionIncrementAttribute.class);
}
@@ -70,7 +113,10 @@
if (typeAtt != null) typeAtt.setType("bogusType");
if (posIncrAtt != null) posIncrAtt.setPositionIncrement(45987657);
- assertTrue("token "+i+" exists", ts.incrementToken());
+ checkClearAtt.getAndResetClearCalled(); // reset it, because we called clearAttribute() before
+ assertTrue("token "+i+" does not exist", ts.incrementToken());
+ assertTrue("clearAttributes() was not called correctly in TokenStream chain", checkClearAtt.getAndResetClearCalled());
+
assertEquals("term "+i, output[i], termAtt.term());
if (startOffsets != null)
assertEquals("startOffset "+i, startOffsets[i], offsetAtt.startOffset());
@@ -83,32 +129,45 @@
}
assertFalse("end of stream", ts.incrementToken());
ts.end();
+ if (finalOffset != null)
+ assertEquals("finalOffset ", finalOffset.intValue(), offsetAtt.endOffset());
ts.close();
}
+ public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException {
+ assertTokenStreamContents(ts, output, startOffsets, endOffsets, types, posIncrements, null);
+ }
+
public static void assertTokenStreamContents(TokenStream ts, String[] output) throws IOException {
- assertTokenStreamContents(ts, output, null, null, null, null);
+ assertTokenStreamContents(ts, output, null, null, null, null, null);
}
public static void assertTokenStreamContents(TokenStream ts, String[] output, String[] types) throws IOException {
- assertTokenStreamContents(ts, output, null, null, types, null);
+ assertTokenStreamContents(ts, output, null, null, types, null, null);
}
public static void assertTokenStreamContents(TokenStream ts, String[] output, int[] posIncrements) throws IOException {
- assertTokenStreamContents(ts, output, null, null, null, posIncrements);
+ assertTokenStreamContents(ts, output, null, null, null, posIncrements, null);
}
public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[]) throws IOException {
- assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, null);
+ assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, null, null);
+ }
+
+ public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], Integer finalOffset) throws IOException {
+ assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, null, finalOffset);
}
public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements) throws IOException {
- assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, posIncrements);
+ assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, posIncrements, null);
}
+ public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements, Integer finalOffset) throws IOException {
+ assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, posIncrements, finalOffset);
+ }
public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException {
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements);
+ assertTokenStreamContents(a.tokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements, input.length());
}
public static void assertAnalyzesTo(Analyzer a, String input, String[] output) throws IOException {
@@ -133,7 +192,7 @@
public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException {
- assertTokenStreamContents(a.reusableTokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements);
+ assertTokenStreamContents(a.reusableTokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements, input.length());
}
public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output) throws IOException {
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java Fri Feb 26 13:09:54 2010
@@ -18,7 +18,6 @@
*/
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-
import java.io.StringReader;
import java.util.List;
import java.util.ArrayList;
@@ -28,7 +27,7 @@
// testLain1Accents() is a copy of TestLatin1AccentFilter.testU().
public void testLatin1Accents() throws Exception {
- TokenStream stream = new WhitespaceTokenizer(new StringReader
+ TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader
("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à IJ à Ã"
+" à à à à à à Šà à à à à à Ÿ à á â ã ä å æ ç è é ê ë ì à î ï ij"
+" ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ ï¬ ï¬"));
@@ -1889,7 +1888,7 @@
expectedOutputTokens.add(expected.toString());
}
- TokenStream stream = new WhitespaceTokenizer(new StringReader(inputText.toString()));
+ TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(inputText.toString()));
ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
Iterator<String> expectedIter = expectedOutputTokens.iterator();
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestAnalyzers.java Fri Feb 26 13:09:54 2010
@@ -26,7 +26,6 @@
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.index.Payload;
-import org.apache.lucene.util.Version;
public class TestAnalyzers extends BaseTokenStreamTestCase {
@@ -35,7 +34,7 @@
}
public void testSimple() throws Exception {
- Analyzer a = new SimpleAnalyzer();
+ Analyzer a = new SimpleAnalyzer(TEST_VERSION_CURRENT);
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
@@ -55,7 +54,7 @@
}
public void testNull() throws Exception {
- Analyzer a = new WhitespaceAnalyzer();
+ Analyzer a = new WhitespaceAnalyzer(TEST_VERSION_CURRENT);
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "FOO", "BAR" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
@@ -75,7 +74,7 @@
}
public void testStop() throws Exception {
- Analyzer a = new StopAnalyzer(Version.LUCENE_CURRENT);
+ Analyzer a = new StopAnalyzer(TEST_VERSION_CURRENT);
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo a bar such FOO THESE BAR",
@@ -97,11 +96,11 @@
public void testPayloadCopy() throws IOException {
String s = "how now brown cow";
TokenStream ts;
- ts = new WhitespaceTokenizer(new StringReader(s));
+ ts = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(s));
ts = new PayloadSetter(ts);
verifyPayload(ts);
- ts = new WhitespaceTokenizer(new StringReader(s));
+ ts = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(s));
ts = new PayloadSetter(ts);
verifyPayload(ts);
}
@@ -122,12 +121,12 @@
private static class MyStandardAnalyzer extends StandardAnalyzer {
public MyStandardAnalyzer() {
- super(org.apache.lucene.util.Version.LUCENE_CURRENT);
+ super(TEST_VERSION_CURRENT);
}
@Override
public TokenStream tokenStream(String field, Reader reader) {
- return new WhitespaceAnalyzer().tokenStream(field, reader);
+ return new WhitespaceAnalyzer(TEST_VERSION_CURRENT).tokenStream(field, reader);
}
}
@@ -144,8 +143,8 @@
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new LowerCaseFilter(Version.LUCENE_CURRENT,
- new WhitespaceTokenizer(reader));
+ return new LowerCaseFilter(TEST_VERSION_CURRENT,
+ new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader));
}
}
@@ -192,9 +191,9 @@
public void testLowerCaseFilterLowSurrogateLeftover() throws IOException {
// test if the limit of the termbuffer is correctly used with supplementary
// chars
- WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(new StringReader(
- "BogustermBogusterm\udc16"));
- LowerCaseFilter filter = new LowerCaseFilter(Version.LUCENE_CURRENT,
+ WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT,
+ new StringReader("BogustermBogusterm\udc16"));
+ LowerCaseFilter filter = new LowerCaseFilter(TEST_VERSION_CURRENT,
tokenizer);
assertTokenStreamContents(filter, new String[] {"bogustermbogusterm\udc16"});
filter.reset();
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java Fri Feb 26 13:09:54 2010
@@ -37,7 +37,7 @@
public void testCaching() throws IOException {
Directory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
+ IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer(TEST_VERSION_CURRENT), IndexWriter.MaxFieldLength.LIMITED);
Document doc = new Document();
TokenStream stream = new TokenStream() {
private int index = 0;
@@ -49,6 +49,7 @@
if (index == tokens.length) {
return false;
} else {
+ clearAttributes();
termAtt.setTermBuffer(tokens[index++]);
offsetAtt.setOffset(0,0);
return true;
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestCharArraySet.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestCharArraySet.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestCharArraySet.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestCharArraySet.java Fri Feb 26 13:09:54 2010
@@ -19,9 +19,11 @@
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.Iterator;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Version;
@@ -39,7 +41,7 @@
public void testRehash() throws Exception {
- CharArraySet cas = new CharArraySet(Version.LUCENE_CURRENT, 0, true);
+ CharArraySet cas = new CharArraySet(TEST_VERSION_CURRENT, 0, true);
for(int i=0;i<TEST_STOP_WORDS.length;i++)
cas.add(TEST_STOP_WORDS[i]);
assertEquals(TEST_STOP_WORDS.length, cas.size());
@@ -50,7 +52,7 @@
public void testNonZeroOffset() {
String[] words={"Hello","World","this","is","a","test"};
char[] findme="xthisy".toCharArray();
- CharArraySet set=new CharArraySet(Version.LUCENE_CURRENT, 10,true);
+ CharArraySet set=new CharArraySet(TEST_VERSION_CURRENT, 10,true);
set.addAll(Arrays.asList(words));
assertTrue(set.contains(findme, 1, 4));
assertTrue(set.contains(new String(findme,1,4)));
@@ -62,7 +64,7 @@
}
public void testObjectContains() {
- CharArraySet set = new CharArraySet(Version.LUCENE_CURRENT, 10, true);
+ CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 10, true);
Integer val = Integer.valueOf(1);
set.add(val);
assertTrue(set.contains(val));
@@ -78,7 +80,7 @@
}
public void testClear(){
- CharArraySet set=new CharArraySet(Version.LUCENE_CURRENT, 10,true);
+ CharArraySet set=new CharArraySet(TEST_VERSION_CURRENT, 10,true);
set.addAll(Arrays.asList(TEST_STOP_WORDS));
assertEquals("Not all words added", TEST_STOP_WORDS.length, set.size());
set.clear();
@@ -92,7 +94,7 @@
}
public void testModifyOnUnmodifiable(){
- CharArraySet set=new CharArraySet(Version.LUCENE_CURRENT, 10,true);
+ CharArraySet set=new CharArraySet(TEST_VERSION_CURRENT, 10, true);
set.addAll(Arrays.asList(TEST_STOP_WORDS));
final int size = set.size();
set = CharArraySet.unmodifiableSet(set);
@@ -143,8 +145,12 @@
assertFalse("Test String has been added to unmodifiable set", set.contains(NOT_IN_SET));
assertEquals("Size of unmodifiable set has changed", size, set.size());
}
+
+ // This test was changed in 3.1, as a contains() call on the given Collection using the "correct" iterator's
+ // current key (now a char[]) on a Set<String> would not hit any element of the CAS and therefor never call
+ // remove() on the iterator
try{
- set.removeAll(Arrays.asList(TEST_STOP_WORDS));
+ set.removeAll(new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList(TEST_STOP_WORDS), true));
fail("Modified unmodifiable set");
}catch (UnsupportedOperationException e) {
// expected
@@ -152,7 +158,7 @@
}
try{
- set.retainAll(Arrays.asList(new String[]{NOT_IN_SET}));
+ set.retainAll(new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList(NOT_IN_SET), true));
fail("Modified unmodifiable set");
}catch (UnsupportedOperationException e) {
// expected
@@ -173,7 +179,7 @@
}
public void testUnmodifiableSet(){
- CharArraySet set = new CharArraySet(Version.LUCENE_CURRENT, 10,true);
+ CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 10,true);
set.addAll(Arrays.asList(TEST_STOP_WORDS));
set.add(Integer.valueOf(1));
final int size = set.size();
@@ -203,7 +209,7 @@
"\ud801\udc1c\ud801\udc1cCDE", "A\ud801\udc1cB"};
String[] lowerArr = new String[] {"abc\ud801\udc44",
"\ud801\udc44\ud801\udc44cde", "a\ud801\udc44b"};
- CharArraySet set = new CharArraySet(Version.LUCENE_31, Arrays.asList(TEST_STOP_WORDS), true);
+ CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList(TEST_STOP_WORDS), true);
for (String upper : upperArr) {
set.add(upper);
}
@@ -211,7 +217,7 @@
assertTrue(String.format(missing, upperArr[i]), set.contains(upperArr[i]));
assertTrue(String.format(missing, lowerArr[i]), set.contains(lowerArr[i]));
}
- set = new CharArraySet(Version.LUCENE_31, Arrays.asList(TEST_STOP_WORDS), false);
+ set = new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList(TEST_STOP_WORDS), false);
for (String upper : upperArr) {
set.add(upper);
}
@@ -229,7 +235,7 @@
String[] lowerArr = new String[] { "abc\uD800", "abc\uD800efg",
"\uD800efg", "\uD800\ud801\udc44b" };
- CharArraySet set = new CharArraySet(Version.LUCENE_31, Arrays
+ CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, Arrays
.asList(TEST_STOP_WORDS), true);
for (String upper : upperArr) {
set.add(upper);
@@ -238,7 +244,7 @@
assertTrue(String.format(missing, upperArr[i]), set.contains(upperArr[i]));
assertTrue(String.format(missing, lowerArr[i]), set.contains(lowerArr[i]));
}
- set = new CharArraySet(Version.LUCENE_31, Arrays.asList(TEST_STOP_WORDS),
+ set = new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList(TEST_STOP_WORDS),
false);
for (String upper : upperArr) {
set.add(upper);
@@ -321,12 +327,56 @@
}
}
+ public void testCopyCharArraySetBWCompat() {
+ CharArraySet setIngoreCase = new CharArraySet(TEST_VERSION_CURRENT, 10, true);
+ CharArraySet setCaseSensitive = new CharArraySet(TEST_VERSION_CURRENT, 10, false);
+
+ List<String> stopwords = Arrays.asList(TEST_STOP_WORDS);
+ List<String> stopwordsUpper = new ArrayList<String>();
+ for (String string : stopwords) {
+ stopwordsUpper.add(string.toUpperCase());
+ }
+ setIngoreCase.addAll(Arrays.asList(TEST_STOP_WORDS));
+ setIngoreCase.add(Integer.valueOf(1));
+ setCaseSensitive.addAll(Arrays.asList(TEST_STOP_WORDS));
+ setCaseSensitive.add(Integer.valueOf(1));
+
+ CharArraySet copy = CharArraySet.copy(setIngoreCase);
+ CharArraySet copyCaseSens = CharArraySet.copy(setCaseSensitive);
+
+ assertEquals(setIngoreCase.size(), copy.size());
+ assertEquals(setCaseSensitive.size(), copy.size());
+
+ assertTrue(copy.containsAll(stopwords));
+ assertTrue(copy.containsAll(stopwordsUpper));
+ assertTrue(copyCaseSens.containsAll(stopwords));
+ for (String string : stopwordsUpper) {
+ assertFalse(copyCaseSens.contains(string));
+ }
+ // test adding terms to the copy
+ List<String> newWords = new ArrayList<String>();
+ for (String string : stopwords) {
+ newWords.add(string+"_1");
+ }
+ copy.addAll(newWords);
+
+ assertTrue(copy.containsAll(stopwords));
+ assertTrue(copy.containsAll(stopwordsUpper));
+ assertTrue(copy.containsAll(newWords));
+ // new added terms are not in the source set
+ for (String string : newWords) {
+ assertFalse(setIngoreCase.contains(string));
+ assertFalse(setCaseSensitive.contains(string));
+
+ }
+ }
+
/**
* Test the static #copy() function with a CharArraySet as a source
*/
public void testCopyCharArraySet() {
- CharArraySet setIngoreCase = new CharArraySet(Version.LUCENE_CURRENT, 10, true);
- CharArraySet setCaseSensitive = new CharArraySet(Version.LUCENE_CURRENT, 10, false);
+ CharArraySet setIngoreCase = new CharArraySet(TEST_VERSION_CURRENT, 10, true);
+ CharArraySet setCaseSensitive = new CharArraySet(TEST_VERSION_CURRENT, 10, false);
List<String> stopwords = Arrays.asList(TEST_STOP_WORDS);
List<String> stopwordsUpper = new ArrayList<String>();
@@ -338,8 +388,8 @@
setCaseSensitive.addAll(Arrays.asList(TEST_STOP_WORDS));
setCaseSensitive.add(Integer.valueOf(1));
- CharArraySet copy = CharArraySet.copy(Version.LUCENE_CURRENT, setIngoreCase);
- CharArraySet copyCaseSens = CharArraySet.copy(Version.LUCENE_CURRENT, setCaseSensitive);
+ CharArraySet copy = CharArraySet.copy(TEST_VERSION_CURRENT, setIngoreCase);
+ CharArraySet copyCaseSens = CharArraySet.copy(TEST_VERSION_CURRENT, setCaseSensitive);
assertEquals(setIngoreCase.size(), copy.size());
assertEquals(setCaseSensitive.size(), copy.size());
@@ -381,7 +431,7 @@
}
set.addAll(Arrays.asList(TEST_STOP_WORDS));
- CharArraySet copy = CharArraySet.copy(Version.LUCENE_CURRENT, set);
+ CharArraySet copy = CharArraySet.copy(TEST_VERSION_CURRENT, set);
assertEquals(set.size(), copy.size());
assertEquals(set.size(), copy.size());
@@ -404,4 +454,85 @@
assertFalse(set.contains(string));
}
}
+
+ /**
+ * Tests a special case of {@link CharArraySet#copy(Version, Set)} where the
+ * set to copy is the {@link CharArraySet#EMPTY_SET}
+ */
+ public void testCopyEmptySet() {
+ assertSame(CharArraySet.EMPTY_SET,
+ CharArraySet.copy(TEST_VERSION_CURRENT, CharArraySet.EMPTY_SET));
+ }
+
+ /**
+ * Smoketests the static empty set
+ */
+ public void testEmptySet() {
+ assertEquals(0, CharArraySet.EMPTY_SET.size());
+
+ assertTrue(CharArraySet.EMPTY_SET.isEmpty());
+ for (String stopword : TEST_STOP_WORDS) {
+ assertFalse(CharArraySet.EMPTY_SET.contains(stopword));
+ }
+ assertFalse(CharArraySet.EMPTY_SET.contains((Object) "foo"));
+ assertFalse(CharArraySet.EMPTY_SET.contains((Object) "foo".toCharArray()));
+ assertFalse(CharArraySet.EMPTY_SET.contains("foo".toCharArray(),0,3));
+ }
+
+ /**
+ * Test for NPE
+ */
+ public void testContainsWithNull() {
+ CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
+ try {
+ set.contains((char[]) null, 0, 10);
+ fail("null value must raise NPE");
+ } catch (NullPointerException e) {}
+ try {
+ set.contains((CharSequence) null);
+ fail("null value must raise NPE");
+ } catch (NullPointerException e) {}
+ try {
+ set.contains((Object) null);
+ fail("null value must raise NPE");
+ } catch (NullPointerException e) {}
+ }
+
+ @Deprecated @SuppressWarnings("unchecked")
+ public void testIterator() {
+ HashSet<String> hset = new HashSet<String>();
+ hset.addAll(Arrays.asList(TEST_STOP_WORDS));
+
+ assertTrue("in 3.0 version, iterator should be CharArraySetIterator",
+ ((Iterator) CharArraySet.copy(Version.LUCENE_30, hset).iterator()) instanceof CharArraySet.CharArraySetIterator);
+
+ CharArraySet set = CharArraySet.copy(TEST_VERSION_CURRENT, hset);
+ assertFalse("in current version, iterator should not be CharArraySetIterator",
+ ((Iterator) set.iterator()) instanceof CharArraySet.CharArraySetIterator);
+
+ Iterator<String> it = set.stringIterator();
+ assertTrue(it instanceof CharArraySet.CharArraySetIterator);
+ while (it.hasNext()) {
+ // as the set returns String instances, this must work:
+ assertTrue(hset.contains(it.next()));
+ try {
+ it.remove();
+ fail("remove() should not work on CharArraySetIterator");
+ } catch (UnsupportedOperationException uoe) {
+ // pass
+ }
+ }
+ }
+
+ public void testToString() {
+ CharArraySet set = CharArraySet.copy(TEST_VERSION_CURRENT, Collections.singleton("test"));
+ assertEquals("[test]", set.toString());
+ set.add("test2");
+ assertTrue(set.toString().contains(", "));
+
+ set = CharArraySet.copy(Version.LUCENE_30, Collections.singleton("test"));
+ assertEquals("[test]", set.toString());
+ set.add("test2");
+ assertTrue(set.toString().contains(", "));
+ }
}
Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java Fri Feb 26 13:09:54 2010
@@ -18,12 +18,11 @@
*/
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-
import java.io.StringReader;
public class TestISOLatin1AccentFilter extends BaseTokenStreamTestCase {
public void testU() throws Exception {
- TokenStream stream = new WhitespaceTokenizer(new StringReader("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à IJ à à à à à à à à Šà à à à à à Ÿ à á â ã ä Ã¥ æ ç è é ê ë ì à î ï ij ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ ï¬ ï¬"));
+ TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à IJ à à à à à à à à Šà à à à à à Ÿ à á â ã ä Ã¥ æ ç è é ê ë ì à î ï ij ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ ï¬ ï¬"));
ISOLatin1AccentFilter filter = new ISOLatin1AccentFilter(stream);
TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
assertTermEquals("Des", filter, termAtt);