You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by us...@apache.org on 2010/04/06 21:19:36 UTC
svn commit: r931278 [2/10] - in /lucene/dev/trunk: lucene/
lucene/backwards/src/ lucene/backwards/src/java/org/apache/lucene/index/
lucene/backwards/src/java/org/apache/lucene/index/codecs/
lucene/backwards/src/java/org/apache/lucene/search/ lucene/bac...
Modified: lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/store/IndexOutput.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/store/IndexOutput.java Tue Apr 6 19:19:27 2010
@@ -17,166 +17,15 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.Closeable;
-import java.util.Map;
-import org.apache.lucene.util.UnicodeUtil;
+import java.io.IOException;
/** Abstract base class for output to a file in a Directory. A random-access
* output stream. Used for all Lucene index output operations.
* @see Directory
* @see IndexInput
*/
-public abstract class IndexOutput implements Closeable {
-
- private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result();
-
- /** Writes a single byte.
- * @see IndexInput#readByte()
- */
- public abstract void writeByte(byte b) throws IOException;
-
- /** Writes an array of bytes.
- * @param b the bytes to write
- * @param length the number of bytes to write
- * @see IndexInput#readBytes(byte[],int,int)
- */
- public void writeBytes(byte[] b, int length) throws IOException {
- writeBytes(b, 0, length);
- }
-
- /** Writes an array of bytes.
- * @param b the bytes to write
- * @param offset the offset in the byte array
- * @param length the number of bytes to write
- * @see IndexInput#readBytes(byte[],int,int)
- */
- public abstract void writeBytes(byte[] b, int offset, int length) throws IOException;
-
- /** Writes an int as four bytes.
- * @see IndexInput#readInt()
- */
- public void writeInt(int i) throws IOException {
- writeByte((byte)(i >> 24));
- writeByte((byte)(i >> 16));
- writeByte((byte)(i >> 8));
- writeByte((byte) i);
- }
-
- /** Writes an int in a variable-length format. Writes between one and
- * five bytes. Smaller values take fewer bytes. Negative numbers are not
- * supported.
- * @see IndexInput#readVInt()
- */
- public void writeVInt(int i) throws IOException {
- while ((i & ~0x7F) != 0) {
- writeByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writeByte((byte)i);
- }
-
- /** Writes a long as eight bytes.
- * @see IndexInput#readLong()
- */
- public void writeLong(long i) throws IOException {
- writeInt((int) (i >> 32));
- writeInt((int) i);
- }
-
- /** Writes an long in a variable-length format. Writes between one and five
- * bytes. Smaller values take fewer bytes. Negative numbers are not
- * supported.
- * @see IndexInput#readVLong()
- */
- public void writeVLong(long i) throws IOException {
- while ((i & ~0x7F) != 0) {
- writeByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writeByte((byte)i);
- }
-
- /** Writes a string.
- * @see IndexInput#readString()
- */
- public void writeString(String s) throws IOException {
- UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
- writeVInt(utf8Result.length);
- writeBytes(utf8Result.result, 0, utf8Result.length);
- }
-
- /** Writes a sub sequence of characters from s as the old
- * format (modified UTF-8 encoded bytes).
- * @param s the source of the characters
- * @param start the first character in the sequence
- * @param length the number of characters in the sequence
- * @deprecated -- please pre-convert to utf8 bytes
- * instead or use {@link #writeString}
- */
- public void writeChars(String s, int start, int length)
- throws IOException {
- final int end = start + length;
- for (int i = start; i < end; i++) {
- final int code = (int)s.charAt(i);
- if (code >= 0x01 && code <= 0x7F)
- writeByte((byte)code);
- else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
- writeByte((byte)(0xC0 | (code >> 6)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- } else {
- writeByte((byte)(0xE0 | (code >>> 12)));
- writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- }
- }
- }
-
- /** Writes a sub sequence of characters from char[] as
- * the old format (modified UTF-8 encoded bytes).
- * @param s the source of the characters
- * @param start the first character in the sequence
- * @param length the number of characters in the sequence
- * @deprecated -- please pre-convert to utf8 bytes instead or use {@link #writeString}
- */
- public void writeChars(char[] s, int start, int length)
- throws IOException {
- final int end = start + length;
- for (int i = start; i < end; i++) {
- final int code = (int)s[i];
- if (code >= 0x01 && code <= 0x7F)
- writeByte((byte)code);
- else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
- writeByte((byte)(0xC0 | (code >> 6)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- } else {
- writeByte((byte)(0xE0 | (code >>> 12)));
- writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- }
- }
- }
-
- private static int COPY_BUFFER_SIZE = 16384;
- private byte[] copyBuffer;
-
- /** Copy numBytes bytes from input to ourself. */
- public void copyBytes(IndexInput input, long numBytes) throws IOException {
- assert numBytes >= 0: "numBytes=" + numBytes;
- long left = numBytes;
- if (copyBuffer == null)
- copyBuffer = new byte[COPY_BUFFER_SIZE];
- while(left > 0) {
- final int toCopy;
- if (left > COPY_BUFFER_SIZE)
- toCopy = COPY_BUFFER_SIZE;
- else
- toCopy = (int) left;
- input.readBytes(copyBuffer, 0, toCopy);
- writeBytes(copyBuffer, 0, toCopy);
- left -= toCopy;
- }
- }
+public abstract class IndexOutput extends DataOutput implements Closeable {
/** Forces any buffered output to be written. */
public abstract void flush() throws IOException;
@@ -208,17 +57,5 @@ public abstract class IndexOutput implem
* undefined. Otherwise the file is truncated.
* @param length file length
*/
- public void setLength(long length) throws IOException {};
-
- public void writeStringStringMap(Map<String,String> map) throws IOException {
- if (map == null) {
- writeInt(0);
- } else {
- writeInt(map.size());
- for(final Map.Entry<String, String> entry: map.entrySet()) {
- writeString(entry.getKey());
- writeString(entry.getValue());
- }
- }
- }
+ public void setLength(long length) throws IOException {}
}
Modified: lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/util/UnicodeUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/util/UnicodeUtil.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/util/UnicodeUtil.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/util/UnicodeUtil.java Tue Apr 6 19:19:27 2010
@@ -106,6 +106,10 @@ final public class UnicodeUtil {
}
}
+ // stubs for tests only
+ public static void UTF16toUTF8(char[] source, int offset, int length, BytesRef result) {}
+ public static void UTF16toUTF8(CharSequence s, int offset, int length, BytesRef result) {}
+
/** Encode characters from a char[] source, starting at
* offset and stopping when the character 0xffff is seen.
* Returns the number of bytes written to bytesOut. */
@@ -223,7 +227,7 @@ final public class UnicodeUtil {
/** Encode characters from this String, starting at offset
* for length characters. Returns the number of bytes
* written to bytesOut. */
- public static void UTF16toUTF8(final String s, final int offset, final int length, UTF8Result result) {
+ public static void UTF16toUTF8(final CharSequence s, final int offset, final int length, UTF8Result result) {
final int end = offset + length;
byte[] out = result.result;
Propchange: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 6 19:19:27 2010
@@ -1,5 +1,7 @@
+/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036
/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:924483-925561
+/lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:880869-912395
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:924483-924731,924781,925176-925462
/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java Tue Apr 6 19:19:27 2010
@@ -107,10 +107,10 @@ public class TestTermAttributeImpl exten
char[] b = {'a', 'l', 'o', 'h', 'a'};
TermAttributeImpl t = new TermAttributeImpl();
t.setTermBuffer(b, 0, 5);
- assertEquals("term=aloha", t.toString());
+ assertEquals("aloha", t.toString());
t.setTermBuffer("hi there");
- assertEquals("term=hi there", t.toString());
+ assertEquals("hi there", t.toString());
}
public void testMixedStringArray() throws Exception {
Propchange: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 6 19:19:27 2010
@@ -1,5 +1,7 @@
+/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestDateTools.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036
/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestDateTools.java:924483-925561
+/lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/document/TestDateTools.java:880869-912395
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestDateTools.java:924483-924731,924781,925176-925462
/lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372
Propchange: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 6 19:19:27 2010
@@ -1,5 +1,7 @@
+/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestNumberTools.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036
/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestNumberTools.java:924483-925561
+/lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/document/TestNumberTools.java:880869-912395
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestNumberTools.java:924483-924731,924781,925176-925462
/lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372
Propchange: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 6 19:19:27 2010
@@ -1,5 +1,7 @@
+/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036
/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:924483-925561
+/lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880869-912395
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:924483-924731,924781,925176-925462
/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java Tue Apr 6 19:19:27 2010
@@ -35,6 +35,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.index.codecs.CodecProvider;
/** JUnit adaptation of an older test case DocTest. */
@@ -180,20 +181,24 @@ public class TestDoc extends LuceneTestC
SegmentReader r1 = SegmentReader.get(true, si1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
SegmentReader r2 = SegmentReader.get(true, si2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
- SegmentMerger merger = new SegmentMerger(si1.dir, merged);
+ SegmentMerger merger = new SegmentMerger(si1.dir, IndexWriter.DEFAULT_TERM_INDEX_INTERVAL, merged, null, CodecProvider.getDefault());
merger.add(r1);
merger.add(r2);
merger.merge();
merger.closeReaders();
+ final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir,
+ useCompoundFile, true, -1, null, false, merger.hasProx(),
+ merger.getCodec());
+
if (useCompoundFile) {
- List filesToDelete = merger.createCompoundFile(merged + ".cfs");
+ List filesToDelete = merger.createCompoundFile(merged + ".cfs", info);
for (Iterator iter = filesToDelete.iterator(); iter.hasNext();)
si1.dir.deleteFile((String) iter.next());
}
- return new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir, useCompoundFile, true);
+ return info;
}
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java Tue Apr 6 19:19:27 2010
@@ -986,29 +986,7 @@ public class TestIndexReader extends Luc
// new IndexFileDeleter, have it delete
// unreferenced files, then verify that in fact
// no files were deleted:
- String[] startFiles = dir.listAll();
- SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
- new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
- String[] endFiles = dir.listAll();
-
- Arrays.sort(startFiles);
- Arrays.sort(endFiles);
-
- //for(int i=0;i<startFiles.length;i++) {
- // System.out.println(" startFiles: " + i + ": " + startFiles[i]);
- //}
-
- if (!Arrays.equals(startFiles, endFiles)) {
- String successStr;
- if (success) {
- successStr = "success";
- } else {
- successStr = "IOException";
- err.printStackTrace();
- }
- fail("reader.close() failed to delete unreferenced files after " + successStr + " (" + diskFree + " bytes): before delete:\n " + arrayToString(startFiles) + "\n after delete:\n " + arrayToString(endFiles));
- }
+ TestIndexWriter.assertNoUnreferencedFiles(dir, "reader.close() failed to delete unreferenced files");
// Finally, verify index is not corrupt, and, if
// we succeeded, we see all docs changed, and if
@@ -1760,7 +1738,6 @@ public class TestIndexReader extends Luc
} catch (IllegalStateException ise) {
// expected
}
- assertFalse(((SegmentReader) r.getSequentialSubReaders()[0]).termsIndexLoaded());
assertEquals(-1, ((SegmentReader) r.getSequentialSubReaders()[0]).getTermInfosIndexDivisor());
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
@@ -1773,7 +1750,12 @@ public class TestIndexReader extends Luc
IndexReader[] subReaders = r2.getSequentialSubReaders();
assertEquals(2, subReaders.length);
for(int i=0;i<2;i++) {
- assertFalse(((SegmentReader) subReaders[i]).termsIndexLoaded());
+ try {
+ subReaders[i].docFreq(new Term("field", "f"));
+ fail("did not hit expected exception");
+ } catch (IllegalStateException ise) {
+ // expected
+ }
}
r2.close();
dir.close();
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Apr 6 19:19:27 2010
@@ -61,8 +61,10 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.MockRAMDirectory;
+import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.Version;
@@ -524,10 +526,15 @@ public class TestIndexWriter extends Luc
}
public static void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
- String[] startFiles = dir.listAll();
- SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
- new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
+ final LockFactory lf = dir.getLockFactory();
+ String[] startFiles;
+ try {
+ dir.setLockFactory(new NoLockFactory());
+ startFiles = dir.listAll();
+ new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED).close();
+ } finally {
+ dir.setLockFactory(lf);
+ }
String[] endFiles = dir.listAll();
Arrays.sort(startFiles);
@@ -3309,7 +3316,7 @@ public class TestIndexWriter extends Luc
// LUCENE-510
public void testAllUnicodeChars() throws Throwable {
- UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result();
+ BytesRef utf8 = new BytesRef(10);
UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
char[] chars = new char[2];
for(int ch=0;ch<0x0010FFFF;ch++) {
@@ -3329,16 +3336,16 @@ public class TestIndexWriter extends Luc
UnicodeUtil.UTF16toUTF8(chars, 0, len, utf8);
String s1 = new String(chars, 0, len);
- String s2 = new String(utf8.result, 0, utf8.length, "UTF-8");
+ String s2 = new String(utf8.bytes, 0, utf8.length, "UTF-8");
assertEquals("codepoint " + ch, s1, s2);
- UnicodeUtil.UTF8toUTF16(utf8.result, 0, utf8.length, utf16);
+ UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16);
assertEquals("codepoint " + ch, s1, new String(utf16.result, 0, utf16.length));
byte[] b = s1.getBytes("UTF-8");
assertEquals(utf8.length, b.length);
for(int j=0;j<utf8.length;j++)
- assertEquals(utf8.result[j], b[j]);
+ assertEquals(utf8.bytes[j], b[j]);
}
}
@@ -3403,7 +3410,7 @@ public class TestIndexWriter extends Luc
char[] buffer = new char[20];
char[] expected = new char[20];
- UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result();
+ BytesRef utf8 = new BytesRef(20);
UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
for(int iter=0;iter<100000;iter++) {
@@ -3414,10 +3421,10 @@ public class TestIndexWriter extends Luc
byte[] b = new String(buffer, 0, 20).getBytes("UTF-8");
assertEquals(b.length, utf8.length);
for(int i=0;i<b.length;i++)
- assertEquals(b[i], utf8.result[i]);
+ assertEquals(b[i], utf8.bytes[i]);
}
- UnicodeUtil.UTF8toUTF16(utf8.result, 0, utf8.length, utf16);
+ UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16);
assertEquals(utf16.length, 20);
for(int i=0;i<20;i++)
assertEquals(expected[i], utf16.result[i]);
@@ -3430,7 +3437,7 @@ public class TestIndexWriter extends Luc
char[] buffer = new char[20];
char[] expected = new char[20];
- UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result();
+ BytesRef utf8 = new BytesRef(20);
UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
UnicodeUtil.UTF16Result utf16a = new UnicodeUtil.UTF16Result();
@@ -3453,7 +3460,7 @@ public class TestIndexWriter extends Luc
byte[] b = new String(buffer, 0, 20).getBytes("UTF-8");
assertEquals(b.length, utf8.length);
for(int i=0;i<b.length;i++)
- assertEquals(b[i], utf8.result[i]);
+ assertEquals(b[i], utf8.bytes[i]);
}
int bytePrefix = 20;
@@ -3461,18 +3468,18 @@ public class TestIndexWriter extends Luc
bytePrefix = 0;
else
for(int i=0;i<20;i++)
- if (last[i] != utf8.result[i]) {
+ if (last[i] != utf8.bytes[i]) {
bytePrefix = i;
break;
}
- System.arraycopy(utf8.result, 0, last, 0, utf8.length);
+ System.arraycopy(utf8.bytes, 0, last, 0, utf8.length);
- UnicodeUtil.UTF8toUTF16(utf8.result, bytePrefix, utf8.length-bytePrefix, utf16);
+ UnicodeUtil.UTF8toUTF16(utf8.bytes, bytePrefix, utf8.length-bytePrefix, utf16);
assertEquals(20, utf16.length);
for(int i=0;i<20;i++)
assertEquals(expected[i], utf16.result[i]);
- UnicodeUtil.UTF8toUTF16(utf8.result, 0, utf8.length, utf16a);
+ UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16a);
assertEquals(20, utf16a.length);
for(int i=0;i<20;i++)
assertEquals(expected[i], utf16a.result[i]);
@@ -4331,10 +4338,6 @@ public class TestIndexWriter extends Luc
assertTrue(dir.fileExists("myrandomfile"));
- // Make sure this does not copy myrandomfile:
- Directory dir2 = new RAMDirectory(dir);
- assertTrue(!dir2.fileExists("myrandomfile"));
-
} finally {
dir.close();
_TestUtil.rmDir(indexDir);
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Tue Apr 6 19:19:27 2010
@@ -784,20 +784,8 @@ public class TestIndexWriterDelete exten
}
}
- String[] startFiles = dir.listAll();
- SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
- new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
- String[] endFiles = dir.listAll();
-
- if (!Arrays.equals(startFiles, endFiles)) {
- fail("docswriter abort() failed to delete unreferenced files:\n before delete:\n "
- + arrayToString(startFiles) + "\n after delete:\n "
- + arrayToString(endFiles));
- }
-
- modifier.close();
-
+ TestIndexWriter.assertNoUnreferencedFiles(dir, "docsWriter.abort() failed to delete unreferenced files");
+ modifier.close();
}
private String arrayToString(String[] l) {
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterReader.java Tue Apr 6 19:19:27 2010
@@ -86,7 +86,7 @@ public class TestIndexWriterReader exten
// get a reader
IndexReader r1 = writer.getReader();
- assertTrue(r1.isCurrent());
+ //assertTrue(r1.isCurrent());
String id10 = r1.document(10).getField("id").stringValue();
@@ -94,7 +94,7 @@ public class TestIndexWriterReader exten
newDoc.removeField("id");
newDoc.add(new Field("id", Integer.toString(8000), Store.YES, Index.NOT_ANALYZED));
writer.updateDocument(new Term("id", id10), newDoc);
- assertFalse(r1.isCurrent());
+ //assertFalse(r1.isCurrent());
IndexReader r2 = writer.getReader();
assertTrue(r2.isCurrent());
@@ -157,7 +157,7 @@ public class TestIndexWriterReader exten
IndexReader r0 = writer.getReader();
assertTrue(r0.isCurrent());
writer.addIndexesNoOptimize(new Directory[] { dir2 });
- assertFalse(r0.isCurrent());
+ //assertFalse(r0.isCurrent());
r0.close();
IndexReader r1 = writer.getReader();
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Tue Apr 6 19:19:27 2010
@@ -48,7 +48,7 @@ public class TestLazyProxSkipping extend
@Override
public IndexInput openInput(String name) throws IOException {
IndexInput ii = super.openInput(name);
- if (name.endsWith(".prx")) {
+ if (name.endsWith(".prx") || name.endsWith(".pos")) {
// we decorate the proxStream with a wrapper class that allows to count the number of calls of seek()
ii = new SeeksCountingStream(ii);
}
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Tue Apr 6 19:19:27 2010
@@ -30,6 +30,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MockRAMDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
@@ -42,6 +43,16 @@ import org.apache.lucene.util.LuceneTest
*
*/
public class TestMultiLevelSkipList extends LuceneTestCase {
+
+ class CountingRAMDirectory extends MockRAMDirectory {
+ public IndexInput openInput(String fileName) throws IOException {
+ IndexInput in = super.openInput(fileName);
+ if (fileName.endsWith(".frq"))
+ in = new CountingStream(in);
+ return in;
+ }
+ }
+
public void testSimpleSkip() throws IOException {
RAMDirectory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new PayloadAnalyzer(), true,
@@ -57,8 +68,7 @@ public class TestMultiLevelSkipList exte
writer.close();
IndexReader reader = SegmentReader.getOnlySegmentReader(dir);
- SegmentTermPositions tp = (SegmentTermPositions) reader.termPositions();
- tp.freqStream = new CountingStream(tp.freqStream);
+ TermPositions tp = reader.termPositions();
for (int i = 0; i < 2; i++) {
counter = 0;
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestPayloads.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestPayloads.java Tue Apr 6 19:19:27 2010
@@ -39,6 +39,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
@@ -257,10 +258,12 @@ public class TestPayloads extends Lucene
tp.next();
tp.nextPosition();
// now we don't read this payload
+ tp.next();
tp.nextPosition();
assertEquals("Wrong payload length.", 1, tp.getPayloadLength());
byte[] payload = tp.getPayload(null, 0);
assertEquals(payload[0], payloadData[numTerms]);
+ tp.next();
tp.nextPosition();
// we don't read this payload and skip to a different document
@@ -559,13 +562,13 @@ public class TestPayloads extends Lucene
}
}
- private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result();
+ private BytesRef utf8Result = new BytesRef(10);
synchronized String bytesToString(byte[] bytes) {
String s = new String(bytes);
UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
try {
- return new String(utf8Result.result, 0, utf8Result.length, "UTF-8");
+ return new String(utf8Result.bytes, 0, utf8Result.length, "UTF-8");
} catch (UnsupportedEncodingException uee) {
return null;
}
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java Tue Apr 6 19:19:27 2010
@@ -18,9 +18,11 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.codecs.CodecProvider;
import java.io.IOException;
import java.util.Collection;
@@ -63,14 +65,16 @@ public class TestSegmentMerger extends L
}
public void testMerge() throws IOException {
- SegmentMerger merger = new SegmentMerger(mergedDir, mergedSegment);
+ SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriter.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, CodecProvider.getDefault());
merger.add(reader1);
merger.add(reader2);
int docsMerged = merger.merge();
merger.closeReaders();
assertTrue(docsMerged == 2);
//Should be able to open a new SegmentReader against the new directory
- SegmentReader mergedReader = SegmentReader.get(true, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
+ SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true,
+ -1, null, false, merger.hasProx(), merger.getCodec()), BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, null);
+
assertTrue(mergedReader != null);
assertTrue(mergedReader.numDocs() == 2);
Document newDoc1 = mergedReader.document(0);
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentReader.java Tue Apr 6 19:19:27 2010
@@ -137,6 +137,7 @@ public class TestSegmentReader extends L
TermPositions positions = reader.termPositions();
positions.seek(new Term(DocHelper.TEXT_FIELD_1_KEY, "field"));
assertTrue(positions != null);
+ assertTrue(positions.next());
assertTrue(positions.doc() == 0);
assertTrue(positions.nextPosition() >= 0);
}
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermDocs.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermDocs.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermDocs.java Tue Apr 6 19:19:27 2010
@@ -56,14 +56,13 @@ public class TestSegmentTermDocs extends
SegmentReader reader = SegmentReader.get(true, info, indexDivisor);
assertTrue(reader != null);
assertEquals(indexDivisor, reader.getTermInfosIndexDivisor());
- SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
- assertTrue(segTermDocs != null);
- segTermDocs.seek(new Term(DocHelper.TEXT_FIELD_2_KEY, "field"));
- if (segTermDocs.next() == true)
- {
- int docId = segTermDocs.doc();
+ TermDocs termDocs = reader.termDocs();
+ assertTrue(termDocs != null);
+ termDocs.seek(new Term(DocHelper.TEXT_FIELD_2_KEY, "field"));
+ if (termDocs.next() == true) {
+ int docId = termDocs.doc();
assertTrue(docId == 0);
- int freq = segTermDocs.freq();
+ int freq = termDocs.freq();
assertTrue(freq == 3);
}
reader.close();
@@ -78,20 +77,20 @@ public class TestSegmentTermDocs extends
//After adding the document, we should be able to read it back in
SegmentReader reader = SegmentReader.get(true, info, indexDivisor);
assertTrue(reader != null);
- SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
- assertTrue(segTermDocs != null);
- segTermDocs.seek(new Term("textField2", "bad"));
- assertTrue(segTermDocs.next() == false);
+ TermDocs termDocs = reader.termDocs();
+ assertTrue(termDocs != null);
+ termDocs.seek(new Term("textField2", "bad"));
+ assertTrue(termDocs.next() == false);
reader.close();
}
{
//After adding the document, we should be able to read it back in
SegmentReader reader = SegmentReader.get(true, info, indexDivisor);
assertTrue(reader != null);
- SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
- assertTrue(segTermDocs != null);
- segTermDocs.seek(new Term("junk", "bad"));
- assertTrue(segTermDocs.next() == false);
+ TermDocs termDocs = reader.termDocs();
+ assertTrue(termDocs != null);
+ termDocs.seek(new Term("junk", "bad"));
+ assertTrue(termDocs.next() == false);
reader.close();
}
}
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermEnum.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermEnum.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentTermEnum.java Tue Apr 6 19:19:27 2010
@@ -61,23 +61,6 @@ public class TestSegmentTermEnum extends
verifyDocFreq();
}
- public void testPrevTermAtEnd() throws IOException
- {
- Directory dir = new MockRAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
- addDoc(writer, "aaa bbb");
- writer.close();
- SegmentReader reader = SegmentReader.getOnlySegmentReader(dir);
- SegmentTermEnum termEnum = (SegmentTermEnum) reader.terms();
- assertTrue(termEnum.next());
- assertEquals("aaa", termEnum.term().text());
- assertTrue(termEnum.next());
- assertEquals("aaa", termEnum.prev().text());
- assertEquals("bbb", termEnum.term().text());
- assertFalse(termEnum.next());
- assertEquals("bbb", termEnum.prev().text());
- }
-
private void verifyDocFreq()
throws IOException
{
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/index/TestStressIndexing2.java Tue Apr 6 19:19:27 2010
@@ -352,7 +352,7 @@ public class TestStressIndexing2 extends
if (!termEnum1.next()) break;
}
- // iterate until we get some docs
+ // iterate until we get some docs
int len2;
for(;;) {
len2=0;
@@ -369,12 +369,12 @@ public class TestStressIndexing2 extends
if (!termEnum2.next()) break;
}
- if (!hasDeletes)
- assertEquals(termEnum1.docFreq(), termEnum2.docFreq());
-
assertEquals(len1, len2);
if (len1==0) break; // no more terms
+ if (!hasDeletes)
+ assertEquals(termEnum1.docFreq(), termEnum2.docFreq());
+
assertEquals(term1, term2);
// sort info2 to get it into ascending docid
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/CheckHits.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/CheckHits.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/CheckHits.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/CheckHits.java Tue Apr 6 19:19:27 2010
@@ -33,7 +33,7 @@ public class CheckHits {
* different order of operations from the actual scoring method ...
* this allows for a small amount of variation
*/
- public static float EXPLAIN_SCORE_TOLERANCE_DELTA = 0.00005f;
+ public static float EXPLAIN_SCORE_TOLERANCE_DELTA = 0.0002f;
/**
* Tests that all documents up to maxDoc which are *not* in the
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Tue Apr 6 19:19:27 2010
@@ -65,7 +65,7 @@ public class TestCachingWrapperFilter ex
if (originalSet.isCacheable()) {
assertEquals("Cached DocIdSet must be of same class like uncached, if cacheable", originalSet.getClass(), cachedSet.getClass());
} else {
- assertTrue("Cached DocIdSet must be an OpenBitSet if the original one was not cacheable", cachedSet instanceof OpenBitSetDISI);
+ assertTrue("Cached DocIdSet must be an OpenBitSet if the original one was not cacheable", cachedSet instanceof OpenBitSetDISI || cachedSet == DocIdSet.EMPTY_DOCIDSET);
}
}
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java Tue Apr 6 19:19:27 2010
@@ -230,6 +230,8 @@ public class TestNumericRangeQuery32 ext
testRightOpenRange(2);
}
+ /* TESTs disabled, because incompatible API change in 3.1/flex:
+
private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
final Random rnd=newRandom();
String field="field"+precisionStep;
@@ -298,6 +300,8 @@ public class TestNumericRangeQuery32 ext
testRandomTrieAndClassicRangeQuery(Integer.MAX_VALUE);
}
+ */
+
private void testRangeSplit(int precisionStep) throws Exception {
final Random rnd=newRandom();
String field="ascfield"+precisionStep;
@@ -443,37 +447,39 @@ public class TestNumericRangeQuery32 ext
assertFalse(q2.equals(q1));
}
- private void testEnum(int lower, int upper) throws Exception {
- NumericRangeQuery<Integer> q = NumericRangeQuery.newIntRange("field4", 4, lower, upper, true, true);
- FilteredTermEnum termEnum = q.getEnum(searcher.getIndexReader());
- try {
- int count = 0;
- do {
- final Term t = termEnum.term();
- if (t != null) {
- final int val = NumericUtils.prefixCodedToInt(t.text());
- assertTrue("value not in bounds", val >= lower && val <= upper);
- count++;
- } else break;
- } while (termEnum.next());
- assertFalse(termEnum.next());
- System.out.println("TermEnum on 'field4' for range [" + lower + "," + upper + "] contained " + count + " terms.");
- } finally {
- termEnum.close();
- }
- }
+// Removed for now - NumericRangeQuery does not currently implement getEnum
- public void testEnum() throws Exception {
- int count=3000;
- int lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3);
- // test enum with values
- testEnum(lower, upper);
- // test empty enum
- testEnum(upper, lower);
- // test empty enum outside of bounds
- lower = distance*noDocs+startOffset;
- upper = 2 * lower;
- testEnum(lower, upper);
- }
+// private void testEnum(int lower, int upper) throws Exception {
+// NumericRangeQuery<Integer> q = NumericRangeQuery.newIntRange("field4", 4, lower, upper, true, true);
+// FilteredTermEnum termEnum = q.getEnum(searcher.getIndexReader());
+// try {
+// int count = 0;
+// do {
+// final Term t = termEnum.term();
+// if (t != null) {
+// final int val = NumericUtils.prefixCodedToInt(t.text());
+// assertTrue("value not in bounds", val >= lower && val <= upper);
+// count++;
+// } else break;
+// } while (termEnum.next());
+// assertFalse(termEnum.next());
+// System.out.println("TermEnum on 'field4' for range [" + lower + "," + upper + "] contained " + count + " terms.");
+// } finally {
+// termEnum.close();
+// }
+// }
+//
+// public void testEnum() throws Exception {
+// int count=3000;
+// int lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3);
+// // test enum with values
+// testEnum(lower, upper);
+// // test empty enum
+// testEnum(upper, lower);
+// // test empty enum outside of bounds
+// lower = distance*noDocs+startOffset;
+// upper = 2 * lower;
+// testEnum(lower, upper);
+// }
}
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java Tue Apr 6 19:19:27 2010
@@ -245,6 +245,8 @@ public class TestNumericRangeQuery64 ext
testRightOpenRange(2);
}
+ /* TESTs disabled, because incompatible API change in 3.1/flex:
+
private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
final Random rnd=newRandom();
String field="field"+precisionStep;
@@ -317,6 +319,8 @@ public class TestNumericRangeQuery64 ext
testRandomTrieAndClassicRangeQuery(Integer.MAX_VALUE);
}
+ */
+
private void testRangeSplit(int precisionStep) throws Exception {
final Random rnd=newRandom();
String field="ascfield"+precisionStep;
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestSort.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestSort.java Tue Apr 6 19:19:27 2010
@@ -35,6 +35,7 @@ import org.apache.lucene.index.CorruptIn
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.LockObtainFailedException;
@@ -332,20 +333,28 @@ public class TestSort extends LuceneTest
FieldCache fc = FieldCache.DEFAULT;
- sort.setSort (new SortField ("parser", new FieldCache.IntParser(){
- public final int parseInt(final String val) {
- return (val.charAt(0)-'A') * 123456;
+ sort.setSort ( new SortField ("parser", new FieldCache.IntParser(){
+ public final int parseInt(final String term) {
+ // dummy
+ return 0;
}
- }), SortField.FIELD_DOC );
+ public final int parseInt(final BytesRef term) {
+ return (term.bytes[term.offset]-'A') * 123456;
+ }
+ }), SortField.FIELD_DOC);
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " IntParser");
fc.purgeAllCaches();
- sort.setSort (new SortField ("parser", new FieldCache.FloatParser(){
- public final float parseFloat(final String val) {
- return (float) Math.sqrt( val.charAt(0) );
+ sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.FloatParser(){
+ public final float parseFloat(final String term) {
+ // dummy
+ return 0;
+ }
+ public final float parseFloat(final BytesRef term) {
+ return (float) Math.sqrt( term.bytes[term.offset] );
}
- }), SortField.FIELD_DOC );
+ }), SortField.FIELD_DOC });
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " FloatParser");
fc.purgeAllCaches();
@@ -354,34 +363,49 @@ public class TestSort extends LuceneTest
public final long parseLong(final String val) {
return (val.charAt(0)-'A') * 1234567890L;
}
- }), SortField.FIELD_DOC );
+ public final long parseLong(final BytesRef term) {
+ return (term.bytes[term.offset]-'A') * 1234567890L;
+ }
+ }), SortField.FIELD_DOC);
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " LongParser");
fc.purgeAllCaches();
- sort.setSort (new SortField ("parser", new FieldCache.DoubleParser(){
- public final double parseDouble(final String val) {
- return Math.pow( val.charAt(0), (val.charAt(0)-'A') );
+ sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.DoubleParser(){
+ public final double parseDouble(final String term) {
+ // dummy
+ return 0;
+ }
+ public final double parseDouble(final BytesRef term) {
+ return Math.pow( term.bytes[term.offset], (term.bytes[term.offset]-'A') );
}
- }), SortField.FIELD_DOC );
+ }), SortField.FIELD_DOC });
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " DoubleParser");
fc.purgeAllCaches();
- sort.setSort (new SortField ("parser", new FieldCache.ByteParser(){
- public final byte parseByte(final String val) {
- return (byte) (val.charAt(0)-'A');
+ sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ByteParser(){
+ public final byte parseByte(final String term) {
+ // dummy
+ return 0;
}
- }), SortField.FIELD_DOC );
+ public final byte parseByte(final BytesRef term) {
+ return (byte) (term.bytes[term.offset]-'A');
+ }
+ }), SortField.FIELD_DOC });
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " ByteParser");
fc.purgeAllCaches();
- sort.setSort (new SortField ("parser", new FieldCache.ShortParser(){
- public final short parseShort(final String val) {
- return (short) (val.charAt(0)-'A');
+ sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ShortParser(){
+ public final short parseShort(final String term) {
+ // dummy
+ return 0;
+ }
+ public final short parseShort(final BytesRef term) {
+ return (short) (term.bytes[term.offset]-'A');
}
- }), SortField.FIELD_DOC );
+ }), SortField.FIELD_DOC });
assertMatches (full, queryA, sort, "JIHGFEDCBA");
assertSaneFieldCaches(getName() + " ShortParser");
fc.purgeAllCaches();
@@ -439,8 +463,12 @@ public class TestSort extends LuceneTest
@Override
public void setNextReader(IndexReader reader, int docBase) throws IOException {
docValues = FieldCache.DEFAULT.getInts(reader, "parser", new FieldCache.IntParser() {
- public final int parseInt(final String val) {
- return (val.charAt(0)-'A') * 123456;
+ public final int parseInt(final String term) {
+ // dummy
+ return 0;
+ }
+ public final int parseInt(final BytesRef term) {
+ return (term.bytes[term.offset]-'A') * 123456;
}
});
}
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestTermScorer.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestTermScorer.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestTermScorer.java Tue Apr 6 19:19:27 2010
@@ -72,9 +72,9 @@ public class TestTermScorer extends Luce
Weight weight = termQuery.weight(indexSearcher);
- TermScorer ts = new TermScorer(weight,
- indexReader.termDocs(allTerm), indexSearcher.getSimilarity(),
- indexReader.norms(FIELD));
+ Scorer ts = weight.scorer(indexSearcher.getIndexReader(),
+ true, true);
+
//we have 2 documents with the term all in them, one document for all the other values
final List docs = new ArrayList();
//must call next first
@@ -138,9 +138,9 @@ public class TestTermScorer extends Luce
Weight weight = termQuery.weight(indexSearcher);
- TermScorer ts = new TermScorer(weight,
- indexReader.termDocs(allTerm), indexSearcher.getSimilarity(),
- indexReader.norms(FIELD));
+ Scorer ts = weight.scorer(indexSearcher.getIndexReader(),
+ true, true);
+
assertTrue("next did not return a doc", ts.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertTrue("score is not correct", ts.score() == 1.6931472f);
assertTrue("next did not return a doc", ts.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
@@ -155,9 +155,9 @@ public class TestTermScorer extends Luce
Weight weight = termQuery.weight(indexSearcher);
- TermScorer ts = new TermScorer(weight,
- indexReader.termDocs(allTerm), indexSearcher.getSimilarity(),
- indexReader.norms(FIELD));
+ Scorer ts = weight.scorer(indexSearcher.getIndexReader(),
+ true, true);
+
assertTrue("Didn't skip", ts.advance(3) != DocIdSetIterator.NO_MORE_DOCS);
//The next doc should be doc 5
assertTrue("doc should be number 5", ts.docID() == 5);
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestWildcard.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestWildcard.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestWildcard.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/search/TestWildcard.java Tue Apr 6 19:19:27 2010
@@ -114,6 +114,7 @@ public class TestWildcard
* rewritten to a single PrefixQuery. The boost and rewriteMethod should be
* preserved.
*/
+ /* disable because rewrites changed in flex/trunk
public void testPrefixTerm() throws IOException {
RAMDirectory indexStore = getIndexStore("field", new String[]{"prefix", "prefixx"});
IndexSearcher searcher = new IndexSearcher(indexStore, true);
@@ -145,7 +146,7 @@ public class TestWildcard
expected.setRewriteMethod(wq.getRewriteMethod());
expected.setBoost(wq.getBoost());
assertEquals(searcher.rewrite(expected), searcher.rewrite(wq));
- }
+ }*/
/**
* Tests Wildcard queries with an asterisk.
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java Tue Apr 6 19:19:27 2010
@@ -78,22 +78,22 @@ public class TestAttributeSource extends
public void testCloneAttributes() {
final AttributeSource src = new AttributeSource();
- final TermAttribute termAtt = src.addAttribute(TermAttribute.class);
+ final FlagsAttribute flagsAtt = src.addAttribute(FlagsAttribute.class);
final TypeAttribute typeAtt = src.addAttribute(TypeAttribute.class);
- termAtt.setTermBuffer("TestTerm");
+ flagsAtt.setFlags(1234);
typeAtt.setType("TestType");
final AttributeSource clone = src.cloneAttributes();
final Iterator<Class<? extends Attribute>> it = clone.getAttributeClassesIterator();
- assertEquals("TermAttribute must be the first attribute", TermAttribute.class, it.next());
+ assertEquals("FlagsAttribute must be the first attribute", FlagsAttribute.class, it.next());
assertEquals("TypeAttribute must be the second attribute", TypeAttribute.class, it.next());
assertFalse("No more attributes", it.hasNext());
- final TermAttribute termAtt2 = clone.getAttribute(TermAttribute.class);
+ final FlagsAttribute flagsAtt2 = clone.getAttribute(FlagsAttribute.class);
final TypeAttribute typeAtt2 = clone.getAttribute(TypeAttribute.class);
- assertNotSame("TermAttribute of original and clone must be different instances", termAtt2, termAtt);
+ assertNotSame("FlagsAttribute of original and clone must be different instances", flagsAtt2, flagsAtt);
assertNotSame("TypeAttribute of original and clone must be different instances", typeAtt2, typeAtt);
- assertEquals("TermAttribute of original and clone must be equal", termAtt2, termAtt);
+ assertEquals("FlagsAttribute of original and clone must be equal", flagsAtt2, flagsAtt);
assertEquals("TypeAttribute of original and clone must be equal", typeAtt2, typeAtt);
}
Propchange: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 6 19:19:27 2010
@@ -1,6 +1,8 @@
+/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/util/TestAttributeSource.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036
/lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336
/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java:924483-925561
+/lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:880869-912395
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java:924483-924731,924781,925176-925462
/lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,883079,885214,912319,912324,912328,912335,912372
Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java Tue Apr 6 19:19:27 2010
@@ -26,6 +26,8 @@ import java.util.Iterator;
public class TestNumericUtils extends LuceneTestCase {
+ /* TESTs disabled, because incompatible API change in 3.1/flex:
+
public void testLongConversionAndOrdering() throws Exception {
// generate a series of encoded longs, each numerical one bigger than the one before
String last=null;
@@ -131,6 +133,8 @@ public class TestNumericUtils extends Lu
}
}
}
+
+ */
public void testDoubles() throws Exception {
double[] vals=new double[]{
Modified: lucene/dev/trunk/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/build.xml?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/build.xml (original)
+++ lucene/dev/trunk/lucene/build.xml Tue Apr 6 19:19:27 2010
@@ -104,24 +104,24 @@ The source distribution does not contain
<target name="compile-backwards" depends="compile-core, jar-core, test-backwards-message"
description="Runs tests of a previous Lucene version." if="backwards.available">
- <sequential>
+ <sequential>
<mkdir dir="${build.dir.backwards}"/>
- <!-- first compile branch classes -->
- <compile
+ <!-- first compile branch classes -->
+ <compile
srcdir="${backwards.dir}/src/java"
destdir="${build.dir.backwards}/classes/java"
javac.source="${javac.source.backwards}" javac.target="${javac.target.backwards}"
>
<classpath refid="backwards.compile.classpath"/>
- </compile>
-
+ </compile>
+
<!-- compile branch tests against branch classpath -->
<compile-test-macro srcdir="${backwards.dir}/src/test" destdir="${build.dir.backwards}/classes/test"
test.classpath="backwards.test.compile.classpath" javac.source="${javac.source.backwards}" javac.target="${javac.target.backwards}"/>
- </sequential>
+ </sequential>
</target>
<target name="test-backwards" depends="compile-backwards, junit-backwards-mkdir, junit-backwards-sequential, junit-backwards-parallel"/>
@@ -715,6 +715,41 @@ The source distribution does not contain
</delete>
</target>
+ <macrodef name="createLevAutomaton">
+ <attribute name="n"/>
+ <sequential>
+ <exec dir="src/java/org/apache/lucene/util/automaton"
+ executable="${python.exe}" failonerror="true">
+ <arg line="createLevAutomata.py @{n}"/>
+ </exec>
+ </sequential>
+ </macrodef>
+
+ <target name="createLevAutomata" depends="check-moman,clone-moman,pull-moman">
+ <createLevAutomaton n="1"/>
+ <createLevAutomaton n="2"/>
+ </target>
+
+ <target name="check-moman">
+ <condition property="moman.cloned">
+ <available file="src/java/org/apache/lucene/util/automaton/moman"/>
+ </condition>
+ </target>
+
+ <target name="clone-moman" unless="moman.cloned">
+ <exec dir="src/java/org/apache/lucene/util/automaton"
+ executable="${hg.exe}" failonerror="true">
+ <arg line="clone -r ${moman.rev} ${moman.url} moman"/>
+ </exec>
+ </target>
+
+ <target name="pull-moman" if="moman.cloned">
+ <exec dir="src/java/org/apache/lucene/util/automaton/moman"
+ executable="${hg.exe}" failonerror="true">
+ <arg line="pull -f -r ${moman.rev}"/>
+ </exec>
+ </target>
+
<macrodef name="contrib-crawl">
<attribute name="target" default=""/>
<attribute name="failonerror" default="true"/>
Propchange: lucene/dev/trunk/lucene/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 6 19:19:27 2010
@@ -1,2 +1,3 @@
-/lucene/java/branches/lucene_2_9/build.xml:896850,909334
-/lucene/java/trunk/build.xml:924483-925561
+/lucene/java/branches/flex_1458/build.xml:824912-931101
+/lucene/java/branches/lucene_2_9/build.xml:909334
+/lucene/java/trunk/build.xml:924483-924731,924781,925176-925462
Modified: lucene/dev/trunk/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/common-build.xml?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/common-build.xml (original)
+++ lucene/dev/trunk/lucene/common-build.xml Tue Apr 6 19:19:27 2010
@@ -119,6 +119,11 @@
<property name="svnversion.exe" value="svnversion" />
<property name="svn.exe" value="svn" />
+ <property name="hg.exe" value="hg" />
+ <property name="moman.url" value="https://bitbucket.org/jpbarrette/moman" />
+ <property name="moman.rev" value="115" />
+ <property name="python.exe" value="python" />
+
<property name="gpg.exe" value="gpg" />
<property name="gpg.key" value="CODE SIGNING KEY" />
Propchange: lucene/dev/trunk/lucene/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 6 19:19:27 2010
@@ -1,5 +1,6 @@
+/lucene/java/branches/flex_1458/contrib:824912-931101
/lucene/java/branches/lucene_2_4/contrib:748824
/lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336
/lucene/java/branches/lucene_3_0/contrib:880793,896906
-/lucene/java/trunk/contrib:924483-925561
+/lucene/java/trunk/contrib:924483-924731,924781,925176-925462
Propchange: lucene/dev/trunk/lucene/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 6 19:19:27 2010
@@ -1,5 +1,6 @@
+/lucene/java/branches/flex_1458/contrib/CHANGES.txt:824912-931101
/lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
-/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336
/lucene/java/branches/lucene_3_0/contrib/CHANGES.txt:880793,896906
-/lucene/java/trunk/contrib/CHANGES.txt:924483-925561
+/lucene/java/trunk/contrib/CHANGES.txt:924483-924731,924781,925176-925462
Modified: lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Tue Apr 6 19:19:27 2010
@@ -37,11 +37,12 @@ import org.apache.lucene.benchmark.byTas
import org.apache.lucene.collation.CollationKeyAnalyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LogMergePolicy;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.TermFreqVector;
@@ -474,16 +475,20 @@ public class TestPerfTasksLogic extends
IndexReader reader = IndexReader.open(benchmark.getRunData().getDirectory(), true);
assertEquals(NUM_DOCS, reader.numDocs());
- TermEnum terms = reader.terms();
- TermDocs termDocs = reader.termDocs();
int totalTokenCount2 = 0;
- while(terms.next()) {
- Term term = terms.term();
- /* not-tokenized, but indexed field */
- if (term != null && term.field() != DocMaker.ID_FIELD) {
- termDocs.seek(terms.term());
- while (termDocs.next())
- totalTokenCount2 += termDocs.freq();
+
+ FieldsEnum fields = MultiFields.getFields(reader).iterator();
+ String fieldName = null;
+ while((fieldName = fields.next()) != null) {
+ if (fieldName == DocMaker.ID_FIELD)
+ continue;
+ TermsEnum terms = fields.terms();
+ DocsEnum docs = null;
+ while(terms.next() != null) {
+ docs = terms.docs(MultiFields.getDeletedDocs(reader), docs);
+ while(docs.nextDoc() != docs.NO_MORE_DOCS) {
+ totalTokenCount2 += docs.freq();
+ }
}
}
reader.close();
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Tue Apr 6 19:19:27 2010
@@ -150,11 +150,16 @@ public class WeightedSpanTermExtractor {
mtq.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
query = mtq;
}
- FakeReader fReader = new FakeReader();
- MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE.rewrite(fReader, mtq);
- if (fReader.field != null) {
- IndexReader ir = getReaderForField(fReader.field);
+ if (mtq.getField() != null) {
+ IndexReader ir = getReaderForField(mtq.getField());
extract(query.rewrite(ir), terms);
+ } else {
+ FakeReader fReader = new FakeReader();
+ MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE.rewrite(fReader, mtq);
+ if (fReader.field != null) {
+ IndexReader ir = getReaderForField(fReader.field);
+ extract(query.rewrite(ir), terms);
+ }
}
} else if (query instanceof MultiPhraseQuery) {
final MultiPhraseQuery mpq = (MultiPhraseQuery) query;
Propchange: lucene/dev/trunk/lucene/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 6 19:19:27 2010
@@ -1,5 +1,6 @@
+/lucene/java/branches/flex_1458/contrib/highlighter/src/test:824912-931101
/lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336
/lucene/java/branches/lucene_3_0/contrib/highlighter/src/test:880793,896906
-/lucene/java/trunk/contrib/highlighter/src/test:924483-925561
+/lucene/java/trunk/contrib/highlighter/src/test:924483-924731,924781,925176-925462
Propchange: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 6 19:19:27 2010
@@ -1,2 +1,3 @@
-/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:896850,909334
-/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:924483-925561
+/lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:824912-931101
+/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:909334
+/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:924483-924731,924781,925176-925462
Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java Tue Apr 6 19:19:27 2010
@@ -19,11 +19,15 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.File;
import java.util.Date;
+import java.util.List;
+import java.util.ArrayList;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ReaderUtil;
/**
* Given a directory and a list of fields, updates the fieldNorms in place for every document.
@@ -104,46 +108,46 @@ public class FieldNormModifier {
*/
public void reSetNorms(String field) throws IOException {
String fieldName = StringHelper.intern(field);
- int[] termCounts = new int[0];
IndexReader reader = null;
- TermEnum termEnum = null;
- TermDocs termDocs = null;
try {
- reader = IndexReader.open(dir, true);
- termCounts = new int[reader.maxDoc()];
- try {
- termEnum = reader.terms(new Term(field));
- try {
- termDocs = reader.termDocs();
- do {
- Term term = termEnum.term();
- if (term != null && term.field().equals(fieldName)) {
- termDocs.seek(termEnum.term());
- while (termDocs.next()) {
- termCounts[termDocs.doc()] += termDocs.freq();
+ reader = IndexReader.open(dir, false);
+
+ final List<IndexReader> subReaders = new ArrayList<IndexReader>();
+ ReaderUtil.gatherSubReaders(subReaders, reader);
+
+ for(IndexReader subReader : subReaders) {
+ final Bits delDocs = subReader.getDeletedDocs();
+
+ int[] termCounts = new int[subReader.maxDoc()];
+ Fields fields = subReader.fields();
+ if (fields != null) {
+ Terms terms = fields.terms(field);
+ if (terms != null) {
+ TermsEnum termsEnum = terms.iterator();
+ DocsEnum docs = null;
+ while(termsEnum.next() != null) {
+ docs = termsEnum.docs(delDocs, docs);
+ while(true) {
+ int docID = docs.nextDoc();
+ if (docID != docs.NO_MORE_DOCS) {
+ termCounts[docID] += docs.freq();
+ } else {
+ break;
+ }
}
}
- } while (termEnum.next());
-
- } finally {
- if (null != termDocs) termDocs.close();
+ }
}
- } finally {
- if (null != termEnum) termEnum.close();
- }
- } finally {
- if (null != reader) reader.close();
- }
-
- try {
- reader = IndexReader.open(dir, false);
- for (int d = 0; d < termCounts.length; d++) {
- if (! reader.isDeleted(d)) {
- if (sim == null)
- reader.setNorm(d, fieldName, Similarity.encodeNorm(1.0f));
- else
- reader.setNorm(d, fieldName, sim.encodeNormValue(sim.lengthNorm(fieldName, termCounts[d])));
+
+ for (int d = 0; d < termCounts.length; d++) {
+ if (delDocs == null || !delDocs.get(d)) {
+ if (sim == null) {
+ subReader.setNorm(d, fieldName, Similarity.encodeNorm(1.0f));
+ } else {
+ subReader.setNorm(d, fieldName, sim.encodeNormValue(sim.lengthNorm(fieldName, termCounts[d])));
+ }
+ }
}
}
@@ -151,5 +155,4 @@ public class FieldNormModifier {
if (null != reader) reader.close();
}
}
-
}
Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java Tue Apr 6 19:19:27 2010
@@ -26,6 +26,7 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.Version;
/**
@@ -172,6 +173,8 @@ public class MultiPassIndexSplitter {
* list of deletions.
*/
public static class FakeDeleteIndexReader extends FilterIndexReader {
+ // TODO: switch to flex api, here
+
OpenBitSet dels;
OpenBitSet oldDels = null;
@@ -202,6 +205,7 @@ public class MultiPassIndexSplitter {
if (oldDels != null) {
dels.or(oldDels);
}
+ storeDelDocs(null);
}
@Override
@@ -215,6 +219,16 @@ public class MultiPassIndexSplitter {
}
@Override
+ public IndexReader[] getSequentialSubReaders() {
+ return null;
+ }
+
+ @Override
+ public Bits getDeletedDocs() {
+ return dels;
+ }
+
+ @Override
public boolean isDeleted(int n) {
return dels.get(n);
}
@@ -235,5 +249,29 @@ public class MultiPassIndexSplitter {
}
};
}
+
+ @Override
+ public TermDocs termDocs() throws IOException {
+ return new FilterTermDocs(in.termDocs()) {
+
+ @Override
+ public boolean next() throws IOException {
+ boolean res;
+ while ((res = super.next())) {
+ if (!dels.get(doc())) {
+ break;
+ }
+ }
+ return res;
+ }
+ };
+ }
+
+ @Override
+ public TermDocs termDocs(Term term) throws IOException {
+ TermDocs termDocs = termDocs();
+ termDocs.seek(term);
+ return termDocs;
+ }
}
}
Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java Tue Apr 6 19:19:27 2010
@@ -1,10 +1,5 @@
package org.apache.lucene.index;
-import org.apache.lucene.util.StringHelper;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,6 +15,14 @@ import java.util.List;
*
*/
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* Transparent access to the vector space model,
@@ -97,40 +100,53 @@ public class TermVectorAccessor {
positions.clear();
}
- TermEnum termEnum = indexReader.terms(new Term(field, ""));
- if (termEnum.term() != null) {
- while (termEnum.term().field() == field) {
- TermPositions termPositions = indexReader.termPositions(termEnum.term());
- if (termPositions.skipTo(documentNumber)) {
-
- frequencies.add(Integer.valueOf(termPositions.freq()));
- tokens.add(termEnum.term().text());
-
-
+ final Bits delDocs = MultiFields.getDeletedDocs(indexReader);
+
+ Terms terms = MultiFields.getTerms(indexReader, field);
+ boolean anyTerms = false;
+ if (terms != null) {
+ TermsEnum termsEnum = terms.iterator();
+ DocsEnum docs = null;
+ DocsAndPositionsEnum postings = null;
+ while(true) {
+ BytesRef text = termsEnum.next();
+ if (text != null) {
+ anyTerms = true;
if (!mapper.isIgnoringPositions()) {
- int[] positions = new int[termPositions.freq()];
- for (int i = 0; i < positions.length; i++) {
- positions[i] = termPositions.nextPosition();
- }
- this.positions.add(positions);
+ docs = postings = termsEnum.docsAndPositions(delDocs, postings);
} else {
- positions.add(null);
+ docs = termsEnum.docs(delDocs, docs);
}
- }
- termPositions.close();
- if (!termEnum.next()) {
+
+ int docID = docs.advance(documentNumber);
+ if (docID == documentNumber) {
+
+ frequencies.add(Integer.valueOf(docs.freq()));
+ tokens.add(text.utf8ToString());
+
+ if (!mapper.isIgnoringPositions()) {
+ int[] positions = new int[docs.freq()];
+ for (int i = 0; i < positions.length; i++) {
+ positions[i] = postings.nextPosition();
+ }
+ this.positions.add(positions);
+ } else {
+ positions.add(null);
+ }
+ }
+ } else {
break;
}
}
- mapper.setDocumentNumber(documentNumber);
- mapper.setExpectations(field, tokens.size(), false, !mapper.isIgnoringPositions());
- for (int i = 0; i < tokens.size(); i++) {
- mapper.map(tokens.get(i), frequencies.get(i).intValue(), (TermVectorOffsetInfo[]) null, positions.get(i));
+
+ if (anyTerms) {
+ mapper.setDocumentNumber(documentNumber);
+ mapper.setExpectations(field, tokens.size(), false, !mapper.isIgnoringPositions());
+ for (int i = 0; i < tokens.size(); i++) {
+ mapper.map(tokens.get(i), frequencies.get(i).intValue(), (TermVectorOffsetInfo[]) null, positions.get(i));
+ }
}
}
- termEnum.close();
-
-
}
Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java Tue Apr 6 19:19:27 2010
@@ -18,7 +18,10 @@ package org.apache.lucene.misc;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.Terms;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.PriorityQueue;
@@ -50,20 +53,40 @@ public class HighFreqTerms {
}
TermInfoQueue tiq = new TermInfoQueue(numTerms);
- TermEnum terms = reader.terms();
if (field != null) {
- while (terms.next()) {
- if (terms.term().field().equals(field)) {
- tiq.insertWithOverflow(new TermInfo(terms.term(), terms.docFreq()));
+ Terms terms = reader.fields().terms(field);
+ if (terms != null) {
+ TermsEnum termsEnum = terms.iterator();
+ while(true) {
+ BytesRef term = termsEnum.next();
+ if (term != null) {
+ tiq.insertWithOverflow(new TermInfo(new Term(field, term.utf8ToString()), termsEnum.docFreq()));
+ } else {
+ break;
+ }
}
}
- }
- else {
- while (terms.next()) {
- tiq.insertWithOverflow(new TermInfo(terms.term(), terms.docFreq()));
+ } else {
+ FieldsEnum fields = reader.fields().iterator();
+ while(true) {
+ field = fields.next();
+ if (field != null) {
+ TermsEnum terms = fields.terms();
+ while(true) {
+ BytesRef term = terms.next();
+ if (term != null) {
+ tiq.insertWithOverflow(new TermInfo(new Term(field, term.toString()), terms.docFreq()));
+ } else {
+ break;
+ }
+ }
+ } else {
+ break;
+ }
}
}
+
while (tiq.size() != 0) {
TermInfo termInfo = tiq.pop();
System.out.println(termInfo.term + " " + termInfo.docFreq);