You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by bu...@apache.org on 2010/07/22 21:34:52 UTC
svn commit: r966819 [9/20] - in /lucene/dev/branches/realtime_search: ./
lucene/ lucene/backwards/ lucene/contrib/ lucene/contrib/benchmark/conf/
lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
lucene/contrib/benchmark/src/j...
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/document/TestDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/document/TestDocument.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/document/TestDocument.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/document/TestDocument.java Thu Jul 22 19:34:35 2010
@@ -1,8 +1,9 @@
package org.apache.lucene.document;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -32,17 +33,15 @@ import org.apache.lucene.util.LuceneTest
/**
* Tests {@link Document} class.
*/
-public class TestDocument extends LuceneTestCase
-{
-
+public class TestDocument extends LuceneTestCase {
+
String binaryVal = "this text will be stored as a byte array in the index";
String binaryVal2 = "this text will be also stored as a byte array in the index";
- public void testBinaryField()
- throws Exception
- {
+ public void testBinaryField() throws Exception {
Document doc = new Document();
- Fieldable stringFld = new Field("string", binaryVal, Field.Store.YES, Field.Index.NO);
+ Fieldable stringFld = new Field("string", binaryVal, Field.Store.YES,
+ Field.Index.NO);
Fieldable binaryFld = new Field("binary", binaryVal.getBytes());
Fieldable binaryFld2 = new Field("binary", binaryVal2.getBytes());
@@ -67,7 +66,7 @@ public class TestDocument extends Lucene
assertEquals(3, doc.fields.size());
byte[][] binaryTests = doc.getBinaryValues("binary");
-
+
assertEquals(2, binaryTests.length);
binaryTest = new String(binaryTests[0]);
@@ -88,17 +87,17 @@ public class TestDocument extends Lucene
/**
* Tests {@link Document#removeField(String)} method for a brand new Document
* that has not been indexed yet.
- *
+ *
* @throws Exception on error
*/
- public void testRemoveForNewDocument() throws Exception
- {
+ public void testRemoveForNewDocument() throws Exception {
Document doc = makeDocumentWithFields();
assertEquals(8, doc.fields.size());
doc.removeFields("keyword");
assertEquals(6, doc.fields.size());
- doc.removeFields("doesnotexists"); // removing non-existing fields is siltenlty ignored
- doc.removeFields("keyword"); // removing a field more than once
+ doc.removeFields("doesnotexists"); // removing non-existing fields is
+ // siltenlty ignored
+ doc.removeFields("keyword"); // removing a field more than once
assertEquals(6, doc.fields.size());
doc.removeField("text");
assertEquals(5, doc.fields.size());
@@ -106,164 +105,171 @@ public class TestDocument extends Lucene
assertEquals(4, doc.fields.size());
doc.removeField("text");
assertEquals(4, doc.fields.size());
- doc.removeField("doesnotexists"); // removing non-existing fields is siltenlty ignored
+ doc.removeField("doesnotexists"); // removing non-existing fields is
+ // siltenlty ignored
assertEquals(4, doc.fields.size());
doc.removeFields("unindexed");
assertEquals(2, doc.fields.size());
doc.removeFields("unstored");
assertEquals(0, doc.fields.size());
- doc.removeFields("doesnotexists"); // removing non-existing fields is siltenlty ignored
+ doc.removeFields("doesnotexists"); // removing non-existing fields is
+ // siltenlty ignored
assertEquals(0, doc.fields.size());
}
-
- public void testConstructorExceptions()
- {
- new Field("name", "value", Field.Store.YES, Field.Index.NO); // okay
- new Field("name", "value", Field.Store.NO, Field.Index.NOT_ANALYZED); // okay
+
+ public void testConstructorExceptions() {
+ new Field("name", "value", Field.Store.YES, Field.Index.NO); // okay
+ new Field("name", "value", Field.Store.NO, Field.Index.NOT_ANALYZED); // okay
try {
new Field("name", "value", Field.Store.NO, Field.Index.NO);
fail();
- } catch(IllegalArgumentException e) {
+ } catch (IllegalArgumentException e) {
// expected exception
}
- new Field("name", "value", Field.Store.YES, Field.Index.NO, Field.TermVector.NO); // okay
+ new Field("name", "value", Field.Store.YES, Field.Index.NO,
+ Field.TermVector.NO); // okay
try {
- new Field("name", "value", Field.Store.YES, Field.Index.NO, Field.TermVector.YES);
+ new Field("name", "value", Field.Store.YES, Field.Index.NO,
+ Field.TermVector.YES);
fail();
- } catch(IllegalArgumentException e) {
+ } catch (IllegalArgumentException e) {
// expected exception
}
}
- /**
- * Tests {@link Document#getValues(String)} method for a brand new Document
- * that has not been indexed yet.
- *
- * @throws Exception on error
- */
- public void testGetValuesForNewDocument() throws Exception
- {
- doAssert(makeDocumentWithFields(), false);
- }
-
- /**
- * Tests {@link Document#getValues(String)} method for a Document retrieved from
- * an index.
- *
- * @throws Exception on error
- */
- public void testGetValuesForIndexedDocument() throws Exception {
- RAMDirectory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer()));
- writer.addDocument(makeDocumentWithFields());
- writer.close();
-
- Searcher searcher = new IndexSearcher(dir, true);
-
- // search for something that does exists
- Query query = new TermQuery(new Term("keyword", "test1"));
-
- // ensure that queries return expected results without DateFilter first
- ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
- assertEquals(1, hits.length);
-
- doAssert(searcher.doc(hits[0].doc), true);
- searcher.close();
- }
-
- private Document makeDocumentWithFields()
- {
- Document doc = new Document();
- doc.add(new Field( "keyword", "test1", Field.Store.YES, Field.Index.NOT_ANALYZED));
- doc.add(new Field( "keyword", "test2", Field.Store.YES, Field.Index.NOT_ANALYZED));
- doc.add(new Field( "text", "test1", Field.Store.YES, Field.Index.ANALYZED));
- doc.add(new Field( "text", "test2", Field.Store.YES, Field.Index.ANALYZED));
- doc.add(new Field("unindexed", "test1", Field.Store.YES, Field.Index.NO));
- doc.add(new Field("unindexed", "test2", Field.Store.YES, Field.Index.NO));
- doc.add(new Field( "unstored", "test1", Field.Store.NO, Field.Index.ANALYZED));
- doc.add(new Field( "unstored", "test2", Field.Store.NO, Field.Index.ANALYZED));
- return doc;
+ /**
+ * Tests {@link Document#getValues(String)} method for a brand new Document
+ * that has not been indexed yet.
+ *
+ * @throws Exception on error
+ */
+ public void testGetValuesForNewDocument() throws Exception {
+ doAssert(makeDocumentWithFields(), false);
+ }
+
+ /**
+ * Tests {@link Document#getValues(String)} method for a Document retrieved
+ * from an index.
+ *
+ * @throws Exception on error
+ */
+ public void testGetValuesForIndexedDocument() throws Exception {
+ RAMDirectory dir = new RAMDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(newRandom(), dir,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer.addDocument(makeDocumentWithFields());
+ IndexReader reader = writer.getReader();
+
+ Searcher searcher = new IndexSearcher(reader);
+
+ // search for something that does exists
+ Query query = new TermQuery(new Term("keyword", "test1"));
+
+ // ensure that queries return expected results without DateFilter first
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(1, hits.length);
+
+ doAssert(searcher.doc(hits[0].doc), true);
+ writer.close();
+ searcher.close();
+ reader.close();
+ dir.close();
+ }
+
+ private Document makeDocumentWithFields() {
+ Document doc = new Document();
+ doc.add(new Field("keyword", "test1", Field.Store.YES,
+ Field.Index.NOT_ANALYZED));
+ doc.add(new Field("keyword", "test2", Field.Store.YES,
+ Field.Index.NOT_ANALYZED));
+ doc.add(new Field("text", "test1", Field.Store.YES, Field.Index.ANALYZED));
+ doc.add(new Field("text", "test2", Field.Store.YES, Field.Index.ANALYZED));
+ doc.add(new Field("unindexed", "test1", Field.Store.YES, Field.Index.NO));
+ doc.add(new Field("unindexed", "test2", Field.Store.YES, Field.Index.NO));
+ doc
+ .add(new Field("unstored", "test1", Field.Store.NO,
+ Field.Index.ANALYZED));
+ doc
+ .add(new Field("unstored", "test2", Field.Store.NO,
+ Field.Index.ANALYZED));
+ return doc;
+ }
+
+ private void doAssert(Document doc, boolean fromIndex) {
+ String[] keywordFieldValues = doc.getValues("keyword");
+ String[] textFieldValues = doc.getValues("text");
+ String[] unindexedFieldValues = doc.getValues("unindexed");
+ String[] unstoredFieldValues = doc.getValues("unstored");
+
+ assertTrue(keywordFieldValues.length == 2);
+ assertTrue(textFieldValues.length == 2);
+ assertTrue(unindexedFieldValues.length == 2);
+ // this test cannot work for documents retrieved from the index
+ // since unstored fields will obviously not be returned
+ if (!fromIndex) {
+ assertTrue(unstoredFieldValues.length == 2);
}
-
- private void doAssert(Document doc, boolean fromIndex)
- {
- String[] keywordFieldValues = doc.getValues("keyword");
- String[] textFieldValues = doc.getValues("text");
- String[] unindexedFieldValues = doc.getValues("unindexed");
- String[] unstoredFieldValues = doc.getValues("unstored");
-
- assertTrue(keywordFieldValues.length == 2);
- assertTrue(textFieldValues.length == 2);
- assertTrue(unindexedFieldValues.length == 2);
- // this test cannot work for documents retrieved from the index
- // since unstored fields will obviously not be returned
- if (! fromIndex)
- {
- assertTrue(unstoredFieldValues.length == 2);
- }
-
- assertTrue(keywordFieldValues[0].equals("test1"));
- assertTrue(keywordFieldValues[1].equals("test2"));
- assertTrue(textFieldValues[0].equals("test1"));
- assertTrue(textFieldValues[1].equals("test2"));
- assertTrue(unindexedFieldValues[0].equals("test1"));
- assertTrue(unindexedFieldValues[1].equals("test2"));
- // this test cannot work for documents retrieved from the index
- // since unstored fields will obviously not be returned
- if (! fromIndex)
- {
- assertTrue(unstoredFieldValues[0].equals("test1"));
- assertTrue(unstoredFieldValues[1].equals("test2"));
- }
+
+ assertTrue(keywordFieldValues[0].equals("test1"));
+ assertTrue(keywordFieldValues[1].equals("test2"));
+ assertTrue(textFieldValues[0].equals("test1"));
+ assertTrue(textFieldValues[1].equals("test2"));
+ assertTrue(unindexedFieldValues[0].equals("test1"));
+ assertTrue(unindexedFieldValues[1].equals("test2"));
+ // this test cannot work for documents retrieved from the index
+ // since unstored fields will obviously not be returned
+ if (!fromIndex) {
+ assertTrue(unstoredFieldValues[0].equals("test1"));
+ assertTrue(unstoredFieldValues[1].equals("test2"));
}
-
- public void testFieldSetValue() throws Exception {
-
- Field field = new Field("id", "id1", Field.Store.YES, Field.Index.NOT_ANALYZED);
- Document doc = new Document();
- doc.add(field);
- doc.add(new Field("keyword", "test", Field.Store.YES, Field.Index.NOT_ANALYZED));
-
- RAMDirectory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer()));
- writer.addDocument(doc);
- field.setValue("id2");
- writer.addDocument(doc);
- field.setValue("id3");
- writer.addDocument(doc);
- writer.close();
-
- Searcher searcher = new IndexSearcher(dir, true);
-
- Query query = new TermQuery(new Term("keyword", "test"));
-
- // ensure that queries return expected results without DateFilter first
- ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
- assertEquals(3, hits.length);
- int result = 0;
- for(int i=0;i<3;i++) {
- Document doc2 = searcher.doc(hits[i].doc);
- Field f = doc2.getField("id");
- if (f.stringValue().equals("id1"))
- result |= 1;
- else if (f.stringValue().equals("id2"))
- result |= 2;
- else if (f.stringValue().equals("id3"))
- result |= 4;
- else
- fail("unexpected id field");
- }
- searcher.close();
- dir.close();
- assertEquals("did not see all IDs", 7, result);
+ }
+
+ public void testFieldSetValue() throws Exception {
+
+ Field field = new Field("id", "id1", Field.Store.YES,
+ Field.Index.NOT_ANALYZED);
+ Document doc = new Document();
+ doc.add(field);
+ doc.add(new Field("keyword", "test", Field.Store.YES,
+ Field.Index.NOT_ANALYZED));
+
+ RAMDirectory dir = new RAMDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(newRandom(), dir,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer.addDocument(doc);
+ field.setValue("id2");
+ writer.addDocument(doc);
+ field.setValue("id3");
+ writer.addDocument(doc);
+
+ IndexReader reader = writer.getReader();
+ Searcher searcher = new IndexSearcher(reader);
+
+ Query query = new TermQuery(new Term("keyword", "test"));
+
+ // ensure that queries return expected results without DateFilter first
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(3, hits.length);
+ int result = 0;
+ for (int i = 0; i < 3; i++) {
+ Document doc2 = searcher.doc(hits[i].doc);
+ Field f = doc2.getField("id");
+ if (f.stringValue().equals("id1")) result |= 1;
+ else if (f.stringValue().equals("id2")) result |= 2;
+ else if (f.stringValue().equals("id3")) result |= 4;
+ else fail("unexpected id field");
}
-
+ writer.close();
+ searcher.close();
+ reader.close();
+ dir.close();
+ assertEquals("did not see all IDs", 7, result);
+ }
+
public void testFieldSetValueChangeBinary() {
Field field1 = new Field("field1", new byte[0]);
- Field field2 = new Field("field2", "",
- Field.Store.YES, Field.Index.ANALYZED);
+ Field field2 = new Field("field2", "", Field.Store.YES,
+ Field.Index.ANALYZED);
try {
field1.setValue("abc");
fail("did not hit expected exception");
Propchange: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jul 22 19:34:35 2010
@@ -1,4 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:943137,949730
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:943137,949730,957490,960490,961612
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:953476-966816
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:829134,829881,831036,896850,909334,948516
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Thu Jul 22 19:34:35 2010
@@ -464,7 +464,7 @@ public class TestAddIndexes extends Luce
private void verifyTermDocs(Directory dir, Term term, int numDocs)
throws IOException {
IndexReader reader = IndexReader.open(dir, true);
- DocsEnum docsEnum = MultiFields.getTermDocsEnum(reader, null, term.field, new BytesRef(term.text));
+ DocsEnum docsEnum = MultiFields.getTermDocsEnum(reader, null, term.field, term.bytes);
int count = 0;
while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS)
count++;
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Thu Jul 22 19:34:35 2010
@@ -23,10 +23,12 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
import java.util.Arrays;
-import java.util.Random;
import java.util.Enumeration;
import java.util.List;
+import java.util.Random;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
@@ -128,21 +130,86 @@ public class TestBackwardsCompatibility
"31.nocfs",
};
+ final String[] unsupportedNames = {"19.cfs",
+ "19.nocfs",
+ "20.cfs",
+ "20.nocfs",
+ "21.cfs",
+ "21.nocfs",
+ "22.cfs",
+ "22.nocfs",
+ "23.cfs",
+ "23.nocfs",
+ "24.cfs",
+ "24.nocfs",
+ "29.cfs",
+ "29.nocfs",
+ };
+
+ /** This test checks that *only* IndexFormatTooOldExceptions are throws when you open and operate on too old indexes! */
+ public void testUnsupportedOldIndexes() throws Exception {
+ final Random rnd = newRandom();
+ for(int i=0;i<unsupportedNames.length;i++) {
+ unzip(getDataFile("unsupported." + unsupportedNames[i] + ".zip"), unsupportedNames[i]);
+
+ String fullPath = fullDir(unsupportedNames[i]);
+ Directory dir = FSDirectory.open(new File(fullPath));
+
+ IndexReader reader = null;
+ IndexWriter writer = null;
+ try {
+ reader = IndexReader.open(dir);
+ fail("IndexReader.open should not pass for "+unsupportedNames[i]);
+ } catch (IndexFormatTooOldException e) {
+ // pass
+ } finally {
+ if (reader != null) reader.close();
+ reader = null;
+ }
+
+ try {
+ writer = new IndexWriter(dir, new IndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer())
+ .setMergeScheduler(new SerialMergeScheduler()) // no threads!
+ );
+ // TODO: Make IndexWriter fail on open!
+ if (rnd.nextBoolean()) {
+ writer.optimize();
+ } else {
+ reader = writer.getReader();
+ }
+ fail("IndexWriter creation should not pass for "+unsupportedNames[i]);
+ } catch (IndexFormatTooOldException e) {
+ // pass
+ } finally {
+ if (reader != null) reader.close();
+ reader = null;
+ if (writer != null) writer.close();
+ writer = null;
+ }
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
+ CheckIndex checker = new CheckIndex(dir);
+ checker.setInfoStream(new PrintStream(bos));
+ CheckIndex.Status indexStatus = checker.checkIndex();
+ assertFalse(indexStatus.clean);
+ assertTrue(bos.toString().contains(IndexFormatTooOldException.class.getName()));
+
+ dir.close();
+ rmDir(unsupportedNames[i]);
+ }
+ }
+
public void testOptimizeOldIndex() throws Exception {
- Random rand = newRandom();
-
for(int i=0;i<oldNames.length;i++) {
unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
String fullPath = fullDir(oldNames[i]);
Directory dir = FSDirectory.open(new File(fullPath));
- FlexTestUtil.verifyFlexVsPreFlex(rand, dir);
-
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer()));
w.optimize();
- FlexTestUtil.verifyFlexVsPreFlex(rand, w);
w.close();
_TestUtil.checkIndex(dir);
@@ -241,30 +308,24 @@ public class TestBackwardsCompatibility
if (!reader.isDeleted(i)) {
Document d = reader.document(i);
List<Fieldable> fields = d.getFields();
- if (!oldName.startsWith("19.") &&
- !oldName.startsWith("20.") &&
- !oldName.startsWith("21.") &&
- !oldName.startsWith("22.")) {
-
- if (d.getField("content3") == null) {
- final int numFields = 5;
- assertEquals(numFields, fields.size());
- Field f = d.getField("id");
- assertEquals(""+i, f.stringValue());
-
- f = d.getField("utf8");
- assertEquals("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());
+ if (d.getField("content3") == null) {
+ final int numFields = 5;
+ assertEquals(numFields, fields.size());
+ Field f = d.getField("id");
+ assertEquals(""+i, f.stringValue());
+
+ f = d.getField("utf8");
+ assertEquals("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());
- f = d.getField("autf8");
- assertEquals("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());
-
- f = d.getField("content2");
- assertEquals("here is more content with aaa aaa aaa", f.stringValue());
+ f = d.getField("autf8");
+ assertEquals("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());
+
+ f = d.getField("content2");
+ assertEquals("here is more content with aaa aaa aaa", f.stringValue());
- f = d.getField("fie\u2C77ld");
- assertEquals("field with non-ascii name", f.stringValue());
- }
- }
+ f = d.getField("fie\u2C77ld");
+ assertEquals("field with non-ascii name", f.stringValue());
+ }
} else
// Only ID 7 is deleted
assertEquals(7, i);
@@ -279,18 +340,12 @@ public class TestBackwardsCompatibility
doTestHits(hits, 34, searcher.getIndexReader());
- if (!oldName.startsWith("19.") &&
- !oldName.startsWith("20.") &&
- !oldName.startsWith("21.") &&
- !oldName.startsWith("22.")) {
- // Test on indices >= 2.3
- hits = searcher.search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).scoreDocs;
- assertEquals(34, hits.length);
- hits = searcher.search(new TermQuery(new Term("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).scoreDocs;
- assertEquals(34, hits.length);
- hits = searcher.search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).scoreDocs;
- assertEquals(34, hits.length);
- }
+ hits = searcher.search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).scoreDocs;
+ assertEquals(34, hits.length);
+ hits = searcher.search(new TermQuery(new Term("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).scoreDocs;
+ assertEquals(34, hits.length);
+ hits = searcher.search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).scoreDocs;
+ assertEquals(34, hits.length);
searcher.close();
dir.close();
@@ -570,12 +625,6 @@ public class TestBackwardsCompatibility
return new File(TEMP_DIR, dirName).getCanonicalPath();
}
- static final String TEXT_TO_COMPRESS = "this is a compressed field and should appear in 3.0 as an uncompressed field after merge";
-
- // FieldSelectorResult.SIZE returns compressed size for compressed fields, which are internally handled as binary;
- // do it in the same way like FieldsWriter, do not use CompressionTools.compressString() for compressed fields:
- static final byte[] BINARY_TO_COMPRESS = new byte[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20};
-
private int countDocs(DocsEnum docs) throws IOException {
int count = 0;
while((docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
Propchange: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jul 22 19:34:35 2010
@@ -1,4 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:943137,949730
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:943137,949730,957490,960490,961612
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:953476-966816
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:829134,829881,831036,896850,909334,948516
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestByteSlices.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestByteSlices.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestByteSlices.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestByteSlices.java Thu Jul 22 19:34:35 2010
@@ -55,7 +55,7 @@ public class TestByteSlices extends Luce
public void testBasic() throws Throwable {
ByteBlockPool pool = new ByteBlockPool(new ByteBlockAllocator());
- final int NUM_STREAM = 25;
+ final int NUM_STREAM = 100*_TestUtil.getRandomMultiplier();
ByteSliceWriter writer = new ByteSliceWriter(pool);
@@ -91,8 +91,9 @@ public class TestByteSlices extends Luce
for(int j=0;j<numValue;j++) {
if (VERBOSE)
System.out.println(" write " + (counters[stream]+j));
+ // write some large (incl. negative) ints:
+ writer.writeVInt(r.nextInt());
writer.writeVInt(counters[stream]+j);
- //writer.writeVInt(ti);
}
counters[stream] += numValue;
uptos[stream] = writer.getAddress();
@@ -104,11 +105,12 @@ public class TestByteSlices extends Luce
if (VERBOSE)
System.out.println(" stream=" + stream + " count=" + counters[stream]);
- if (starts[stream] != uptos[stream]) {
+ if (starts[stream] != -1 && starts[stream] != uptos[stream]) {
reader.init(pool, starts[stream], uptos[stream]);
- for(int j=0;j<counters[stream];j++)
- assertEquals(j, reader.readVInt());
- //assertEquals(ti, reader.readVInt());
+ for(int j=0;j<counters[stream];j++) {
+ reader.readVInt();
+ assertEquals(j, reader.readVInt());
+ }
}
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java Thu Jul 22 19:34:35 2010
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.BytesRef;
import java.io.IOException;
@@ -67,7 +68,6 @@ public class TestDirectoryReader extends
}
public void test() throws Exception {
- setUp();
doTestDocument();
doTestUndeleteAll();
}
@@ -158,35 +158,29 @@ public class TestDirectoryReader extends
MultiReader mr3 = new MultiReader(readers2);
// test mixing up TermDocs and TermEnums from different readers.
- TermDocs td2 = mr2.termDocs();
- TermEnum te3 = mr3.terms(new Term("body","wow"));
- td2.seek(te3);
+ TermsEnum te2 = MultiFields.getTerms(mr2, "body").iterator();
+ te2.seek(new BytesRef("wow"));
+ DocsEnum td = MultiFields.getTermDocsEnum(mr2,
+ MultiFields.getDeletedDocs(mr2),
+ "body",
+ te2.term());
+
+ TermsEnum te3 = MultiFields.getTerms(mr3, "body").iterator();
+ te3.seek(new BytesRef("wow"));
+ td = te3.docs(MultiFields.getDeletedDocs(mr3),
+ td);
+
int ret = 0;
// This should blow up if we forget to check that the TermEnum is from the same
// reader as the TermDocs.
- while (td2.next()) ret += td2.doc();
- td2.close();
- te3.close();
+ while (td.nextDoc() != td.NO_MORE_DOCS) ret += td.docID();
// really a dummy assert to ensure that we got some docs and to ensure that
// nothing is optimized out.
assertTrue(ret > 0);
}
- public void testAllTermDocs() throws IOException {
- IndexReader reader = openReader();
- int NUM_DOCS = 2;
- TermDocs td = reader.termDocs(null);
- for(int i=0;i<NUM_DOCS;i++) {
- assertTrue(td.next());
- assertEquals(i, td.doc());
- assertEquals(1, td.freq());
- }
- td.close();
- reader.close();
- }
-
private void addDoc(RAMDirectory ramDir1, String s, boolean create) throws IOException {
IndexWriter iw = new IndexWriter(ramDir1, new IndexWriterConfig(
TEST_VERSION_CURRENT,
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDoc.java Thu Jul 22 19:34:35 2010
@@ -213,15 +213,19 @@ public class TestDoc extends LuceneTestC
for (int i = 0; i < reader.numDocs(); i++)
out.println(reader.document(i));
- TermEnum tis = reader.terms();
- while (tis.next()) {
- out.print(tis.term());
- out.println(" DF=" + tis.docFreq());
+ FieldsEnum fis = reader.fields().iterator();
+ String field = fis.next();
+ while(field != null) {
+ TermsEnum tis = fis.terms();
+ while(tis.next() != null) {
- TermPositions positions = reader.termPositions(tis.term());
- try {
- while (positions.next()) {
- out.print(" doc=" + positions.doc());
+ out.print(" term=" + field + ":" + tis.term());
+ out.println(" DF=" + tis.docFreq());
+
+ DocsAndPositionsEnum positions = tis.docsAndPositions(reader.getDeletedDocs(), null);
+
+ while (positions.nextDoc() != positions.NO_MORE_DOCS) {
+ out.print(" doc=" + positions.docID());
out.print(" TF=" + positions.freq());
out.print(" pos=");
out.print(positions.nextPosition());
@@ -229,11 +233,9 @@ public class TestDoc extends LuceneTestC
out.print("," + positions.nextPosition());
out.println("");
}
- } finally {
- positions.close();
}
+ field = fis.next();
}
- tis.close();
reader.close();
}
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Thu Jul 22 19:34:35 2010
@@ -38,6 +38,7 @@ import org.apache.lucene.store.RAMDirect
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.BytesRef;
public class TestDocumentWriter extends LuceneTestCase {
private RAMDirectory dir;
@@ -128,8 +129,9 @@ public class TestDocumentWriter extends
writer.close();
SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
- TermPositions termPositions = reader.termPositions(new Term("repeated", "repeated"));
- assertTrue(termPositions.next());
+ DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, MultiFields.getDeletedDocs(reader),
+ "repeated", new BytesRef("repeated"));
+ assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS);
int freq = termPositions.freq();
assertEquals(2, freq);
assertEquals(0, termPositions.nextPosition());
@@ -190,16 +192,16 @@ public class TestDocumentWriter extends
writer.close();
SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
- TermPositions termPositions = reader.termPositions(new Term("f1", "a"));
- assertTrue(termPositions.next());
+ DocsAndPositionsEnum termPositions = reader.fields().terms("f1").docsAndPositions(reader.getDeletedDocs(), new BytesRef("a"), null);
+ assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS);
int freq = termPositions.freq();
assertEquals(3, freq);
assertEquals(0, termPositions.nextPosition());
- assertEquals(true, termPositions.isPayloadAvailable());
+ assertEquals(true, termPositions.hasPayload());
assertEquals(6, termPositions.nextPosition());
- assertEquals(false, termPositions.isPayloadAvailable());
+ assertEquals(false, termPositions.hasPayload());
assertEquals(7, termPositions.nextPosition());
- assertEquals(false, termPositions.isPayloadAvailable());
+ assertEquals(false, termPositions.hasPayload());
}
@@ -233,19 +235,19 @@ public class TestDocumentWriter extends
writer.close();
SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
- TermPositions termPositions = reader.termPositions(new Term("preanalyzed", "term1"));
- assertTrue(termPositions.next());
+ DocsAndPositionsEnum termPositions = reader.fields().terms("preanalyzed").docsAndPositions(reader.getDeletedDocs(), new BytesRef("term1"), null);
+ assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS);
assertEquals(1, termPositions.freq());
assertEquals(0, termPositions.nextPosition());
- termPositions.seek(new Term("preanalyzed", "term2"));
- assertTrue(termPositions.next());
+ termPositions = reader.fields().terms("preanalyzed").docsAndPositions(reader.getDeletedDocs(), new BytesRef("term2"), null);
+ assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS);
assertEquals(2, termPositions.freq());
assertEquals(1, termPositions.nextPosition());
assertEquals(3, termPositions.nextPosition());
- termPositions.seek(new Term("preanalyzed", "term3"));
- assertTrue(termPositions.next());
+ termPositions = reader.fields().terms("preanalyzed").docsAndPositions(reader.getDeletedDocs(), new BytesRef("term3"), null);
+ assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS);
assertEquals(1, termPositions.freq());
assertEquals(2, termPositions.nextPosition());
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Thu Jul 22 19:34:35 2010
@@ -128,6 +128,8 @@ public class TestFieldsReader extends Lu
String value = field.stringValue();
assertTrue("value is null and it shouldn't be", value != null);
assertTrue(value + " is not equal to " + DocHelper.LAZY_FIELD_TEXT, value.equals(DocHelper.LAZY_FIELD_TEXT) == true);
+ assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue());
+
field = doc.getFieldable(DocHelper.TEXT_FIELD_1_KEY);
assertTrue("field is null and it shouldn't be", field != null);
assertTrue("Field is lazy and it should not be", field.isLazy() == false);
@@ -148,12 +150,85 @@ public class TestFieldsReader extends Lu
byte [] bytes = field.getBinaryValue();
assertTrue("bytes is null and it shouldn't be", bytes != null);
assertTrue("", DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
+ assertTrue("calling binaryValue() twice should give same reference", field.getBinaryValue() == field.getBinaryValue());
for (int i = 0; i < bytes.length; i++) {
assertTrue("byte[" + i + "] is mismatched", bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i]);
}
}
+ public void testLatentFields() throws Exception {
+ assertTrue(dir != null);
+ assertTrue(fieldInfos != null);
+ FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos);
+ assertTrue(reader != null);
+ assertTrue(reader.size() == 1);
+ Set<String> loadFieldNames = new HashSet<String>();
+ loadFieldNames.add(DocHelper.TEXT_FIELD_1_KEY);
+ loadFieldNames.add(DocHelper.TEXT_FIELD_UTF1_KEY);
+ Set<String> lazyFieldNames = new HashSet<String>();
+ //new String[]{DocHelper.LARGE_LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_BINARY_KEY};
+ lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
+ lazyFieldNames.add(DocHelper.LAZY_FIELD_KEY);
+ lazyFieldNames.add(DocHelper.LAZY_FIELD_BINARY_KEY);
+ lazyFieldNames.add(DocHelper.TEXT_FIELD_UTF2_KEY);
+
+ // Use LATENT instead of LAZY
+ SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames) {
+ public FieldSelectorResult accept(String fieldName) {
+ final FieldSelectorResult result = super.accept(fieldName);
+ if (result == FieldSelectorResult.LAZY_LOAD) {
+ return FieldSelectorResult.LATENT;
+ } else {
+ return result;
+ }
+ }
+ };
+
+ Document doc = reader.doc(0, fieldSelector);
+ assertTrue("doc is null and it shouldn't be", doc != null);
+ Fieldable field = doc.getFieldable(DocHelper.LAZY_FIELD_KEY);
+ assertTrue("field is null and it shouldn't be", field != null);
+ assertTrue("field is not lazy and it should be", field.isLazy());
+ String value = field.stringValue();
+ assertTrue("value is null and it shouldn't be", value != null);
+ assertTrue(value + " is not equal to " + DocHelper.LAZY_FIELD_TEXT, value.equals(DocHelper.LAZY_FIELD_TEXT) == true);
+ assertTrue("calling stringValue() twice should give different references", field.stringValue() != field.stringValue());
+
+ field = doc.getFieldable(DocHelper.TEXT_FIELD_1_KEY);
+ assertTrue("field is null and it shouldn't be", field != null);
+ assertTrue("Field is lazy and it should not be", field.isLazy() == false);
+ assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue());
+
+ field = doc.getFieldable(DocHelper.TEXT_FIELD_UTF1_KEY);
+ assertTrue("field is null and it shouldn't be", field != null);
+ assertTrue("Field is lazy and it should not be", field.isLazy() == false);
+ assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF1_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF1_TEXT) == true);
+ assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue());
+
+ field = doc.getFieldable(DocHelper.TEXT_FIELD_UTF2_KEY);
+ assertTrue("field is null and it shouldn't be", field != null);
+ assertTrue("Field is lazy and it should not be", field.isLazy() == true);
+ assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF2_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF2_TEXT) == true);
+ assertTrue("calling stringValue() twice should give different references", field.stringValue() != field.stringValue());
+
+ field = doc.getFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
+ assertTrue("field is null and it shouldn't be", field != null);
+ assertTrue("stringValue isn't null for lazy binary field", field.stringValue() == null);
+ assertTrue("calling binaryValue() twice should give different references", field.getBinaryValue() != field.getBinaryValue());
+
+ byte [] bytes = field.getBinaryValue();
+ assertTrue("bytes is null and it shouldn't be", bytes != null);
+ assertTrue("", DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
+ for (int i = 0; i < bytes.length; i++) {
+ assertTrue("byte[" + i + "] is mismatched", bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i]);
+
+ }
+ }
+
+
+
+
public void testLazyFieldsAfterClose() throws Exception {
assertTrue(dir != null);
assertTrue(fieldInfos != null);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java Thu Jul 22 19:34:35 2010
@@ -159,26 +159,19 @@ public class TestFilterIndexReader exten
assertTrue(reader.isOptimized());
- TermEnum terms = reader.terms();
- while (terms.next()) {
- assertTrue(terms.term().text().indexOf('e') != -1);
+ TermsEnum terms = MultiFields.getTerms(reader, "default").iterator();
+ while (terms.next() != null) {
+ assertTrue(terms.term().utf8ToString().indexOf('e') != -1);
}
- terms.close();
- TermPositions positions = reader.termPositions(new Term("default", "one"));
- while (positions.next()) {
- assertTrue((positions.doc() % 2) == 1);
+ assertEquals(TermsEnum.SeekStatus.FOUND, terms.seek(new BytesRef("one")));
+
+ DocsAndPositionsEnum positions = terms.docsAndPositions(MultiFields.getDeletedDocs(reader),
+ null);
+ while (positions.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+ assertTrue((positions.docID() % 2) == 1);
}
- int NUM_DOCS = 3;
-
- TermDocs td = reader.termDocs(null);
- for(int i=0;i<NUM_DOCS;i++) {
- assertTrue(td.next());
- assertEquals(i, td.doc());
- assertEquals(1, td.freq());
- }
- td.close();
reader.close();
directory.close();
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFlex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFlex.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFlex.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestFlex.java Thu Jul 22 19:34:35 2010
@@ -52,13 +52,9 @@ public class TestFlex extends LuceneTest
}
IndexReader r = w.getReader();
- TermEnum terms = r.terms(new Term("field3", "bbb"));
- // pre-flex API should seek to the next field
- assertNotNull(terms.term());
- assertEquals("field4", terms.term().field());
- terms = r.terms(new Term("field5", "abc"));
- assertNull(terms.term());
+ TermsEnum terms = MultiFields.getTerms(r, "field3").iterator();
+ assertEquals(TermsEnum.SeekStatus.END, terms.seek(new BytesRef("abc")));
r.close();
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Thu Jul 22 19:34:35 2010
@@ -55,6 +55,8 @@ import org.apache.lucene.store.NoSuchDir
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Bits;
public class TestIndexReader extends LuceneTestCase
{
@@ -287,22 +289,17 @@ public class TestIndexReader extends Luc
int expected)
throws IOException
{
- TermDocs tdocs = null;
-
- try {
- tdocs = reader.termDocs(term);
- assertNotNull(msg + ", null TermDocs", tdocs);
- int count = 0;
- while(tdocs.next()) {
- count++;
- }
- assertEquals(msg + ", count mismatch", expected, count);
-
- } finally {
- if (tdocs != null)
- tdocs.close();
+ DocsEnum tdocs = MultiFields.getTermDocsEnum(reader,
+ MultiFields.getDeletedDocs(reader),
+ term.field(),
+ new BytesRef(term.text()));
+ int count = 0;
+ if (tdocs != null) {
+ while(tdocs.nextDoc()!= tdocs.NO_MORE_DOCS) {
+ count++;
+ }
}
-
+ assertEquals(msg + ", count mismatch", expected, count);
}
public void testBasicDelete() throws IOException {
@@ -1348,21 +1345,26 @@ public class TestIndexReader extends Luc
}
// check dictionary and posting lists
- TermEnum enum1 = index1.terms();
- TermEnum enum2 = index2.terms();
- TermPositions tp1 = index1.termPositions();
- TermPositions tp2 = index2.termPositions();
- while(enum1.next()) {
- assertTrue(enum2.next());
- assertEquals("Different term in dictionary.", enum1.term(), enum2.term());
- tp1.seek(enum1.term());
- tp2.seek(enum1.term());
- while(tp1.next()) {
- assertTrue(tp2.next());
- assertEquals("Different doc id in postinglist of term " + enum1.term() + ".", tp1.doc(), tp2.doc());
- assertEquals("Different term frequence in postinglist of term " + enum1.term() + ".", tp1.freq(), tp2.freq());
- for (int i = 0; i < tp1.freq(); i++) {
- assertEquals("Different positions in postinglist of term " + enum1.term() + ".", tp1.nextPosition(), tp2.nextPosition());
+ FieldsEnum fenum1 = MultiFields.getFields(index1).iterator();
+ FieldsEnum fenum2 = MultiFields.getFields(index1).iterator();
+ String field1 = null;
+ Bits delDocs = MultiFields.getDeletedDocs(index1);
+ while((field1=fenum1.next()) != null) {
+ assertEquals("Different fields", field1, fenum2.next());
+ TermsEnum enum1 = fenum1.terms();
+ TermsEnum enum2 = fenum2.terms();
+ while(enum1.next() != null) {
+ assertEquals("Different terms", enum1.term(), enum2.next());
+ DocsAndPositionsEnum tp1 = enum1.docsAndPositions(delDocs, null);
+ DocsAndPositionsEnum tp2 = enum2.docsAndPositions(delDocs, null);
+
+ while(tp1.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+ assertTrue(tp2.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals("Different doc id in postinglist of term " + enum1.term() + ".", tp1.docID(), tp2.docID());
+ assertEquals("Different term frequence in postinglist of term " + enum1.term() + ".", tp1.freq(), tp2.freq());
+ for (int i = 0; i < tp1.freq(); i++) {
+ assertEquals("Different positions in postinglist of term " + enum1.term() + ".", tp1.nextPosition(), tp2.nextPosition());
+ }
}
}
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Thu Jul 22 19:34:35 2010
@@ -863,8 +863,6 @@ public class TestIndexReaderReopen exten
assertReaderClosed(reader, true, true);
assertReaderClosed(firstReader, true, true);
- FlexTestUtil.verifyFlexVsPreFlex(rnd, dir);
-
dir.close();
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Thu Jul 22 19:34:35 2010
@@ -743,6 +743,7 @@ public class TestIndexWriter extends Luc
try {
writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE));
} catch (Exception e) {
+ e.printStackTrace(System.out);
fail("writer failed to open on a crashed index");
}
@@ -1379,8 +1380,11 @@ public class TestIndexWriter extends Luc
assertEquals(1, reader.numDocs());
Term t = new Term("field", "a");
assertEquals(1, reader.docFreq(t));
- TermDocs td = reader.termDocs(t);
- td.next();
+ DocsEnum td = MultiFields.getTermDocsEnum(reader,
+ MultiFields.getDeletedDocs(reader),
+ "field",
+ new BytesRef("a"));
+ td.nextDoc();
assertEquals(128*1024, td.freq());
reader.close();
dir.close();
@@ -1701,9 +1705,13 @@ public class TestIndexWriter extends Luc
// Make sure the doc that hit the exception was marked
// as deleted:
- TermDocs tdocs = reader.termDocs(t);
+ DocsEnum tdocs = MultiFields.getTermDocsEnum(reader,
+ MultiFields.getDeletedDocs(reader),
+ t.field(),
+ new BytesRef(t.text()));
+
int count = 0;
- while(tdocs.next()) {
+ while(tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
count++;
}
assertEquals(2, count);
@@ -2244,9 +2252,12 @@ public class TestIndexWriter extends Luc
// Quick test to make sure index is not corrupt:
IndexReader reader = IndexReader.open(dir, true);
- TermDocs tdocs = reader.termDocs(new Term("field", "aaa"));
+ DocsEnum tdocs = MultiFields.getTermDocsEnum(reader,
+ MultiFields.getDeletedDocs(reader),
+ "field",
+ new BytesRef("aaa"));
int count = 0;
- while(tdocs.next()) {
+ while(tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
count++;
}
assertTrue(count > 0);
@@ -3454,8 +3465,13 @@ public class TestIndexWriter extends Luc
Query q = new SpanTermQuery(new Term("field", "a"));
hits = s.search(q, null, 1000).scoreDocs;
assertEquals(1, hits.length);
- TermPositions tps = s.getIndexReader().termPositions(new Term("field", "a"));
- assertTrue(tps.next());
+
+ DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(s.getIndexReader(),
+ MultiFields.getDeletedDocs(s.getIndexReader()),
+ "field",
+ new BytesRef("a"));
+
+ assertTrue(tps.nextDoc() != DocsEnum.NO_MORE_DOCS);
assertEquals(1, tps.freq());
assertEquals(0, tps.nextPosition());
w.close();
@@ -4465,12 +4481,12 @@ public class TestIndexWriter extends Luc
// test that the terms were indexed.
- assertTrue(ir.termDocs(new Term("binary","doc1field1")).next());
- assertTrue(ir.termDocs(new Term("binary","doc2field1")).next());
- assertTrue(ir.termDocs(new Term("binary","doc3field1")).next());
- assertTrue(ir.termDocs(new Term("string","doc1field2")).next());
- assertTrue(ir.termDocs(new Term("string","doc2field2")).next());
- assertTrue(ir.termDocs(new Term("string","doc3field2")).next());
+ assertTrue(MultiFields.getTermDocsEnum(ir, null, "binary", new BytesRef("doc1field1")).nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertTrue(MultiFields.getTermDocsEnum(ir, null, "binary", new BytesRef("doc2field1")).nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertTrue(MultiFields.getTermDocsEnum(ir, null, "binary", new BytesRef("doc3field1")).nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertTrue(MultiFields.getTermDocsEnum(ir, null, "string", new BytesRef("doc1field2")).nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertTrue(MultiFields.getTermDocsEnum(ir, null, "string", new BytesRef("doc2field2")).nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertTrue(MultiFields.getTermDocsEnum(ir, null, "string", new BytesRef("doc3field2")).nextDoc() != DocsEnum.NO_MORE_DOCS);
ir.close();
dir.close();
@@ -4606,38 +4622,22 @@ public class TestIndexWriter extends Luc
private void checkTermsOrder(IndexReader r, Set<String> allTerms, boolean isTop) throws IOException {
TermsEnum terms = MultiFields.getFields(r).terms("f").iterator();
- char[] last = new char[2];
- int lastLength = 0;
+ BytesRef last = new BytesRef();
Set<String> seenTerms = new HashSet<String>();
- UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
while(true) {
final BytesRef term = terms.next();
if (term == null) {
break;
}
- UnicodeUtil.UTF8toUTF16(term.bytes, term.offset, term.length, utf16);
- assertTrue(utf16.length <= 2);
- // Make sure last term comes before current one, in
- // UTF16 sort order
- int i = 0;
- for(i=0;i<lastLength && i<utf16.length;i++) {
- assertTrue("UTF16 code unit " + termDesc(new String(utf16.result, 0, utf16.length)) + " incorrectly sorted after code unit " + termDesc(new String(last, 0, lastLength)), last[i] <= utf16.result[i]);
- if (last[i] < utf16.result[i]) {
- break;
- }
- }
- // Terms should not have been identical
- assertTrue(lastLength != utf16.length || i < lastLength);
+ assertTrue(last.compareTo(term) < 0);
+ last.copy(term);
- final String s = new String(utf16.result, 0, utf16.length);
+ final String s = term.utf8ToString();
assertTrue("term " + termDesc(s) + " was not added to index (count=" + allTerms.size() + ")", allTerms.contains(s));
seenTerms.add(s);
-
- System.arraycopy(utf16.result, 0, last, 0, utf16.length);
- lastLength = utf16.length;
}
if (isTop) {
@@ -4955,5 +4955,105 @@ public class TestIndexWriter extends Luc
writer.close();
assertEquals("expected a no-op close after IW.rollback()", 0, dir.listAll().length);
}
-
+
+ public void testNoSegmentFile() throws IOException {
+ File tempDir = _TestUtil.getTempDir("noSegmentFile");
+ try {
+ Directory dir = FSDirectory.open(tempDir);
+ dir.setLockFactory(NoLockFactory.getNoLockFactory());
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(2));
+
+ Document doc = new Document();
+ doc.add(new Field("c", "val", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
+ w.addDocument(doc);
+ w.addDocument(doc);
+ IndexWriter w2 = new IndexWriter(dir, new IndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(2)
+ .setOpenMode(OpenMode.CREATE));
+
+ w2.close();
+ // If we don't do that, the test fails on Windows
+ w.rollback();
+ dir.close();
+ } finally {
+ _TestUtil.rmDir(tempDir);
+ }
+ }
+
+ public void testFutureCommit() throws Exception {
+ Directory dir = new MockRAMDirectory();
+
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
+ Document doc = new Document();
+ w.addDocument(doc);
+
+ // commit to "first"
+ Map<String,String> commitData = new HashMap<String,String>();
+ commitData.put("tag", "first");
+ w.commit(commitData);
+
+ // commit to "second"
+ w.addDocument(doc);
+ commitData.put("tag", "second");
+ w.commit(commitData);
+ w.close();
+
+ // open "first" with IndexWriter
+ IndexCommit commit = null;
+ for(IndexCommit c : IndexReader.listCommits(dir)) {
+ if (c.getUserData().get("tag").equals("first")) {
+ commit = c;
+ break;
+ }
+ }
+
+ assertNotNull(commit);
+
+ w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE).setIndexCommit(commit));
+
+ assertEquals(1, w.numDocs());
+
+ // commit IndexWriter to "third"
+ w.addDocument(doc);
+ commitData.put("tag", "third");
+ w.commit(commitData);
+ w.close();
+
+ // make sure "second" commit is still there
+ commit = null;
+ for(IndexCommit c : IndexReader.listCommits(dir)) {
+ if (c.getUserData().get("tag").equals("second")) {
+ commit = c;
+ break;
+ }
+ }
+
+ assertNotNull(commit);
+
+ IndexReader r = IndexReader.open(commit, true);
+ assertEquals(2, r.numDocs());
+ r.close();
+
+ // open "second", w/ writeable IndexReader & commit
+ r = IndexReader.open(commit, NoDeletionPolicy.INSTANCE, false);
+ assertEquals(2, r.numDocs());
+ r.deleteDocument(0);
+ r.deleteDocument(1);
+ commitData.put("tag", "fourth");
+ r.commit(commitData);
+ r.close();
+
+ // make sure "third" commit is still there
+ commit = null;
+ for(IndexCommit c : IndexReader.listCommits(dir)) {
+ if (c.getUserData().get("tag").equals("third")) {
+ commit = c;
+ break;
+ }
+ }
+ assertNotNull(commit);
+
+ dir.close();
+ }
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Thu Jul 22 19:34:35 2010
@@ -38,6 +38,7 @@ import org.apache.lucene.store.MockRAMDi
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ThreadInterruptedException;
public class TestIndexWriterReader extends LuceneTestCase {
@@ -63,12 +64,16 @@ public class TestIndexWriterReader exten
public static int count(Term t, IndexReader r) throws IOException {
int count = 0;
- TermDocs td = r.termDocs(t);
- while (td.next()) {
- td.doc();
- count++;
+ DocsEnum td = MultiFields.getTermDocsEnum(r,
+ MultiFields.getDeletedDocs(r),
+ t.field(), new BytesRef(t.text()));
+
+ if (td != null) {
+ while (td.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+ td.docID();
+ count++;
+ }
}
- td.close();
return count;
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Thu Jul 22 19:34:35 2010
@@ -30,6 +30,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
/**
* Tests lazy skipping on the proximity file.
@@ -127,17 +128,26 @@ public class TestLazyProxSkipping extend
writer.close();
IndexReader reader = IndexReader.open(directory, true);
- TermPositions tp = reader.termPositions();
- tp.seek(new Term(this.field, "b"));
+
+ DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader,
+ MultiFields.getDeletedDocs(reader),
+ this.field,
+ new BytesRef("b"));
+
for (int i = 0; i < 10; i++) {
- tp.next();
- assertEquals(tp.doc(), i);
+ tp.nextDoc();
+ assertEquals(tp.docID(), i);
assertEquals(tp.nextPosition(), 1);
}
- tp.seek(new Term(this.field, "a"));
+
+ tp = MultiFields.getTermPositionsEnum(reader,
+ MultiFields.getDeletedDocs(reader),
+ this.field,
+ new BytesRef("a"));
+
for (int i = 0; i < 10; i++) {
- tp.next();
- assertEquals(tp.doc(), i);
+ tp.nextDoc();
+ assertEquals(tp.docID(), i);
assertEquals(tp.nextPosition(), 0);
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiFields.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiFields.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiFields.java Thu Jul 22 19:34:35 2010
@@ -37,7 +37,7 @@ public class TestMultiFields extends Luc
Set<Integer> deleted = new HashSet<Integer>();
List<BytesRef> terms = new ArrayList<BytesRef>();
- int numDocs = r.nextInt(100*_TestUtil.getRandomMultiplier());
+ int numDocs = _TestUtil.nextInt(r, 1, 100*_TestUtil.getRandomMultiplier());
Document doc = new Document();
Field f = new Field("field", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
doc.add(f);
@@ -88,14 +88,13 @@ public class TestMultiFields extends Luc
BytesRef term = terms.get(r.nextInt(terms.size()));
DocsEnum docsEnum = terms2.docs(delDocs, term, null);
- int count = 0;
+ assertNotNull(docsEnum);
+
for(int docID : docs.get(term)) {
if (!deleted.contains(docID)) {
assertEquals(docID, docsEnum.nextDoc());
- count++;
}
}
- //System.out.println("c=" + count + " t=" + term);
assertEquals(docsEnum.NO_MORE_DOCS, docsEnum.nextDoc());
}
@@ -104,6 +103,7 @@ public class TestMultiFields extends Luc
}
}
+ /*
private void verify(IndexReader r, String term, List<Integer> expected) throws Exception {
DocsEnum docs = MultiFields.getTermDocsEnum(r,
MultiFields.getDeletedDocs(r),
@@ -115,4 +115,23 @@ public class TestMultiFields extends Luc
}
assertEquals(docs.NO_MORE_DOCS, docs.nextDoc());
}
+ */
+
+ public void testSeparateEnums() throws Exception {
+ Directory dir = new MockRAMDirectory();
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ Document d = new Document();
+ d.add(new Field("f", "j", Field.Store.NO, Field.Index.NOT_ANALYZED));
+ w.addDocument(d);
+ w.commit();
+ w.addDocument(d);
+ IndexReader r = w.getReader();
+ w.close();
+ DocsEnum d1 = MultiFields.getTermDocsEnum(r, null, "f", new BytesRef("j"));
+ DocsEnum d2 = MultiFields.getTermDocsEnum(r, null, "f", new BytesRef("j"));
+ assertEquals(0, d1.nextDoc());
+ assertEquals(0, d2.nextDoc());
+ r.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Thu Jul 22 19:34:35 2010
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -34,6 +33,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.MockRAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
/**
* This testcase tests whether multi-level skipping is being used
@@ -68,11 +68,12 @@ public class TestMultiLevelSkipList exte
writer.close();
IndexReader reader = SegmentReader.getOnlySegmentReader(dir);
- TermPositions tp = reader.termPositions();
for (int i = 0; i < 2; i++) {
counter = 0;
- tp.seek(term);
+ DocsAndPositionsEnum tp = reader.termPositionsEnum(reader.getDeletedDocs(),
+ term.field(),
+ new BytesRef(term.text()));
checkSkipTo(tp, 14, 185); // no skips
checkSkipTo(tp, 17, 190); // one skip on level 0
@@ -84,18 +85,18 @@ public class TestMultiLevelSkipList exte
}
}
- public void checkSkipTo(TermPositions tp, int target, int maxCounter) throws IOException {
- tp.skipTo(target);
+ public void checkSkipTo(DocsAndPositionsEnum tp, int target, int maxCounter) throws IOException {
+ tp.advance(target);
if (maxCounter < counter) {
fail("Too many bytes read: " + counter);
}
- assertEquals("Wrong document " + tp.doc() + " after skipTo target " + target, target, tp.doc());
+ assertEquals("Wrong document " + tp.docID() + " after skipTo target " + target, target, tp.docID());
assertEquals("Frequency is not 1: " + tp.freq(), 1,tp.freq());
tp.nextPosition();
- byte[] b = new byte[1];
- tp.getPayload(b, 0);
- assertEquals("Wrong payload for the target " + target + ": " + b[0], (byte) target, b[0]);
+ BytesRef b = tp.getPayload();
+ assertEquals(1, b.length);
+ assertEquals("Wrong payload for the target " + target + ": " + b.bytes[b.offset], (byte) target, b.bytes[b.offset]);
}
private static class PayloadAnalyzer extends Analyzer {
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Thu Jul 22 19:34:35 2010
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collection;
-import java.util.Random;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -87,10 +86,7 @@ public class TestOmitTf extends LuceneTe
f2.setOmitTermFreqAndPositions(false);
d.add(f2);
- Random rnd = newRandom();
-
writer.addDocument(d);
- FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
// force merge
writer.optimize();
@@ -98,8 +94,6 @@ public class TestOmitTf extends LuceneTe
writer.close();
_TestUtil.checkIndex(ram);
- FlexTestUtil.verifyFlexVsPreFlex(rnd, ram);
-
SegmentReader reader = SegmentReader.getOnlySegmentReader(ram);
FieldInfos fi = reader.fieldInfos();
assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f1").omitTermFreqAndPositions);
@@ -145,12 +139,8 @@ public class TestOmitTf extends LuceneTe
for(int i=0;i<30;i++)
writer.addDocument(d);
- Random rnd = newRandom();
- FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
-
// force merge
writer.optimize();
- FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
// flush
writer.close();
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestParallelReader.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestParallelReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestParallelReader.java Thu Jul 22 19:34:35 2010
@@ -195,26 +195,6 @@ public class TestParallelReader extends
}
- public void testAllTermDocs() throws IOException {
- Directory dir1 = getDir1();
- Directory dir2 = getDir2();
- ParallelReader pr = new ParallelReader();
- pr.add(IndexReader.open(dir1, false));
- pr.add(IndexReader.open(dir2, false));
- int NUM_DOCS = 2;
- TermDocs td = pr.termDocs(null);
- for(int i=0;i<NUM_DOCS;i++) {
- assertTrue(td.next());
- assertEquals(i, td.doc());
- assertEquals(1, td.freq());
- }
- td.close();
- pr.close();
- dir1.close();
- dir2.close();
- }
-
-
private void queryTest(Query query) throws IOException {
ScoreDoc[] parallelHits = parallel.search(query, null, 1000).scoreDocs;
ScoreDoc[] singleHits = single.search(query, null, 1000).scoreDocs;
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java Thu Jul 22 19:34:35 2010
@@ -26,6 +26,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Bits;
public class TestParallelTermEnum extends LuceneTestCase {
private IndexReader ir1;
@@ -77,105 +78,123 @@ public class TestParallelTermEnum extend
pr.add(ir1);
pr.add(ir2);
- TermDocs td = pr.termDocs();
+ Bits delDocs = pr.getDeletedDocs();
- TermEnum te = pr.terms();
- assertTrue(te.next());
- assertEquals("field1:brown", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field1:fox", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field1:jumps", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field1:quick", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field1:the", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field2:brown", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field2:fox", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field2:jumps", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field2:quick", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field2:the", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field3:dog", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field3:fox", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field3:jumps", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field3:lazy", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field3:over", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertTrue(te.next());
- assertEquals("field3:the", te.term().toString());
- td.seek(te.term());
- assertTrue(td.next());
- assertEquals(0, td.doc());
- assertFalse(td.next());
- assertFalse(te.next());
+ FieldsEnum fe = pr.fields().iterator();
+
+ String f = fe.next();
+ assertEquals("field0", f);
+ f = fe.next();
+ assertEquals("field1", f);
+
+ TermsEnum te = fe.terms();
+
+ assertEquals("brown", te.next().utf8ToString());
+ DocsEnum td = te.docs(delDocs, null);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("fox", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("jumps", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("quick", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("the", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertNull(te.next());
+ f = fe.next();
+ assertEquals("field2", f);
+ te = fe.terms();
+
+ assertEquals("brown", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("fox", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("jumps", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("quick", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("the", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertNull(te.next());
+ f = fe.next();
+ assertEquals("field3", f);
+ te = fe.terms();
+
+ assertEquals("dog", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("fox", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("jumps", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("lazy", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("over", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertEquals("the", te.next().utf8ToString());
+ td = te.docs(delDocs, td);
+ assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(0, td.docID());
+ assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS);
+
+ assertNull(te.next());
}
}