You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/11/17 01:43:47 UTC
svn commit: r1640053 [5/10] - in /lucene/dev/branches/lucene6005/lucene:
core/src/java/org/apache/lucene/document/
core/src/java/org/apache/lucene/index/ core/src/test/org/apache/lucene/
core/src/test/org/apache/lucene/codecs/compressing/ core/src/test...
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1640053&r1=1640052&r2=1640053&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Mon Nov 17 00:43:44 2014
@@ -32,9 +32,11 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.search.IndexSearcher;
@@ -61,14 +63,16 @@ public class TestIndexWriterDelete exten
IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))
.setMaxBufferedDeleteTerms(1));
- FieldType custom1 = new FieldType();
- custom1.setStored(true);
+
+ FieldTypes fieldTypes = modifier.getFieldTypes();
+ fieldTypes.disableStored("contents");
+
for (int i = 0; i < keywords.length; i++) {
- Document doc = new Document();
- doc.add(newStringField("id", keywords[i], Field.Store.YES));
- doc.add(newField("country", unindexed[i], custom1));
- doc.add(newTextField("contents", unstored[i], Field.Store.NO));
- doc.add(newTextField("city", text[i], Field.Store.YES));
+ Document2 doc = modifier.newDocument();
+ doc.addAtom("id", keywords[i]);
+ doc.addStored("country", unindexed[i]);
+ doc.addLargeText("contents", unstored[i]);
+ doc.addLargeText("city", text[i]);
modifier.addDocument(doc);
}
modifier.forceMerge(1);
@@ -100,6 +104,7 @@ public class TestIndexWriterDelete exten
IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))
.setMaxBufferedDocs(2)
.setMaxBufferedDeleteTerms(2));
+ FieldTypes fieldTypes = modifier.getFieldTypes();
int id = 0;
int value = 100;
@@ -117,7 +122,7 @@ public class TestIndexWriterDelete exten
assertEquals(7, reader.numDocs());
reader.close();
- modifier.deleteDocuments(new Term("value", String.valueOf(value)));
+ modifier.deleteDocuments(fieldTypes.newIntTerm("value", value));
modifier.commit();
@@ -133,7 +138,7 @@ public class TestIndexWriterDelete exten
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))
.setMaxBufferedDeleteTerms(1));
- writer.addDocument(new Document());
+ writer.addDocument(writer.newDocument());
writer.deleteDocuments(new Term("foobar", "1"));
writer.deleteDocuments(new Term("foobar", "1"));
writer.deleteDocuments(new Term("foobar", "1"));
@@ -152,22 +157,25 @@ public class TestIndexWriterDelete exten
IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))
.setMaxBufferedDocs(4)
.setMaxBufferedDeleteTerms(4));
+ FieldTypes fieldTypes = modifier.getFieldTypes();
+
int id = 0;
int value = 100;
addDoc(modifier, ++id, value);
- if (0 == t)
- modifier.deleteDocuments(new Term("value", String.valueOf(value)));
- else
- modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value))));
+ if (0 == t) {
+ modifier.deleteDocuments(fieldTypes.newIntTerm("value", value));
+ } else {
+ modifier.deleteDocuments(fieldTypes.newIntTermQuery("value", value));
+ }
addDoc(modifier, ++id, value);
if (0 == t) {
- modifier.deleteDocuments(new Term("value", String.valueOf(value)));
+ modifier.deleteDocuments(fieldTypes.newIntTerm("value", value));
assertEquals(2, modifier.getNumBufferedDeleteTerms());
assertEquals(1, modifier.getBufferedDeleteTermsSize());
+ } else {
+ modifier.deleteDocuments(fieldTypes.newIntTermQuery("value", value));
}
- else
- modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value))));
addDoc(modifier, ++id, value);
assertEquals(0, modifier.getSegmentCount());
@@ -176,7 +184,7 @@ public class TestIndexWriterDelete exten
IndexReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.numDocs());
- int hitCount = getHitCount(dir, new Term("id", String.valueOf(id)));
+ int hitCount = getHitCount(dir, fieldTypes.newIntTerm("id", id));
assertEquals(1, hitCount);
reader.close();
modifier.close();
@@ -190,6 +198,7 @@ public class TestIndexWriterDelete exten
IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))
.setMaxBufferedDocs(100)
.setMaxBufferedDeleteTerms(100));
+ FieldTypes fieldTypes = modifier.getFieldTypes();
int id = 0;
int value = 100;
@@ -207,7 +216,7 @@ public class TestIndexWriterDelete exten
for (int i = 0; i < 5; i++) {
addDoc(modifier, ++id, value);
}
- modifier.deleteDocuments(new Term("value", String.valueOf(value)));
+ modifier.deleteDocuments(fieldTypes.newIntTerm("value", value));
modifier.commit();
@@ -224,6 +233,7 @@ public class TestIndexWriterDelete exten
IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))
.setMaxBufferedDocs(2)
.setMaxBufferedDeleteTerms(2));
+ FieldTypes fieldTypes = modifier.getFieldTypes();
int id = 0;
int value = 100;
@@ -238,8 +248,8 @@ public class TestIndexWriterDelete exten
reader.close();
id = 0;
- modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
- modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
+ modifier.deleteDocuments(fieldTypes.newIntTerm("id", ++id));
+ modifier.deleteDocuments(fieldTypes.newIntTerm("id", ++id));
modifier.commit();
@@ -249,7 +259,7 @@ public class TestIndexWriterDelete exten
Term[] terms = new Term[3];
for (int i = 0; i < terms.length; i++) {
- terms[i] = new Term("id", String.valueOf(++id));
+ terms[i] = fieldTypes.newIntTerm("id", ++id);
}
modifier.deleteDocuments(terms);
modifier.commit();
@@ -325,16 +335,19 @@ public class TestIndexWriterDelete exten
try {
latch.await();
for (int j = 0; j < 1000; j++) {
- Document doc = new Document();
- doc.add(newTextField("content", "aaa", Field.Store.NO));
- doc.add(newStringField("id", String.valueOf(id++), Field.Store.YES));
- doc.add(newStringField("value", String.valueOf(value), Field.Store.NO));
- doc.add(new NumericDocValuesField("dv", value));
+ Document2 doc = modifier.newDocument();
+ doc.addLargeText("content", "aaa");
+ doc.addUniqueInt("id", id++);
+ doc.addInt("value", value);
+ doc.addInt("dv", value);
modifier.addDocument(doc);
if (VERBOSE) {
System.out.println("\tThread["+offset+"]: add doc: " + id);
}
}
+ } catch (IllegalArgumentException e) {
+ // Expected: we are deleting FieldTypes while indexing
+ assertTrue(e.getMessage().contains("is not recognized"));
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
@@ -450,22 +463,24 @@ public class TestIndexWriterDelete exten
private void updateDoc(IndexWriter modifier, int id, int value)
throws IOException {
- Document doc = new Document();
- doc.add(newTextField("content", "aaa", Field.Store.NO));
- doc.add(newStringField("id", String.valueOf(id), Field.Store.YES));
- doc.add(newStringField("value", String.valueOf(value), Field.Store.NO));
- doc.add(new NumericDocValuesField("dv", value));
- modifier.updateDocument(new Term("id", String.valueOf(id)), doc);
+ Document2 doc = modifier.newDocument();
+ doc.addLargeText("content", "aaa");
+ doc.addUniqueInt("id", id);
+ doc.addInt("value", value);
+ doc.addInt("dv", value);
+
+ FieldTypes fieldTypes = modifier.getFieldTypes();
+ modifier.updateDocument(fieldTypes.newIntTerm("id", id), doc);
}
private void addDoc(IndexWriter modifier, int id, int value)
throws IOException {
- Document doc = new Document();
- doc.add(newTextField("content", "aaa", Field.Store.NO));
- doc.add(newStringField("id", String.valueOf(id), Field.Store.YES));
- doc.add(newStringField("value", String.valueOf(value), Field.Store.NO));
- doc.add(new NumericDocValuesField("dv", value));
+ Document2 doc = modifier.newDocument();
+ doc.addLargeText("content", "aaa");
+ doc.addUniqueInt("id", id);
+ doc.addInt("value", value);
+ doc.addInt("dv", value);
modifier.addDocument(doc);
}
@@ -503,10 +518,10 @@ public class TestIndexWriterDelete exten
startDir.setEnableVirusScanner(false);
IndexWriter writer = new IndexWriter(startDir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
for (int i = 0; i < 157; i++) {
- Document d = new Document();
- d.add(newStringField("id", Integer.toString(i), Field.Store.YES));
- d.add(newTextField("content", "aaa " + i, Field.Store.NO));
- d.add(new NumericDocValuesField("dv", i));
+ Document2 d = writer.newDocument();
+ d.addUniqueInt("id", i);
+ d.addLargeText("content", "aaa " + i);
+ d.addInt("dv", i);
writer.addDocument(d);
}
writer.close();
@@ -534,6 +549,7 @@ public class TestIndexWriterDelete exten
.setMaxBufferedDeleteTerms(1000)
.setMergeScheduler(new ConcurrentMergeScheduler()));
((ConcurrentMergeScheduler) modifier.getConfig().getMergeScheduler()).setSuppressExceptions();
+ FieldTypes fieldTypes = modifier.getFieldTypes();
// For each disk size, first try to commit against
// dir that will hit random IOExceptions & disk
@@ -586,13 +602,13 @@ public class TestIndexWriterDelete exten
int docId = 12;
for (int i = 0; i < 13; i++) {
if (updates) {
- Document d = new Document();
- d.add(newStringField("id", Integer.toString(i), Field.Store.YES));
- d.add(newTextField("content", "bbb " + i, Field.Store.NO));
- d.add(new NumericDocValuesField("dv", i));
- modifier.updateDocument(new Term("id", Integer.toString(docId)), d);
+ Document2 d = modifier.newDocument();
+ d.addUniqueInt("id", docId);
+ d.addLargeText("content", "bbb " + i);
+ d.addInt("dv", i);
+ modifier.updateDocument(fieldTypes.newIntTerm("id", docId), d);
} else { // deletes
- modifier.deleteDocuments(new Term("id", Integer.toString(docId)));
+ modifier.deleteDocuments(fieldTypes.newIntTerm("id", docId));
// modifier.setNorm(docId, "contents", (float)2.0);
}
docId += 12;
@@ -779,14 +795,12 @@ public class TestIndexWriterDelete exten
dir.failOn(failure.reset());
- FieldType custom1 = new FieldType();
- custom1.setStored(true);
for (int i = 0; i < keywords.length; i++) {
- Document doc = new Document();
- doc.add(newStringField("id", keywords[i], Field.Store.YES));
- doc.add(newField("country", unindexed[i], custom1));
- doc.add(newTextField("contents", unstored[i], Field.Store.NO));
- doc.add(newTextField("city", text[i], Field.Store.YES));
+ Document2 doc = modifier.newDocument();
+ doc.addAtom("id", keywords[i]);
+ doc.addStored("country", unindexed[i]);
+ doc.addLargeText("contents", unstored[i]);
+ doc.addLargeText("city", text[i]);
modifier.addDocument(doc);
}
// flush (and commit if ac)
@@ -824,8 +838,7 @@ public class TestIndexWriterDelete exten
if (VERBOSE) {
System.out.println("TEST: add empty doc");
}
- Document doc = new Document();
- modifier.addDocument(doc);
+ modifier.addDocument(modifier.newDocument());
// commit the changes, the buffered deletes, and the new doc
@@ -902,14 +915,12 @@ public class TestIndexWriterDelete exten
modifier.commit();
dir.failOn(failure.reset());
- FieldType custom1 = new FieldType();
- custom1.setStored(true);
for (int i = 0; i < keywords.length; i++) {
- Document doc = new Document();
- doc.add(newStringField("id", keywords[i], Field.Store.YES));
- doc.add(newField("country", unindexed[i], custom1));
- doc.add(newTextField("contents", unstored[i], Field.Store.NO));
- doc.add(newTextField("city", text[i], Field.Store.YES));
+ Document2 doc = modifier.newDocument();
+ doc.addAtom("id", keywords[i]);
+ doc.addStored("country", unindexed[i]);
+ doc.addLargeText("contents", unstored[i]);
+ doc.addLargeText("city", text[i]);
try {
modifier.addDocument(doc);
} catch (IOException io) {
@@ -944,6 +955,7 @@ public class TestIndexWriterDelete exten
public void testDeleteAllSlowly() throws Exception {
final Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ FieldTypes fieldTypes = w.getFieldTypes();
final int NUM_DOCS = atLeast(1000);
final List<Integer> ids = new ArrayList<>(NUM_DOCS);
for(int id=0;id<NUM_DOCS;id++) {
@@ -951,8 +963,8 @@ public class TestIndexWriterDelete exten
}
Collections.shuffle(ids, random());
for(int id : ids) {
- Document doc = new Document();
- doc.add(newStringField("id", ""+id, Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addUniqueInt("id", id);
w.addDocument(doc);
}
Collections.shuffle(ids, random());
@@ -962,7 +974,7 @@ public class TestIndexWriterDelete exten
final int inc = Math.min(left, TestUtil.nextInt(random(), 1, 20));
final int limit = upto + inc;
while(upto < limit) {
- w.deleteDocuments(new Term("id", ""+ids.get(upto++)));
+ w.deleteDocuments(fieldTypes.newIntTerm("id", ids.get(upto++)));
}
final IndexReader r = w.getReader();
assertEquals(NUM_DOCS - upto, r.numDocs());
@@ -993,8 +1005,8 @@ public class TestIndexWriterDelete exten
.setRAMBufferSizeMB(1.0)
.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH));
- Document doc = new Document();
- doc.add(newTextField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20");
int num = atLeast(3);
for (int iter = 0; iter < num; iter++) {
int count = 0;
@@ -1038,19 +1050,20 @@ public class TestIndexWriterDelete exten
.setMaxBufferedDocs(1000)
.setMergePolicy(NoMergePolicy.INSTANCE)
.setReaderPooling(false));
+ FieldTypes fieldTypes = w.getFieldTypes();
int count = 0;
int startDelCount = w.getFlushDeletesCount();
while(true) {
- Document doc = new Document();
- doc.add(new StringField("id", count+"", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addUniqueInt("id", count);
final Term delTerm;
if (count == 1010) {
// This is the only delete that applies
- delTerm = new Term("id", ""+0);
+ delTerm = fieldTypes.newIntTerm("id", 0);
} else {
// These get buffered, taking up RAM, but delete
// nothing when applied:
- delTerm = new Term("id", "x" + count);
+ delTerm = fieldTypes.newIntTerm("id", 100000 + count);
}
w.updateDocument(delTerm, doc);
if (w.getFlushDeletesCount() > startDelCount) {
@@ -1062,7 +1075,7 @@ public class TestIndexWriterDelete exten
// sizable improvements to RAM efficiency of buffered
// del term we're unlikely to go over 100K:
if (count > 100000) {
- fail("delete's were not applied");
+ fail("deletes were not applied");
}
}
w.close();
@@ -1084,19 +1097,20 @@ public class TestIndexWriterDelete exten
.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.setMergePolicy(NoMergePolicy.INSTANCE)
.setReaderPooling(false));
+ FieldTypes fieldTypes = w.getFieldTypes();
int count = 0;
int startDelCount = w.getFlushDeletesCount();
while(true) {
- Document doc = new Document();
- doc.add(new StringField("id", count+"", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addUniqueInt("id", count);
final Term delTerm;
if (count == 1010) {
// This is the only delete that applies
- delTerm = new Term("id", ""+0);
+ delTerm = fieldTypes.newIntTerm("id", 0);
} else {
// These get buffered, taking up RAM, but delete
// nothing when applied:
- delTerm = new Term("id", "x" + count);
+ delTerm = fieldTypes.newIntTerm("id", 100000 + count);
}
w.updateDocument(delTerm, doc);
if (w.getFlushDeletesCount() > startDelCount) {
@@ -1134,6 +1148,9 @@ public class TestIndexWriterDelete exten
sawAfterFlush.set(true);
}
};
+
+ FieldTypes fieldTypes = w.getFieldTypes();
+
int id = 0;
while(true) {
StringBuilder sb = new StringBuilder();
@@ -1143,10 +1160,10 @@ public class TestIndexWriterDelete exten
if (id == 500) {
w.deleteDocuments(new Term("id", "0"));
}
- Document doc = new Document();
- doc.add(newStringField("id", ""+id, Field.Store.NO));
- doc.add(newTextField("body", sb.toString(), Field.Store.NO));
- w.updateDocument(new Term("id", ""+id), doc);
+ Document2 doc = w.newDocument();
+ doc.addUniqueInt("id", id);
+ doc.addLargeText("body", sb.toString());
+ w.updateDocument(fieldTypes.newIntTerm("id", id), doc);
docsInSegment.incrementAndGet();
// TODO: fix this test
if (slowFileExists(dir, "_0_1.del") || slowFileExists(dir, "_0_1.liv")) {
@@ -1169,12 +1186,12 @@ public class TestIndexWriterDelete exten
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
iwc.setMaxBufferedDocs(2);
IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(newField("field", "0", StringField.TYPE_NOT_STORED));
+ Document2 doc = w.newDocument();
+ doc.addAtom("field", "0");
w.addDocument(doc);
- doc = new Document();
- doc.add(newField("field", "1", StringField.TYPE_NOT_STORED));
+ doc = w.newDocument();
+ doc.addAtom("field", "1");
w.addDocument(doc);
w.commit();
assertEquals(1, w.getSegmentCount());
@@ -1216,7 +1233,7 @@ public class TestIndexWriterDelete exten
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
IndexWriter w = new IndexWriter(d, iwc);
- Document doc = new Document();
+ Document2 doc = w.newDocument();
w.addDocument(doc);
w.addDocument(doc);
w.addDocument(doc);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1640053&r1=1640052&r2=1640053&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Mon Nov 17 00:43:44 2014
@@ -38,9 +38,11 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -68,8 +70,8 @@ import org.apache.lucene.util.TestUtil;
public class TestIndexWriterExceptions extends LuceneTestCase {
- private static class DocCopyIterator implements Iterable<Document> {
- private final Document doc;
+ private static class DocCopyIterator implements Iterable<Document2> {
+ private final Document2 doc;
private final int count;
/* private field types */
@@ -101,14 +103,14 @@ public class TestIndexWriterExceptions e
custom5.setStoreTermVectorOffsets(true);
}
- public DocCopyIterator(Document doc, int count) {
+ public DocCopyIterator(Document2 doc, int count) {
this.count = count;
this.doc = doc;
}
@Override
- public Iterator<Document> iterator() {
- return new Iterator<Document>() {
+ public Iterator<Document2> iterator() {
+ return new Iterator<Document2>() {
int upto;
@Override
@@ -117,7 +119,7 @@ public class TestIndexWriterExceptions e
}
@Override
- public Document next() {
+ public Document2 next() {
upto++;
return doc;
}
@@ -144,28 +146,16 @@ public class TestIndexWriterExceptions e
@Override
public void run() {
-
- final Document doc = new Document();
-
- doc.add(newTextField(r, "content1", "aaa bbb ccc ddd", Field.Store.YES));
- doc.add(newField(r, "content6", "aaa bbb ccc ddd", DocCopyIterator.custom1));
- doc.add(newField(r, "content2", "aaa bbb ccc ddd", DocCopyIterator.custom2));
- doc.add(newField(r, "content3", "aaa bbb ccc ddd", DocCopyIterator.custom3));
-
- doc.add(newTextField(r, "content4", "aaa bbb ccc ddd", Field.Store.NO));
- doc.add(newStringField(r, "content5", "aaa bbb ccc ddd", Field.Store.NO));
- doc.add(new NumericDocValuesField("numericdv", 5));
- doc.add(new BinaryDocValuesField("binarydv", new BytesRef("hello")));
- doc.add(new SortedDocValuesField("sorteddv", new BytesRef("world")));
- doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("hellllo")));
- doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("again")));
- doc.add(new SortedNumericDocValuesField("sortednumericdv", 10));
- doc.add(new SortedNumericDocValuesField("sortednumericdv", 5));
-
- doc.add(newField(r, "content7", "aaa bbb ccc ddd", DocCopyIterator.custom4));
-
- final Field idField = newField(r, "id", "", DocCopyIterator.custom2);
- doc.add(idField);
+ FieldTypes fieldTypes = writer.getFieldTypes();
+ fieldTypes.setMultiValued("sortedsetdv");
+ fieldTypes.setMultiValued("sortednumericdv");
+
+ fieldTypes.enableTermVectors("content6");
+ fieldTypes.enableTermVectorPositions("content6");
+ fieldTypes.enableTermVectorOffsets("content6");
+ fieldTypes.enableTermVectors("content4");
+ fieldTypes.enableTermVectorPositions("content4");
+ fieldTypes.enableTermVectorOffsets("content4");
final long stopTime = System.currentTimeMillis() + 500;
@@ -173,9 +163,28 @@ public class TestIndexWriterExceptions e
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": TEST: IndexerThread: cycle");
}
- doFail.set(this);
+ final Document2 doc = writer.newDocument();
+ doc.addLargeText("content1", "aaa bbb ccc ddd");
+ doc.addLargeText("content6", "aaa bbb ccc ddd");
+ doc.addLargeText("content2", "aaa bbb ccc ddd");
+ doc.addStored("content3", "aaa bbb ccc ddd");
+
+ doc.addLargeText("content4", "aaa bbb ccc ddd");
+ doc.addAtom("content5", "aaa bbb ccc ddd");
+ doc.addInt("numericdv", 5);
+ doc.addBinary("binarydv", new BytesRef("hello"));
+ doc.addShortText("sorteddv", "world");
+ doc.addShortText("sortedsetdv", "hellllo");
+ doc.addShortText("sortedsetdv", "again");
+ doc.addInt("sortednumericdv", 10);
+ doc.addInt("sortednumericdv", 5);
+
+ doc.addAtom("content7", "aaa bbb ccc ddd");
final String id = ""+r.nextInt(50);
- idField.setStringValue(id);
+
+ doc.addLargeText("id", id);
+
+ doFail.set(this);
Term idTerm = new Term("id", id);
try {
if (r.nextBoolean()) {
@@ -372,8 +381,8 @@ public class TestIndexWriterExceptions e
Directory dir = newDirectory();
TestPoint2 testPoint = new TestPoint2();
IndexWriter w = RandomIndexWriter.mockIndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())), testPoint);
- Document doc = new Document();
- doc.add(newTextField("field", "a field", Field.Store.YES));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("field", "a field");
w.addDocument(doc);
testPoint.doFail = true;
try {
@@ -389,14 +398,6 @@ public class TestIndexWriterExceptions e
// LUCENE-1208
public void testExceptionJustBeforeFlush() throws IOException {
Directory dir = newDirectory();
- IndexWriter w = RandomIndexWriter.mockIndexWriter(dir,
- newIndexWriterConfig(new MockAnalyzer(random()))
- .setMaxBufferedDocs(2),
- new TestPoint1());
- Document doc = new Document();
- doc.add(newTextField("field", "a field", Field.Store.YES));
- w.addDocument(doc);
-
Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
public TokenStreamComponents createComponents(String fieldName) {
@@ -406,10 +407,18 @@ public class TestIndexWriterExceptions e
}
};
- Document crashDoc = new Document();
- crashDoc.add(newTextField("crash", "do it on token 4", Field.Store.YES));
+ IndexWriter w = RandomIndexWriter.mockIndexWriter(dir,
+ newIndexWriterConfig(analyzer)
+ .setMaxBufferedDocs(2),
+ new TestPoint1());
+ Document2 doc = w.newDocument();
+ doc.addLargeText("field", "a field");
+ w.addDocument(doc);
+
+ Document2 crashDoc = w.newDocument();
+ crashDoc.addLargeText("crash", "do it on token 4");
try {
- w.addDocument(crashDoc, analyzer);
+ w.addDocument(crashDoc);
fail("did not hit expected exception");
} catch (IOException ioe) {
// expected
@@ -445,8 +454,8 @@ public class TestIndexWriterExceptions e
TestPoint3 testPoint = new TestPoint3();
IndexWriter w = RandomIndexWriter.mockIndexWriter(dir, conf, testPoint);
testPoint.doFail = true;
- Document doc = new Document();
- doc.add(newTextField("field", "a field", Field.Store.YES));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("field", "a field");
for(int i=0;i<10;i++)
try {
w.addDocument(doc);
@@ -493,9 +502,9 @@ public class TestIndexWriterExceptions e
IndexWriter writer = new IndexWriter(dir, conf);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
String contents = "aa bb cc dd ee ff gg hh ii jj kk";
- doc.add(newTextField("content", contents, Field.Store.NO));
+ doc.addLargeText("content", contents);
try {
writer.addDocument(doc);
fail("did not hit expected exception");
@@ -503,13 +512,13 @@ public class TestIndexWriterExceptions e
}
// Make sure we can add another normal document
- doc = new Document();
- doc.add(newTextField("content", "aa bb cc dd", Field.Store.NO));
+ doc = writer.newDocument();
+ doc.addLargeText("content", "aa bb cc dd");
writer.addDocument(doc);
// Make sure we can add another normal document
- doc = new Document();
- doc.add(newTextField("content", "aa bb cc dd", Field.Store.NO));
+ doc = writer.newDocument();
+ doc.addLargeText("content", "aa bb cc dd");
writer.addDocument(doc);
writer.close();
@@ -583,9 +592,9 @@ public class TestIndexWriterExceptions e
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(2));
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
String contents = "aa bb cc dd ee ff gg hh ii jj kk";
- doc.add(newTextField("content", contents, Field.Store.NO));
+ doc.addLargeText("content", contents);
boolean hitError = false;
for(int i=0;i<200;i++) {
try {
@@ -833,11 +842,10 @@ public class TestIndexWriterExceptions e
// TODO: these are also in TestIndexWriter... add a simple doc-writing method
// like this to LuceneTestCase?
- private void addDoc(IndexWriter writer) throws IOException
- {
- Document doc = new Document();
- doc.add(newTextField("content", "aaa", Field.Store.NO));
- writer.addDocument(doc);
+ private void addDoc(IndexWriter writer) throws IOException {
+ Document2 doc = writer.newDocument();
+ doc.addLargeText("content", "aaa");
+ writer.addDocument(doc);
}
// LUCENE-1044: test exception during sync
@@ -938,8 +946,8 @@ public class TestIndexWriterExceptions e
dir.setFailOnCreateOutput(false);
dir.setEnableVirusScanner(false); // we check for specific list of files
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- Document doc = new Document();
- doc.add(newTextField("field", "a field", Field.Store.YES));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("field", "a field");
w.addDocument(doc);
dir.failOn(failure);
try {
@@ -1053,7 +1061,7 @@ public class TestIndexWriterExceptions e
public void close() {}
}));
doFail.set(true);
- writer.addDocument(new Document());
+ writer.addDocument(writer.newDocument());
try {
writer.commit();
@@ -1068,7 +1076,7 @@ public class TestIndexWriterExceptions e
}
try {
- writer.addDocument(new Document());
+ writer.addDocument(writer.newDocument());
} catch (AlreadyClosedException ace) {
// expected
}
@@ -1269,16 +1277,16 @@ public class TestIndexWriterExceptions e
for (FailOnTermVectors failure : failures) {
MockDirectoryWrapper dir = newMockDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ FieldTypes fieldTypes = w.getFieldTypes();
dir.failOn(failure);
int numDocs = 10 + random().nextInt(30);
for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
- Field field = newTextField(random(), "field", "a field", Field.Store.YES);
- doc.add(field);
+ Document2 doc = w.newDocument();
+ doc.addLargeText("field", "a field");
// random TV
try {
w.addDocument(doc);
- assertFalse(field.fieldType().storeTermVectors());
+ assertFalse(fieldTypes.getTermVectors("field"));
} catch (RuntimeException e) {
assertTrue(e.getMessage().startsWith(FailOnTermVectors.EXC_MSG));
}
@@ -1288,18 +1296,17 @@ public class TestIndexWriterExceptions e
}
}
- Document document = new Document();
- document.add(new TextField("field", "a field", Field.Store.YES));
+ Document2 document = w.newDocument();
+ document.addLargeText("field", "a field");
w.addDocument(document);
for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
- Field field = newTextField(random(), "field", "a field", Field.Store.YES);
- doc.add(field);
+ Document2 doc = w.newDocument();
+ doc.addLargeText("field", "a field");
// random TV
try {
w.addDocument(doc);
- assertFalse(field.fieldType().storeTermVectors());
+ assertFalse(fieldTypes.getTermVectors("field"));
} catch (RuntimeException e) {
assertTrue(e.getMessage().startsWith(FailOnTermVectors.EXC_MSG));
}
@@ -1308,8 +1315,8 @@ public class TestIndexWriterExceptions e
TestUtil.checkIndex(dir);
}
}
- document = new Document();
- document.add(new TextField("field", "a field", Field.Store.YES));
+ document = w.newDocument();
+ document.addLargeText("field", "a field");
w.addDocument(document);
w.close();
IndexReader reader = DirectoryReader.open(dir);
@@ -1358,24 +1365,22 @@ public class TestIndexWriterExceptions e
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
final int numDocs1 = random().nextInt(25);
for(int docCount=0;docCount<numDocs1;docCount++) {
- Document doc = new Document();
- doc.add(newTextField("content", "good content", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("content", "good content");
w.addDocument(doc);
}
- final List<Document> docs = new ArrayList<>();
+ final List<Document2> docs = new ArrayList<>();
for(int docCount=0;docCount<7;docCount++) {
- Document doc = new Document();
+ Document2 doc = w.newDocument();
docs.add(doc);
- doc.add(newStringField("id", docCount+"", Field.Store.NO));
- doc.add(newTextField("content", "silly content " + docCount, Field.Store.NO));
+ doc.addAtom("id", docCount+"");
+ doc.addLargeText("content", "silly content " + docCount);
if (docCount == 4) {
- Field f = newTextField("crash", "", Field.Store.NO);
- doc.add(f);
MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenizer.setReader(new StringReader("crash me on the 4th token"));
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- f.setTokenStream(new CrashingFilter("crash", tokenizer));
+ doc.addLargeText("crash", new CrashingFilter("crash", tokenizer));
}
}
try {
@@ -1389,8 +1394,8 @@ public class TestIndexWriterExceptions e
final int numDocs2 = random().nextInt(25);
for(int docCount=0;docCount<numDocs2;docCount++) {
- Document doc = new Document();
- doc.add(newTextField("content", "good content", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("content", "good content");
w.addDocument(doc);
}
@@ -1417,27 +1422,27 @@ public class TestIndexWriterExceptions e
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
final int numDocs1 = random().nextInt(25);
for(int docCount=0;docCount<numDocs1;docCount++) {
- Document doc = new Document();
- doc.add(newTextField("content", "good content", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("content", "good content");
w.addDocument(doc);
}
// Use addDocs (no exception) to get docs in the index:
- final List<Document> docs = new ArrayList<>();
+ final List<Document2> docs = new ArrayList<>();
final int numDocs2 = random().nextInt(25);
for(int docCount=0;docCount<numDocs2;docCount++) {
- Document doc = new Document();
+ Document2 doc = w.newDocument();
docs.add(doc);
- doc.add(newStringField("subid", "subs", Field.Store.NO));
- doc.add(newStringField("id", docCount+"", Field.Store.NO));
- doc.add(newTextField("content", "silly content " + docCount, Field.Store.NO));
+ doc.addAtom("subid", "subs");
+ doc.addAtom("id", docCount+"");
+ doc.addLargeText("content", "silly content " + docCount);
}
w.addDocuments(docs);
final int numDocs3 = random().nextInt(25);
for(int docCount=0;docCount<numDocs3;docCount++) {
- Document doc = new Document();
- doc.add(newTextField("content", "good content", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("content", "good content");
w.addDocument(doc);
}
@@ -1445,17 +1450,15 @@ public class TestIndexWriterExceptions e
final int limit = TestUtil.nextInt(random(), 2, 25);
final int crashAt = random().nextInt(limit);
for(int docCount=0;docCount<limit;docCount++) {
- Document doc = new Document();
+ Document2 doc = w.newDocument();
docs.add(doc);
- doc.add(newStringField("id", docCount+"", Field.Store.NO));
- doc.add(newTextField("content", "silly content " + docCount, Field.Store.NO));
+ doc.addAtom("id", docCount+"");
+ doc.addLargeText("content", "silly content " + docCount);
if (docCount == crashAt) {
- Field f = newTextField("crash", "", Field.Store.NO);
- doc.add(f);
MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenizer.setReader(new StringReader("crash me on the 4th token"));
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- f.setTokenStream(new CrashingFilter("crash", tokenizer));
+ doc.addLargeText("crash", new CrashingFilter("crash", tokenizer));
}
}
@@ -1470,8 +1473,8 @@ public class TestIndexWriterExceptions e
final int numDocs4 = random().nextInt(25);
for(int docCount=0;docCount<numDocs4;docCount++) {
- Document doc = new Document();
- doc.add(newTextField("content", "good content", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("content", "good content");
w.addDocument(doc);
}
@@ -1498,39 +1501,16 @@ public class TestIndexWriterExceptions e
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
// add good document
- Document doc = new Document();
- iw.addDocument(doc);
+ iw.addDocument(iw.newDocument());
+
+ Document2 doc = iw.newDocument();
+ String value = null;
try {
// set to null value
- String value = null;
- doc.add(new StoredField("foo", value));
- iw.addDocument(doc);
+ doc.addStored("foo", value);
fail("didn't get expected exception");
- } catch (IllegalArgumentException expected) {}
- iw.close();
- // make sure we see our good doc
- DirectoryReader r = DirectoryReader.open(dir);
- assertEquals(1, r.numDocs());
- r.close();
- dir.close();
- }
-
- /** test a null string value doesn't abort the entire segment */
- public void testNullStoredFieldReuse() throws Exception {
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random());
- IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
- // add good document
- Document doc = new Document();
- Field theField = new StoredField("foo", "hello", StoredField.TYPE);
- doc.add(theField);
- iw.addDocument(doc);
- try {
- // set to null value
- theField.setStringValue(null);
- iw.addDocument(doc);
- fail("didn't get expected exception");
- } catch (IllegalArgumentException expected) {}
+ } catch (IllegalArgumentException expected) {
+ }
iw.close();
// make sure we see our good doc
DirectoryReader r = DirectoryReader.open(dir);
@@ -1545,42 +1525,16 @@ public class TestIndexWriterExceptions e
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
// add good document
- Document doc = new Document();
- iw.addDocument(doc);
+ iw.addDocument(iw.newDocument());
+ Document2 doc = iw.newDocument();
+ byte v[] = null;
try {
// set to null value
- byte v[] = null;
- Field theField = new StoredField("foo", v);
- doc.add(theField);
- iw.addDocument(doc);
- fail("didn't get expected exception");
- } catch (NullPointerException expected) {}
- iw.close();
- // make sure we see our good doc
- DirectoryReader r = DirectoryReader.open(dir);
- assertEquals(1, r.numDocs());
- r.close();
- dir.close();
- }
-
- /** test a null byte[] value doesn't abort the entire segment */
- public void testNullStoredBytesFieldReuse() throws Exception {
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random());
- IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
- // add good document
- Document doc = new Document();
- Field theField = new StoredField("foo", new BytesRef("hello").bytes);
- doc.add(theField);
- iw.addDocument(doc);
- try {
- // set to null value
- byte v[] = null;
- theField.setBytesValue(v);
- iw.addDocument(doc);
+ doc.addStored("foo", v);
fail("didn't get expected exception");
- } catch (NullPointerException expected) {}
+ } catch (NullPointerException expected) {
+ }
iw.close();
// make sure we see our good doc
DirectoryReader r = DirectoryReader.open(dir);
@@ -1595,42 +1549,17 @@ public class TestIndexWriterExceptions e
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
// add good document
- Document doc = new Document();
- iw.addDocument(doc);
+ iw.addDocument(iw.newDocument());
+ BytesRef v = null;
+ Document2 doc = iw.newDocument();
try {
// set to null value
- BytesRef v = null;
- Field theField = new StoredField("foo", v);
- doc.add(theField);
+ doc.addStored("foo", v);
iw.addDocument(doc);
fail("didn't get expected exception");
- } catch (IllegalArgumentException expected) {}
- iw.close();
- // make sure we see our good doc
- DirectoryReader r = DirectoryReader.open(dir);
- assertEquals(1, r.numDocs());
- r.close();
- dir.close();
- }
-
- /** test a null bytesref value doesn't abort the entire segment */
- public void testNullStoredBytesRefFieldReuse() throws Exception {
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random());
- IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
- // add good document
- Document doc = new Document();
- Field theField = new StoredField("foo", new BytesRef("hello"));
- doc.add(theField);
- iw.addDocument(doc);
- try {
- // set to null value
- BytesRef v = null;
- theField.setBytesValue(v);
- iw.addDocument(doc);
- fail("didn't get expected exception");
- } catch (IllegalArgumentException expected) {}
+ } catch (IllegalArgumentException expected) {
+ }
iw.close();
// make sure we see our good doc
DirectoryReader r = DirectoryReader.open(dir);
@@ -1654,11 +1583,12 @@ public class TestIndexWriterExceptions e
};
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
// add good document
- Document doc = new Document();
+ Document2 doc = iw.newDocument();
iw.addDocument(doc);
+
+ doc.addLargeText("foo", "bar");
+ doc.addLargeText("foo", "bar");
try {
- doc.add(newTextField("foo", "bar", Field.Store.NO));
- doc.add(newTextField("foo", "bar", Field.Store.NO));
iw.addDocument(doc);
fail("didn't get expected exception");
} catch (IllegalArgumentException expected) {}
@@ -1694,7 +1624,7 @@ public class TestIndexWriterExceptions e
UOEDirectory uoe = new UOEDirectory();
Directory d = new MockDirectoryWrapper(random(), uoe);
IndexWriter iw = new IndexWriter(d, newIndexWriterConfig(null));
- iw.addDocument(new Document());
+ iw.addDocument(iw.newDocument());
iw.close();
uoe.doFail = true;
try {
@@ -1710,7 +1640,7 @@ public class TestIndexWriterExceptions e
public void testIllegalPositions() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
- Document doc = new Document();
+ Document2 doc = iw.newDocument();
Token t1 = new Token("foo", 0, 3);
t1.setPositionIncrement(Integer.MAX_VALUE);
Token t2 = new Token("bar", 4, 7);
@@ -1718,8 +1648,7 @@ public class TestIndexWriterExceptions e
TokenStream overflowingTokenStream = new CannedTokenStream(
new Token[] { t1, t2 }
);
- Field field = new TextField("foo", overflowingTokenStream);
- doc.add(field);
+ doc.addLargeText("foo", overflowingTokenStream);
try {
iw.addDocument(doc);
fail();
@@ -1733,7 +1662,7 @@ public class TestIndexWriterExceptions e
public void testLegalbutVeryLargePositions() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
- Document doc = new Document();
+ Document2 doc = iw.newDocument();
Token t1 = new Token("foo", 0, 3);
t1.setPositionIncrement(Integer.MAX_VALUE-500);
if (random().nextBoolean()) {
@@ -1742,8 +1671,7 @@ public class TestIndexWriterExceptions e
TokenStream overflowingTokenStream = new CannedTokenStream(
new Token[] { t1 }
);
- Field field = new TextField("foo", overflowingTokenStream);
- doc.add(field);
+ doc.addLargeText("foo", overflowingTokenStream);
iw.addDocument(doc);
iw.close();
dir.close();
@@ -1754,10 +1682,10 @@ public class TestIndexWriterExceptions e
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
iwc.setMergePolicy(newLogMergePolicy());
IndexWriter iw = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new StringField("field1", "sometext", Field.Store.YES));
- doc.add(new TextField("field2", "sometext", Field.Store.NO));
- doc.add(new StringField("foo", "bar", Field.Store.NO));
+ Document2 doc = iw.newDocument();
+ doc.addAtom("field1", "sometext");
+ doc.addLargeText("field2", "sometext");
+ doc.addAtom("foo", "bar");
iw.addDocument(doc); // add an 'ok' document
try {
// try to boost with norms omitted
@@ -1862,8 +1790,8 @@ public class TestIndexWriterExceptions e
// Create an index with one document
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
IndexWriter iw = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new StringField("foo", "bar", Field.Store.NO));
+ Document2 doc = iw.newDocument();
+ doc.addAtom("foo", "bar");
iw.addDocument(doc); // add a document
iw.commit();
DirectoryReader ir = DirectoryReader.open(dir);
@@ -1978,14 +1906,17 @@ public class TestIndexWriterExceptions e
// forceMerge can easily return when there are still
// too many segments in the index:
w.setDoRandomForceMergeAssert(false);
+ FieldTypes fieldTypes = w.getFieldTypes();
+ fieldTypes.disableSorting("bf");
+ fieldTypes.disableSorting("bcf");
}
for(int i=0;i<numDocs;i++) {
- Document doc = new Document();
- doc.add(new StringField("id", ""+(docBase+i), Field.Store.NO));
- doc.add(new NumericDocValuesField("f", 1L));
- doc.add(new NumericDocValuesField("cf", 2L));
- doc.add(new BinaryDocValuesField("bf", TestBinaryDocValuesUpdates.toBytes(1L)));
- doc.add(new BinaryDocValuesField("bcf", TestBinaryDocValuesUpdates.toBytes(2L)));
+ Document2 doc = w.newDocument();
+ doc.addAtom("id", ""+(docBase+i));
+ doc.addLong("f", 1L);
+ doc.addLong("cf", 2L);
+ doc.addBinary("bf", TestBinaryDocValuesUpdates.toBytes(1L));
+ doc.addBinary("bcf", TestBinaryDocValuesUpdates.toBytes(2L));
w.addDocument(doc);
}
docCount += numDocs;
@@ -2009,18 +1940,21 @@ public class TestIndexWriterExceptions e
System.out.println(" update id=" + docid + " to value " + value);
}
Term idTerm = new Term("id", Integer.toString(docid));
+ Document2 update = w.newDocument();
+ update.disableExistsField();
if (random().nextBoolean()) { // update only numeric field
- w.updateDocValues(idTerm, new NumericDocValuesField("f", value), new NumericDocValuesField("cf", value*2));
+ update.addLong("f", value);
+ update.addLong("cf", value*2);
} else if (random().nextBoolean()) {
- w.updateDocValues(idTerm, new BinaryDocValuesField("bf", TestBinaryDocValuesUpdates.toBytes(value)),
- new BinaryDocValuesField("bcf", TestBinaryDocValuesUpdates.toBytes(value*2)));
+ update.addBinary("bf", TestBinaryDocValuesUpdates.toBytes(value));
+ update.addBinary("bcf", TestBinaryDocValuesUpdates.toBytes(value*2));
} else {
- w.updateDocValues(idTerm,
- new NumericDocValuesField("f", value),
- new NumericDocValuesField("cf", value*2),
- new BinaryDocValuesField("bf", TestBinaryDocValuesUpdates.toBytes(value)),
- new BinaryDocValuesField("bcf", TestBinaryDocValuesUpdates.toBytes(value*2)));
+ update.addLong("f", value);
+ update.addLong("cf", value*2);
+ update.addBinary("bf", TestBinaryDocValuesUpdates.toBytes(value));
+ update.addBinary("bcf", TestBinaryDocValuesUpdates.toBytes(value*2));
}
+ w.updateDocValues(idTerm, update);
}
// sometimes do both deletes and updates
@@ -2142,10 +2076,10 @@ public class TestIndexWriterExceptions e
public void testTooManyTokens() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
- Document doc = new Document();
- FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
- ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
- doc.add(new Field("foo", new TokenStream() {
+ FieldTypes fieldTypes = iw.getFieldTypes();
+ fieldTypes.setIndexOptions("foo", IndexOptions.DOCS_AND_FREQS);
+ Document2 doc = iw.newDocument();
+ doc.addLargeText("foo", new TokenStream() {
CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
long num = 0;
@@ -2168,7 +2102,7 @@ public class TestIndexWriterExceptions e
}
return true;
}
- }, ft));
+ });
try {
iw.addDocument(doc);
fail("didn't hit exception");
@@ -2206,7 +2140,7 @@ public class TestIndexWriterExceptions e
IndexWriterConfig iwc = new IndexWriterConfig(null);
iwc.setInfoStream(evilInfoStream);
IndexWriter iw = new IndexWriter(dir, iwc);
- Document doc = new Document();
+ Document2 doc = iw.newDocument();
for (int i = 0; i < 10; i++) {
iw.addDocument(doc);
}
@@ -2274,7 +2208,7 @@ public class TestIndexWriterExceptions e
IndexWriterConfig iwc = new IndexWriterConfig(null);
IndexWriter iw = new IndexWriter(dir, iwc);
- Document doc = new Document();
+ Document2 doc = iw.newDocument();
for (int i = 0; i < 10; i++) {
iw.addDocument(doc);
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java?rev=1640053&r1=1640052&r2=1640053&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java Mon Nov 17 00:43:44 2014
@@ -31,9 +31,11 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.asserting.AssertingCodec;
import org.apache.lucene.codecs.cranky.CrankyCodec;
import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -45,8 +47,8 @@ import org.apache.lucene.store.MockDirec
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Rethrow;
+import org.apache.lucene.util.TestUtil;
/**
* Causes a bunch of non-aborting and aborting exceptions and checks that
@@ -98,28 +100,32 @@ public class TestIndexWriterExceptions2
int numDocs = atLeast(2000);
IndexWriter iw = new IndexWriter(dir, conf);
+ FieldTypes fieldTypes = iw.getFieldTypes();
+ fieldTypes.enableTermVectors("text_vectors");
+ fieldTypes.disableSorting("dv2");
+ fieldTypes.setMultiValued("dv4");
+ fieldTypes.setMultiValued("dv5");
+
try {
for (int i = 0; i < numDocs; i++) {
// TODO: add crankyDocValuesFields, etc
- Document doc = new Document();
- doc.add(newStringField("id", Integer.toString(i), Field.Store.NO));
- doc.add(new NumericDocValuesField("dv", i));
- doc.add(new BinaryDocValuesField("dv2", new BytesRef(Integer.toString(i))));
- doc.add(new SortedDocValuesField("dv3", new BytesRef(Integer.toString(i))));
- doc.add(new SortedSetDocValuesField("dv4", new BytesRef(Integer.toString(i))));
- doc.add(new SortedSetDocValuesField("dv4", new BytesRef(Integer.toString(i-1))));
- doc.add(new SortedNumericDocValuesField("dv5", i));
- doc.add(new SortedNumericDocValuesField("dv5", i-1));
- doc.add(newTextField("text1", TestUtil.randomAnalysisString(random(), 20, true), Field.Store.NO));
+ Document2 doc = iw.newDocument();
+ doc.addAtom("id", Integer.toString(i));
+ doc.addInt("dv", i);
+ doc.addBinary("dv2", new BytesRef(Integer.toString(i)));
+ doc.addShortText("dv3", Integer.toString(i));
+ doc.addShortText("dv4", Integer.toString(i));
+ doc.addShortText("dv4", Integer.toString(i-1));
+ doc.addInt("dv5", i);
+ doc.addInt("dv5", i-1);
+ doc.addLargeText("text1", TestUtil.randomAnalysisString(random(), 20, true));
// ensure we store something
- doc.add(new StoredField("stored1", "foo"));
- doc.add(new StoredField("stored1", "bar"));
+ doc.addStored("stored1", "foo");
+ doc.addStored("stored1", "bar");
// ensure we get some payloads
- doc.add(newTextField("text_payloads", TestUtil.randomAnalysisString(random(), 6, true), Field.Store.NO));
+ doc.addLargeText("text_payloads", TestUtil.randomAnalysisString(random(), 6, true));
// ensure we get some vectors
- FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
- ft.setStoreTermVectors(true);
- doc.add(newField("text_vectors", TestUtil.randomAnalysisString(random(), 6, true), ft));
+ doc.addLargeText("text_vectors", TestUtil.randomAnalysisString(random(), 6, true));
if (random().nextInt(10) > 0) {
// single doc
@@ -144,12 +150,12 @@ public class TestIndexWriterExceptions2
}
} else {
// block docs
- Document doc2 = new Document();
- doc2.add(newStringField("id", Integer.toString(-i), Field.Store.NO));
- doc2.add(newTextField("text1", TestUtil.randomAnalysisString(random(), 20, true), Field.Store.NO));
- doc2.add(new StoredField("stored1", "foo"));
- doc2.add(new StoredField("stored1", "bar"));
- doc2.add(newField("text_vectors", TestUtil.randomAnalysisString(random(), 6, true), ft));
+ Document2 doc2 = iw.newDocument();
+ doc2.addAtom("id", Integer.toString(-i));
+ doc2.addLargeText("text1", TestUtil.randomAnalysisString(random(), 20, true));
+ doc2.addStored("stored1", "foo");
+ doc2.addStored("stored1", "bar");
+ doc2.addLargeText("text_vectors", TestUtil.randomAnalysisString(random(), 6, true));
try {
iw.addDocuments(Arrays.asList(doc, doc2));
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java?rev=1640053&r1=1640052&r2=1640053&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java Mon Nov 17 00:43:44 2014
@@ -20,8 +20,10 @@ package org.apache.lucene.index;
import java.util.Arrays;
import java.util.Collections;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
@@ -43,9 +45,9 @@ public class TestIndexWriterMaxDocs exte
public void testExactlyAtTrueLimit() throws Exception {
Directory dir = newFSDirectory(createTempDir("2BDocs3"));
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null));
- Document doc = new Document();
- doc.add(newStringField("field", "text", Field.Store.NO));
for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
+ Document2 doc = iw.newDocument();
+ doc.addAtom("field", "text");
iw.addDocument(doc);
/*
if (i%1000000 == 0) {
@@ -84,12 +86,12 @@ public class TestIndexWriterMaxDocs exte
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
for(int i=0;i<10;i++) {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
}
// 11th document should fail:
try {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
fail("didn't hit exception");
} catch (IllegalStateException ise) {
// expected
@@ -107,12 +109,12 @@ public class TestIndexWriterMaxDocs exte
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
for(int i=0;i<10;i++) {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
}
// 11th document should fail:
try {
- w.addDocuments(Collections.singletonList(new Document()));
+ w.addDocuments(Collections.singletonList(w.newDocument()));
fail("didn't hit exception");
} catch (IllegalStateException ise) {
// expected
@@ -130,12 +132,12 @@ public class TestIndexWriterMaxDocs exte
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
for(int i=0;i<10;i++) {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
}
// 11th document should fail:
try {
- w.updateDocument(new Term("field", "foo"), new Document());
+ w.updateDocument(new Term("field", "foo"), w.newDocument());
fail("didn't hit exception");
} catch (IllegalStateException ise) {
// expected
@@ -153,12 +155,12 @@ public class TestIndexWriterMaxDocs exte
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
for(int i=0;i<10;i++) {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
}
// 11th document should fail:
try {
- w.updateDocuments(new Term("field", "foo"), Collections.singletonList(new Document()));
+ w.updateDocuments(new Term("field", "foo"), Collections.singletonList(w.newDocument()));
fail("didn't hit exception");
} catch (IllegalStateException ise) {
// expected
@@ -175,15 +177,16 @@ public class TestIndexWriterMaxDocs exte
try {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
+ FieldTypes fieldTypes = w.getFieldTypes();
for(int i=0;i<10;i++) {
- Document doc = new Document();
- doc.add(newStringField("id", ""+i, Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addUniqueInt("id", i);
w.addDocument(doc);
}
// Delete 5 of them:
for(int i=0;i<5;i++) {
- w.deleteDocuments(new Term("id", ""+i));
+ w.deleteDocuments(fieldTypes.newIntTerm("id", i));
}
w.forceMerge(1);
@@ -192,12 +195,12 @@ public class TestIndexWriterMaxDocs exte
// Add 5 more docs
for(int i=0;i<5;i++) {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
}
// 11th document should fail:
try {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
fail("didn't hit exception");
} catch (IllegalStateException ise) {
// expected
@@ -217,9 +220,11 @@ public class TestIndexWriterMaxDocs exte
IndexWriterConfig iwc = new IndexWriterConfig(null);
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
IndexWriter w = new IndexWriter(dir, iwc);
+ FieldTypes fieldTypes = w.getFieldTypes();
+
for(int i=0;i<10;i++) {
- Document doc = new Document();
- doc.add(newStringField("id", ""+i, Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addUniqueInt("id", i);
w.addDocument(doc);
if (i % 2 == 0) {
// Make a new segment every 2 docs:
@@ -229,7 +234,7 @@ public class TestIndexWriterMaxDocs exte
// Delete 5 of them:
for(int i=0;i<5;i++) {
- w.deleteDocuments(new Term("id", ""+i));
+ w.deleteDocuments(fieldTypes.newIntTerm("id", i));
}
w.forceMerge(1);
@@ -238,12 +243,12 @@ public class TestIndexWriterMaxDocs exte
// Add 5 more docs
for(int i=0;i<5;i++) {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
}
// 11th document should fail:
try {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
fail("didn't hit exception");
} catch (IllegalStateException ise) {
// expected
@@ -261,13 +266,13 @@ public class TestIndexWriterMaxDocs exte
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
for(int i=0;i<10;i++) {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
}
w.close();
Directory dir2 = newDirectory();
IndexWriter w2 = new IndexWriter(dir2, new IndexWriterConfig(null));
- w2.addDocument(new Document());
+ w2.addDocument(w2.newDocument());
try {
w2.addIndexes(new Directory[] {dir});
fail("didn't hit exception");
@@ -294,8 +299,8 @@ public class TestIndexWriterMaxDocs exte
// Make sure MultiReader lets you search exactly the limit number of docs:
public void testMultiReaderExactLimit() throws Exception {
Directory dir = newDirectory();
- Document doc = new Document();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
+ Document2 doc = w.newDocument();
for (int i = 0; i < 100000; i++) {
w.addDocument(doc);
}
@@ -304,6 +309,7 @@ public class TestIndexWriterMaxDocs exte
int remainder = IndexWriter.MAX_DOCS % 100000;
Directory dir2 = newDirectory();
w = new IndexWriter(dir2, new IndexWriterConfig(null));
+ doc = w.newDocument();
for (int i = 0; i < remainder; i++) {
w.addDocument(doc);
}
@@ -329,8 +335,8 @@ public class TestIndexWriterMaxDocs exte
// Make sure MultiReader is upset if you exceed the limit
public void testMultiReaderBeyondLimit() throws Exception {
Directory dir = newDirectory();
- Document doc = new Document();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
+ Document2 doc = w.newDocument();
for (int i = 0; i < 100000; i++) {
w.addDocument(doc);
}
@@ -343,6 +349,7 @@ public class TestIndexWriterMaxDocs exte
Directory dir2 = newDirectory();
w = new IndexWriter(dir2, new IndexWriterConfig(null));
+ doc = w.newDocument();
for (int i = 0; i < remainder; i++) {
w.addDocument(doc);
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java?rev=1640053&r1=1640052&r2=1640053&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java Mon Nov 17 00:43:44 2014
@@ -20,11 +20,11 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
-
import org.apache.lucene.util.LuceneTestCase;
public class TestIndexWriterMergePolicy extends LuceneTestCase {
@@ -228,8 +228,8 @@ public class TestIndexWriterMergePolicy
}
private void addDoc(IndexWriter writer) throws IOException {
- Document doc = new Document();
- doc.add(newTextField("content", "aaa", Field.Store.NO));
+ Document2 doc = writer.newDocument();
+ doc.addLargeText("content", "aaa");
writer.addDocument(doc);
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java?rev=1640053&r1=1640052&r2=1640053&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java Mon Nov 17 00:43:44 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.AlreadyClosedException;
@@ -47,15 +48,13 @@ public class TestIndexWriterMerging exte
fillIndex(random(), indexA, 0, num);
boolean fail = verifyIndex(indexA, 0);
- if (fail)
- {
+ if (fail) {
fail("Index a is invalid");
}
fillIndex(random(), indexB, num, num);
fail = verifyIndex(indexB, num);
- if (fail)
- {
+ if (fail) {
fail("Index b is invalid");
}
@@ -89,10 +88,9 @@ public class TestIndexWriterMerging exte
Document2 temp = reader.document(i);
//System.out.println("doc "+i+"="+temp.getField("count").stringValue());
//compare the index doc number to the value that it should be
- if (!temp.getField("count").stringValue().equals((i + startAt) + ""))
- {
+ if (temp.getInt("count") != i + startAt) {
fail = true;
- System.out.println("Document " + (i + startAt) + " is returning document " + temp.getField("count").stringValue());
+ System.out.println("Document " + (i + startAt) + " is returning document " + temp.getInt("count"));
}
}
reader.close();
@@ -111,9 +109,8 @@ public class TestIndexWriterMerging exte
for (int i = start; i < (start + numDocs); i++)
{
- Document temp = new Document();
- temp.add(newStringField("count", (""+i), Field.Store.YES));
-
+ Document2 temp = writer.newDocument();
+ temp.addUniqueInt("count", i);
writer.addDocument(temp);
}
writer.close();
@@ -126,26 +123,18 @@ public class TestIndexWriterMerging exte
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(2)
.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH));
- Document document = new Document();
- FieldType customType = new FieldType();
- customType.setStored(true);
+ FieldTypes fieldTypes = writer.getFieldTypes();
+ fieldTypes.enableTermVectors("termVector");
+ fieldTypes.enableTermVectorOffsets("termVector");
+ fieldTypes.enableTermVectorPositions("termVector");
- FieldType customType1 = new FieldType(TextField.TYPE_STORED);
- customType1.setTokenized(false);
- customType1.setStoreTermVectors(true);
- customType1.setStoreTermVectorPositions(true);
- customType1.setStoreTermVectorOffsets(true);
-
- Field idField = newStringField("id", "", Field.Store.NO);
- document.add(idField);
- Field storedField = newField("stored", "stored", customType);
- document.add(storedField);
- Field termVectorField = newField("termVector", "termVector", customType1);
- document.add(termVectorField);
for(int i=0;i<10;i++) {
- idField.setStringValue("" + i);
- writer.addDocument(document);
+ Document2 doc = writer.newDocument();
+ doc.addUniqueInt("id", i);
+ doc.addStored("stored", "stored");
+ doc.addAtom("termVector", "termVector");
+ writer.addDocument(doc);
}
writer.close();
@@ -157,8 +146,8 @@ public class TestIndexWriterMerging exte
IndexWriterConfig dontMergeConfig = new IndexWriterConfig(new MockAnalyzer(random()))
.setMergePolicy(NoMergePolicy.INSTANCE);
writer = new IndexWriter(dir, dontMergeConfig);
- writer.deleteDocuments(new Term("id", "0"));
- writer.deleteDocuments(new Term("id", "7"));
+ writer.deleteDocuments(fieldTypes.newIntTerm("id", 0));
+ writer.deleteDocuments(fieldTypes.newIntTerm("id", 7));
writer.close();
ir = DirectoryReader.open(dir);
@@ -190,26 +179,17 @@ public class TestIndexWriterMerging exte
.setMergePolicy(newLogMergePolicy(50))
);
- Document document = new Document();
+ FieldTypes fieldTypes = writer.getFieldTypes();
+ fieldTypes.enableTermVectors("termVector");
+ fieldTypes.enableTermVectorOffsets("termVector");
+ fieldTypes.enableTermVectorPositions("termVector");
- FieldType customType = new FieldType();
- customType.setStored(true);
-
- FieldType customType1 = new FieldType(TextField.TYPE_NOT_STORED);
- customType1.setTokenized(false);
- customType1.setStoreTermVectors(true);
- customType1.setStoreTermVectorPositions(true);
- customType1.setStoreTermVectorOffsets(true);
-
- Field storedField = newField("stored", "stored", customType);
- document.add(storedField);
- Field termVectorField = newField("termVector", "termVector", customType1);
- document.add(termVectorField);
- Field idField = newStringField("id", "", Field.Store.NO);
- document.add(idField);
for(int i=0;i<98;i++) {
- idField.setStringValue("" + i);
- writer.addDocument(document);
+ Document2 doc = writer.newDocument();
+ doc.addUniqueInt("id", i);
+ doc.addStored("stored", "stored");
+ doc.addAtom("termVector", "termVector");
+ writer.addDocument(doc);
}
writer.close();
@@ -222,7 +202,7 @@ public class TestIndexWriterMerging exte
.setMergePolicy(NoMergePolicy.INSTANCE);
writer = new IndexWriter(dir, dontMergeConfig);
for(int i=0;i<98;i+=2) {
- writer.deleteDocuments(new Term("id", "" + i));
+ writer.deleteDocuments(fieldTypes.newIntTerm("id", i));
}
writer.close();
@@ -257,25 +237,17 @@ public class TestIndexWriterMerging exte
.setMergePolicy(newLogMergePolicy(50))
);
- FieldType customType = new FieldType();
- customType.setStored(true);
+ FieldTypes fieldTypes = writer.getFieldTypes();
+ fieldTypes.enableTermVectors("termVector");
+ fieldTypes.enableTermVectorOffsets("termVector");
+ fieldTypes.enableTermVectorPositions("termVector");
- FieldType customType1 = new FieldType(TextField.TYPE_NOT_STORED);
- customType1.setTokenized(false);
- customType1.setStoreTermVectors(true);
- customType1.setStoreTermVectorPositions(true);
- customType1.setStoreTermVectorOffsets(true);
-
- Document document = new Document();
- Field storedField = newField("stored", "stored", customType);
- document.add(storedField);
- Field termVectorField = newField("termVector", "termVector", customType1);
- document.add(termVectorField);
- Field idField = newStringField("id", "", Field.Store.NO);
- document.add(idField);
for(int i=0;i<98;i++) {
- idField.setStringValue("" + i);
- writer.addDocument(document);
+ Document2 doc = writer.newDocument();
+ doc.addUniqueInt("id", i);
+ doc.addStored("stored", "stored");
+ doc.addAtom("termVector", "termVector");
+ writer.addDocument(doc);
}
writer.close();
@@ -288,7 +260,7 @@ public class TestIndexWriterMerging exte
.setMergePolicy(NoMergePolicy.INSTANCE);
writer = new IndexWriter(dir, dontMergeConfig);
for(int i=0;i<98;i+=2) {
- writer.deleteDocuments(new Term("id", "" + i));
+ writer.deleteDocuments(fieldTypes.newIntTerm("id", i));
}
writer.close();
ir = DirectoryReader.open(dir);
@@ -342,14 +314,13 @@ public class TestIndexWriterMerging exte
lmp.setMaxMergeDocs(20);
lmp.setMergeFactor(2);
IndexWriter iw = new IndexWriter(dir, conf);
- Document document = new Document();
-
- FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
- customType.setStoreTermVectors(true);
-
- document.add(newField("tvtest", "a b c", customType));
- for(int i=0;i<177;i++)
+ FieldTypes fieldTypes = iw.getFieldTypes();
+ fieldTypes.enableTermVectors("tvtest");
+ Document2 document = iw.newDocument();
+ document.addLargeText("tvtest", "a b c");
+ for(int i=0;i<177;i++) {
iw.addDocument(document);
+ }
iw.close();
dir.close();
}
@@ -361,13 +332,6 @@ public class TestIndexWriterMerging exte
((MockDirectoryWrapper) directory).setPreventDoubleWrite(false);
}
- final Document doc = new Document();
- FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
- customType.setTokenized(false);
-
- Field idField = newField("id", "", customType);
- doc.add(idField);
-
for(int pass=0;pass<2;pass++) {
if (VERBOSE) {
System.out.println("TEST: pass=" + pass);
@@ -383,6 +347,8 @@ public class TestIndexWriterMerging exte
}
IndexWriter writer = new IndexWriter(directory, conf);
+ FieldTypes fieldTypes = writer.getFieldTypes();
+
((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(100);
for(int iter=0;iter<10;iter++) {
@@ -390,13 +356,14 @@ public class TestIndexWriterMerging exte
System.out.println("TEST: iter=" + iter);
}
for(int j=0;j<199;j++) {
- idField.setStringValue(Integer.toString(iter*201+j));
+ Document2 doc = writer.newDocument();
+ doc.addUniqueInt("id", iter*201+j);
writer.addDocument(doc);
}
int delID = iter*199;
for(int j=0;j<20;j++) {
- writer.deleteDocuments(new Term("id", Integer.toString(delID)));
+ writer.deleteDocuments(fieldTypes.newIntTerm("id", delID));
delID += 5;
}
@@ -415,7 +382,7 @@ public class TestIndexWriterMerging exte
while(!done) {
for(int i=0;i<100;i++) {
try {
- finalWriter.addDocument(doc);
+ finalWriter.addDocument(finalWriter.newDocument());
} catch (AlreadyClosedException e) {
done = true;
break;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1640053&r1=1640052&r2=1640053&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Mon Nov 17 00:43:44 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -501,9 +502,8 @@ public class TestIndexWriterOnDiskFull e
// we can do this because we add/delete/add (and dont merge to "nothing")
w.setKeepFullyDeletedSegments(true);
- Document doc = new Document();
-
- doc.add(newTextField("f", "doctor who", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("f", "doctor who");
w.addDocument(doc);
w.commit();
@@ -541,9 +541,8 @@ public class TestIndexWriterOnDiskFull e
.setCommitOnClose(false));
writer.commit(); // empty commit, to not create confusing situation with first commit
dir.setMaxSizeInBytes(Math.max(1, dir.getRecomputedActualSizeInBytes()));
- final Document doc = new Document();
- FieldType customType = new FieldType(TextField.TYPE_STORED);
- doc.add(newField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", customType));
+ final Document2 doc = writer.newDocument();
+ doc.addLargeText("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj");
try {
writer.addDocument(doc);
fail("did not hit disk full");
@@ -572,20 +571,18 @@ public class TestIndexWriterOnDiskFull e
// TODO: these are also in TestIndexWriter... add a simple doc-writing method
// like this to LuceneTestCase?
- private void addDoc(IndexWriter writer) throws IOException
- {
- Document doc = new Document();
- doc.add(newTextField("content", "aaa", Field.Store.NO));
- doc.add(new NumericDocValuesField("numericdv", 1));
- writer.addDocument(doc);
+ private void addDoc(IndexWriter writer) throws IOException {
+ Document2 doc = writer.newDocument();
+ doc.addLargeText("content", "aaa");
+ doc.addInt("numericdv", 1);
+ writer.addDocument(doc);
}
- private void addDocWithIndex(IndexWriter writer, int index) throws IOException
- {
- Document doc = new Document();
- doc.add(newTextField("content", "aaa " + index, Field.Store.NO));
- doc.add(newTextField("id", "" + index, Field.Store.NO));
- doc.add(new NumericDocValuesField("numericdv", 1));
- writer.addDocument(doc);
+ private void addDocWithIndex(IndexWriter writer, int index) throws IOException {
+ Document2 doc = writer.newDocument();
+ doc.addLargeText("content", "aaa " + index);
+ doc.addLargeText("id", "" + index);
+ doc.addInt("numericdv", 1);
+ writer.addDocument(doc);
}
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1640053&r1=1640052&r2=1640053&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java Mon Nov 17 00:43:44 2014
@@ -1142,7 +1142,7 @@ public class TestIndexWriterReader exten
// Create 500 segments:
for(int i=0;i<500;i++) {
Document2 doc = w.newDocument();
- doc.addAtom("id", ""+i);
+ doc.addUniqueInt("id", i);
w.addDocument(doc);
IndexReader r = DirectoryReader.open(w, true);
// Make sure segment count never exceeds 100:
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java?rev=1640053&r1=1640052&r2=1640053&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java Mon Nov 17 00:43:44 2014
@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldTypes;
@@ -60,8 +61,8 @@ public class TestIndexWriterThreadsToSeg
public void run() {
try {
startingGun.await();
- Document doc = new Document();
- doc.add(newTextField("field", "here is some text", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("field", "here is some text");
w.addDocument(doc);
startDone.countDown();
@@ -210,8 +211,8 @@ public class TestIndexWriterThreadsToSeg
}
// We get to index on this cycle:
- Document doc = new Document();
- doc.add(new TextField("field", "here is some text that is a bit longer than normal trivial text", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("field", "here is some text that is a bit longer than normal trivial text");
for(int j=0;j<200;j++) {
w.addDocument(doc);
}
@@ -253,8 +254,8 @@ public class TestIndexWriterThreadsToSeg
public void run() {
try {
startingGun.await();
- Document doc = new Document();
- doc.add(new TextField("field", "here is some text that is a bit longer than normal trivial text", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addLargeText("field", "here is some text that is a bit longer than normal trivial text");
for(int j=0;j<1000;j++) {
w.addDocument(doc);
}
@@ -301,8 +302,8 @@ public class TestIndexWriterThreadsToSeg
try {
startingGun.await();
for(int j=0;j<1000;j++) {
- Document doc = new Document();
- doc.add(newStringField("field", "threadID" + threadID, Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addAtom("field", "threadID" + threadID);
w.addDocument(doc);
}
} catch (Exception e) {
@@ -325,8 +326,8 @@ public class TestIndexWriterThreadsToSeg
// At this point the writer should have 2 thread states w/ docs; now we index with only 1 thread until we see all 1000 thread0 & thread1
// docs flushed. If the writer incorrectly holds onto previously indexed docs forever then this will run forever:
while (thread0Count < 1000 || thread1Count < 1000) {
- Document doc = new Document();
- doc.add(newStringField("field", "threadIDmain", Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addAtom("field", "threadIDmain");
w.addDocument(doc);
for(String fileName : dir.listAll()) {