You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/10/28 10:04:07 UTC
svn commit: r1634820 [4/4] - in /lucene/dev/branches/lucene6005/lucene:
core/src/java/org/apache/lucene/document/
core/src/java/org/apache/lucene/index/
core/src/java/org/apache/lucene/search/ core/src/test/org/apache/lucene/
core/src/test/org/apache/l...
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java?rev=1634820&r1=1634819&r2=1634820&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java Tue Oct 28 09:04:05 2014
@@ -17,14 +17,20 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.CompositeReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Term;
@@ -32,11 +38,6 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
public class TestTopDocsMerge extends LuceneTestCase {
private static class ShardSearcher extends IndexSearcher {
@@ -102,10 +103,10 @@ public class TestTopDocsMerge extends Lu
}
for(int docIDX=0;docIDX<numDocs;docIDX++) {
- final Document doc = new Document();
- doc.add(newStringField("string", TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
- doc.add(newTextField("text", content[random().nextInt(content.length)], Field.Store.NO));
- doc.add(new FloatField("float", random().nextFloat(), Field.Store.NO));
+ final Document2 doc = w.newDocument();
+ doc.addAtom("string", TestUtil.randomRealisticUnicodeString(random()));
+ doc.addLargeText("text", content[random().nextInt(content.length)]);
+ doc.addFloat("float", random().nextFloat());
final int intValue;
if (random().nextInt(100) == 17) {
intValue = Integer.MIN_VALUE;
@@ -114,7 +115,7 @@ public class TestTopDocsMerge extends Lu
} else {
intValue = random().nextInt();
}
- doc.add(new IntField("int", intValue, Field.Store.NO));
+ doc.addInt("int", intValue);
if (VERBOSE) {
System.out.println(" doc=" + doc);
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1634820&r1=1634819&r2=1634820&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Tue Oct 28 09:04:05 2014
@@ -30,9 +30,11 @@ import java.util.*;
import java.util.concurrent.CountDownLatch;
import org.apache.lucene.analysis.tokenattributes.*;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.RandomIndexWriter;
@@ -550,36 +552,52 @@ public abstract class BaseTokenStreamTes
private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect, RandomIndexWriter iw) throws IOException {
final LineFileDocs docs = new LineFileDocs(random);
- Document doc = null;
- Field field = null, currentField = null;
StringReader bogus = new StringReader("");
+ Document2 doc = null;
if (iw != null) {
- doc = new Document();
- FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ FieldTypes fieldTypes = iw.getFieldTypes();
+ doc = iw.newDocument();
+
+ // Randomize how we index the field:
if (random.nextBoolean()) {
- ft.setStoreTermVectors(true);
- ft.setStoreTermVectorOffsets(random.nextBoolean());
- ft.setStoreTermVectorPositions(random.nextBoolean());
- if (ft.storeTermVectorPositions()) {
- ft.setStoreTermVectorPayloads(random.nextBoolean());
+ fieldTypes.enableTermVectors("dummy");
+ if (random().nextBoolean()) {
+ fieldTypes.enableTermVectorOffsets("dummy");
+ }
+ if (random().nextBoolean()) {
+ fieldTypes.enableTermVectorPositions("dummy");
+ if (random().nextBoolean()) {
+ fieldTypes.enableTermVectorPayloads("dummy");
+ }
}
}
+
if (random.nextBoolean()) {
- ft.setOmitNorms(true);
+ fieldTypes.disableNorms("dummy");
}
+
+ IndexOptions indexOptions;
+
switch(random.nextInt(4)) {
- case 0: ft.setIndexOptions(IndexOptions.DOCS_ONLY); break;
- case 1: ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); break;
- case 2: ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); break;
+ case 0:
+ indexOptions = IndexOptions.DOCS_ONLY;
+ break;
+ case 1:
+ indexOptions = IndexOptions.DOCS_AND_FREQS;
+ break;
+ case 2:
+ indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+ break;
default:
- if (offsetsAreCorrect) {
- ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- } else {
- ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- }
+ if (offsetsAreCorrect) {
+ indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
+ } else {
+ indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+ }
}
- currentField = field = new Field("dummy", bogus, ft);
- doc.add(currentField);
+ fieldTypes.disableHighlighting("dummy");
+ fieldTypes.disableStored("dummy");
+ fieldTypes.setIndexOptions("dummy", indexOptions);
}
try {
@@ -611,22 +629,11 @@ public abstract class BaseTokenStreamTes
}
try {
- checkAnalysisConsistency(random, a, useCharFilter, text, offsetsAreCorrect, currentField);
- if (iw != null) {
- if (random.nextInt(7) == 0) {
- // pile up a multivalued field
- FieldType ft = field.fieldType();
- currentField = new Field("dummy", bogus, ft);
- doc.add(currentField);
- } else {
- iw.addDocument(doc);
- if (doc.getFields().size() > 1) {
- // back to 1 field
- currentField = field;
- doc.removeFields("dummy");
- doc.add(currentField);
- }
- }
+ checkAnalysisConsistency(random, a, useCharFilter, text, offsetsAreCorrect, doc);
+ // We randomly accumulate multiple values for this field:
+ if (iw != null && random.nextInt(7) != 0) {
+ iw.addDocument(doc);
+ doc = iw.newDocument();
}
} catch (Throwable t) {
// TODO: really we should pass a random seed to
@@ -677,7 +684,7 @@ public abstract class BaseTokenStreamTes
checkAnalysisConsistency(random, a, useCharFilter, text, offsetsAreCorrect, null);
}
- private static void checkAnalysisConsistency(Random random, Analyzer a, boolean useCharFilter, String text, boolean offsetsAreCorrect, Field field) throws IOException {
+ private static void checkAnalysisConsistency(Random random, Analyzer a, boolean useCharFilter, String text, boolean offsetsAreCorrect, Document2 doc) throws IOException {
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": NOTE: BaseTokenStreamTestCase: get first token stream now text=" + text);
@@ -871,7 +878,7 @@ public abstract class BaseTokenStreamTes
tokens.toArray(new String[tokens.size()]));
}
- if (field != null) {
+ if (doc != null) {
reader = new StringReader(text);
random = new Random(seed);
if (random.nextInt(30) == 7) {
@@ -881,8 +888,7 @@ public abstract class BaseTokenStreamTes
reader = new MockReaderWrapper(random, reader);
}
-
- field.setReaderValue(useCharFilter ? new MockCharFilter(reader, remainder) : reader);
+ doc.addLargeText("dummy", useCharFilter ? new MockCharFilter(reader, remainder) : reader);
}
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1634820&r1=1634819&r2=1634820&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java Tue Oct 28 09:04:05 2014
@@ -209,7 +209,11 @@ public final class RAMOnlyPostingsFormat
public RAMDoc(int docID, int freq) {
this.docID = docID;
- positions = new int[freq];
+ if (freq >= 0) {
+ positions = new int[freq];
+ } else {
+ positions = null;
+ }
}
@Override
@@ -347,12 +351,14 @@ public final class RAMOnlyPostingsFormat
postingsWriter.finishDoc();
}
- termsConsumer.finishTerm(term, new TermStats(docFreq, totalTermFreq));
- sumDocFreq += docFreq;
- sumTotalTermFreq += totalTermFreq;
+ if (docFreq > 0) {
+ termsConsumer.finishTerm(term, new TermStats(docFreq, writeFreqs ? totalTermFreq : -1));
+ sumDocFreq += docFreq;
+ sumTotalTermFreq += totalTermFreq;
+ }
}
- termsConsumer.finish(sumTotalTermFreq, sumDocFreq, docsSeen.cardinality());
+ termsConsumer.finish(writeFreqs ? sumTotalTermFreq : -1, sumDocFreq, docsSeen.cardinality());
}
}
@@ -421,7 +427,7 @@ public final class RAMOnlyPostingsFormat
}
public void finishDoc() {
- assert posUpto == current.positions.length;
+ assert current.positions == null || posUpto == current.positions.length;
}
}
@@ -539,7 +545,12 @@ public final class RAMOnlyPostingsFormat
@Override
public int freq() throws IOException {
- return current.positions.length;
+ if (current.positions == null) {
+ // Big fat lie:
+ return 1;
+ } else {
+ return current.positions.length;
+ }
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java?rev=1634820&r1=1634819&r2=1634820&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java Tue Oct 28 09:04:05 2014
@@ -33,12 +33,14 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType.NumericType;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
@@ -53,11 +55,10 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
-import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper.Throttling;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TestUtil;
-
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@@ -95,19 +96,16 @@ public abstract class BaseStoredFieldsFo
fieldIDs.add(i);
}
- final Map<String,Document> docs = new HashMap<>();
+ final Map<String,Document2> docs = new HashMap<>();
if (VERBOSE) {
System.out.println("TEST: build index docCount=" + docCount);
}
- FieldType customType2 = new FieldType();
- customType2.setStored(true);
for(int i=0;i<docCount;i++) {
- Document doc = new Document();
- doc.add(idField);
+ Document2 doc = w.newDocument();
final String id = ""+i;
- idField.setStringValue(id);
+ doc.addAtom("id", id);
docs.put(id, doc);
if (VERBOSE) {
System.out.println("TEST: add doc id=" + id);
@@ -117,7 +115,7 @@ public abstract class BaseStoredFieldsFo
final String s;
if (rand.nextInt(4) != 3) {
s = TestUtil.randomUnicodeString(rand, 1000);
- doc.add(newField("f"+field, s, customType2));
+ doc.addStored("f"+field, s);
} else {
s = null;
}
@@ -160,7 +158,7 @@ public abstract class BaseStoredFieldsFo
TopDocs hits = s.search(new TermQuery(new Term("id", testID)), 1);
assertEquals(1, hits.totalHits);
StoredDocument doc = r.document(hits.scoreDocs[0].doc);
- Document docExp = docs.get(testID);
+ Document2 docExp = docs.get(testID);
for(int i=0;i<fieldCount;i++) {
assertEquals("doc " + testID + ", field f" + fieldCount + " is wrong", docExp.get("f"+i), doc.get("f"+i));
}
@@ -238,86 +236,12 @@ public abstract class BaseStoredFieldsFo
dir.close();
}
- public void testNumericField() throws Exception {
- Directory dir = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random(), dir);
- final int numDocs = atLeast(500);
- final Number[] answers = new Number[numDocs];
- final NumericType[] typeAnswers = new NumericType[numDocs];
- for(int id=0;id<numDocs;id++) {
- Document doc = new Document();
- final Field nf;
- final Field sf;
- final Number answer;
- final NumericType typeAnswer;
- if (random().nextBoolean()) {
- // float/double
- if (random().nextBoolean()) {
- final float f = random().nextFloat();
- answer = Float.valueOf(f);
- nf = new FloatField("nf", f, Field.Store.NO);
- sf = new StoredField("nf", f);
- typeAnswer = NumericType.FLOAT;
- } else {
- final double d = random().nextDouble();
- answer = Double.valueOf(d);
- nf = new DoubleField("nf", d, Field.Store.NO);
- sf = new StoredField("nf", d);
- typeAnswer = NumericType.DOUBLE;
- }
- } else {
- // int/long
- if (random().nextBoolean()) {
- final int i = random().nextInt();
- answer = Integer.valueOf(i);
- nf = new IntField("nf", i, Field.Store.NO);
- sf = new StoredField("nf", i);
- typeAnswer = NumericType.INT;
- } else {
- final long l = random().nextLong();
- answer = Long.valueOf(l);
- nf = new LongField("nf", l, Field.Store.NO);
- sf = new StoredField("nf", l);
- typeAnswer = NumericType.LONG;
- }
- }
- doc.add(nf);
- doc.add(sf);
- answers[id] = answer;
- typeAnswers[id] = typeAnswer;
- FieldType ft = new FieldType(IntField.TYPE_STORED);
- ft.setNumericPrecisionStep(Integer.MAX_VALUE);
- doc.add(new IntField("id", id, ft));
- doc.add(new NumericDocValuesField("id", id));
- w.addDocument(doc);
- }
- final DirectoryReader r = w.getReader();
- w.close();
-
- assertEquals(numDocs, r.numDocs());
-
- for(LeafReaderContext ctx : r.leaves()) {
- final LeafReader sub = ctx.reader();
- final NumericDocValues ids = DocValues.getNumeric(sub, "id");
- for(int docID=0;docID<sub.numDocs();docID++) {
- final StoredDocument doc = sub.document(docID);
- final Field f = (Field) doc.getField("nf");
- assertTrue("got f=" + f, f instanceof StoredField);
- assertEquals(answers[(int) ids.get(docID)], f.numericValue());
- }
- }
- r.close();
- dir.close();
- }
-
public void testIndexedBit() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
- FieldType onlyStored = new FieldType();
- onlyStored.setStored(true);
- doc.add(new Field("field", "value", onlyStored));
- doc.add(new StringField("field2", "value", Field.Store.YES));
+ Document2 doc = w.newDocument();
+ doc.addStored("field", "value");
+ doc.addAtom("field2", "value");
w.addDocument(doc);
IndexReader r = w.getReader();
w.close();
@@ -412,12 +336,10 @@ public abstract class BaseStoredFieldsFo
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
// make sure the readers are properly cloned
- final Document doc = new Document();
- final Field field = new StringField("fld", "", Store.YES);
- doc.add(field);
final int numDocs = atLeast(1000);
for (int i = 0; i < numDocs; ++i) {
- field.setStringValue("" + i);
+ final Document2 doc = iw.newDocument();
+ doc.addAtom("fld", "" + i);
iw.addDocument(doc);
}
iw.commit();
@@ -515,17 +437,11 @@ public abstract class BaseStoredFieldsFo
}
}
- final FieldType type = new FieldType(StringField.TYPE_STORED);
- type.setIndexOptions(null);
- type.freeze();
- IntField id = new IntField("id", 0, Store.YES);
for (int i = 0; i < data.length; ++i) {
- Document doc = new Document();
- doc.add(id);
- id.setIntValue(i);
+ Document2 doc = iw.newDocument();
+ doc.addInt("id", i);
for (int j = 0; j < data[i].length; ++j) {
- Field f = new Field("bytes" + j, data[i][j], type);
- doc.add(f);
+ doc.addStored("bytes" + j, new BytesRef(data[i][j]));
}
iw.w.addDocument(doc);
if (random().nextBoolean() && (i % (data.length / 10) == 0)) {
@@ -655,9 +571,9 @@ public abstract class BaseStoredFieldsFo
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE));
for (int i = 0; i < numDocs; ++i) {
- Document doc = new Document();
- doc.add(new StringField("id", Integer.toString(i), Store.YES));
- doc.add(new StoredField("f", TestUtil.randomSimpleString(random())));
+ Document2 doc = w.newDocument();
+ doc.addAtom("id", Integer.toString(i));
+ doc.addAtom("f", TestUtil.randomSimpleString(random()));
w.addDocument(doc);
}
final int deleteCount = TestUtil.nextInt(random(), 5, numDocs);
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java?rev=1634820&r1=1634819&r2=1634820&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java Tue Oct 28 09:04:05 2014
@@ -19,16 +19,22 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
@@ -36,195 +42,70 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
class DocHelper {
-
- public static final FieldType customType;
+
public static final String FIELD_1_TEXT = "field one text";
public static final String TEXT_FIELD_1_KEY = "textField1";
- public static Field textField1;
- static {
- customType = new FieldType(TextField.TYPE_STORED);
- textField1 = new Field(TEXT_FIELD_1_KEY, FIELD_1_TEXT, customType);
- }
- public static final FieldType customType2;
public static final String FIELD_2_TEXT = "field field field two text";
//Fields will be lexicographically sorted. So, the order is: field, text, two
public static final int [] FIELD_2_FREQS = {3, 1, 1};
public static final String TEXT_FIELD_2_KEY = "textField2";
- public static Field textField2;
- static {
- customType2 = new FieldType(TextField.TYPE_STORED);
- customType2.setStoreTermVectors(true);
- customType2.setStoreTermVectorPositions(true);
- customType2.setStoreTermVectorOffsets(true);
- textField2 = new Field(TEXT_FIELD_2_KEY, FIELD_2_TEXT, customType2);
- }
- public static final FieldType customType3;
public static final String FIELD_3_TEXT = "aaaNoNorms aaaNoNorms bbbNoNorms";
public static final String TEXT_FIELD_3_KEY = "textField3";
- public static Field textField3;
- static {
- customType3 = new FieldType(TextField.TYPE_STORED);
- customType3.setOmitNorms(true);
- textField3 = new Field(TEXT_FIELD_3_KEY, FIELD_3_TEXT, customType3);
- }
-
public static final String KEYWORD_TEXT = "Keyword";
public static final String KEYWORD_FIELD_KEY = "keyField";
- public static Field keyField;
- static {
- keyField = new StringField(KEYWORD_FIELD_KEY, KEYWORD_TEXT, Field.Store.YES);
- }
- public static final FieldType customType5;
public static final String NO_NORMS_TEXT = "omitNormsText";
public static final String NO_NORMS_KEY = "omitNorms";
- public static Field noNormsField;
- static {
- customType5 = new FieldType(TextField.TYPE_STORED);
- customType5.setOmitNorms(true);
- customType5.setTokenized(false);
- noNormsField = new Field(NO_NORMS_KEY, NO_NORMS_TEXT, customType5);
- }
- public static final FieldType customType6;
public static final String NO_TF_TEXT = "analyzed with no tf and positions";
public static final String NO_TF_KEY = "omitTermFreqAndPositions";
- public static Field noTFField;
- static {
- customType6 = new FieldType(TextField.TYPE_STORED);
- customType6.setIndexOptions(IndexOptions.DOCS_ONLY);
- noTFField = new Field(NO_TF_KEY, NO_TF_TEXT, customType6);
- }
- public static final FieldType customType7;
public static final String UNINDEXED_FIELD_TEXT = "unindexed field text";
public static final String UNINDEXED_FIELD_KEY = "unIndField";
- public static Field unIndField;
- static {
- customType7 = new FieldType();
- customType7.setStored(true);
- unIndField = new Field(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT, customType7);
- }
-
public static final String UNSTORED_1_FIELD_TEXT = "unstored field text";
public static final String UNSTORED_FIELD_1_KEY = "unStoredField1";
- public static Field unStoredField1 = new TextField(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT, Field.Store.NO);
- public static final FieldType customType8;
public static final String UNSTORED_2_FIELD_TEXT = "unstored field text";
public static final String UNSTORED_FIELD_2_KEY = "unStoredField2";
- public static Field unStoredField2;
- static {
- customType8 = new FieldType(TextField.TYPE_NOT_STORED);
- customType8.setStoreTermVectors(true);
- unStoredField2 = new Field(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT, customType8);
- }
public static final String LAZY_FIELD_BINARY_KEY = "lazyFieldBinary";
- public static byte [] LAZY_FIELD_BINARY_BYTES;
- public static Field lazyFieldBinary;
+ public static final BytesRef LAZY_FIELD_BINARY_BYTES = new BytesRef("These are some binary field bytes");
public static final String LAZY_FIELD_KEY = "lazyField";
public static final String LAZY_FIELD_TEXT = "These are some field bytes";
- public static Field lazyField = new Field(LAZY_FIELD_KEY, LAZY_FIELD_TEXT, customType);
public static final String LARGE_LAZY_FIELD_KEY = "largeLazyField";
public static String LARGE_LAZY_FIELD_TEXT;
- public static Field largeLazyField;
//From Issue 509
public static final String FIELD_UTF1_TEXT = "field one \u4e00text";
public static final String TEXT_FIELD_UTF1_KEY = "textField1Utf8";
- public static Field textUtfField1 = new Field(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT, customType);
public static final String FIELD_UTF2_TEXT = "field field field \u4e00two text";
//Fields will be lexicographically sorted. So, the order is: field, text, two
public static final int [] FIELD_UTF2_FREQS = {3, 1, 1};
public static final String TEXT_FIELD_UTF2_KEY = "textField2Utf8";
- public static Field textUtfField2 = new Field(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT, customType2);
-
-
-
public static Map<String,Object> nameValues = null;
- // ordered list of all the fields...
- // could use LinkedHashMap for this purpose if Java1.4 is OK
- public static Field[] fields = new Field[] {
- textField1,
- textField2,
- textField3,
- keyField,
- noNormsField,
- noTFField,
- unIndField,
- unStoredField1,
- unStoredField2,
- textUtfField1,
- textUtfField2,
- lazyField,
- lazyFieldBinary,//placeholder for binary field, since this is null. It must be second to last.
- largeLazyField//placeholder for large field, since this is null. It must always be last
- };
-
- public static Map<String,IndexableField> all =new HashMap<>();
- public static Map<String,IndexableField> indexed =new HashMap<>();
- public static Map<String,IndexableField> stored =new HashMap<>();
- public static Map<String,IndexableField> unstored=new HashMap<>();
- public static Map<String,IndexableField> unindexed=new HashMap<>();
- public static Map<String,IndexableField> termvector=new HashMap<>();
- public static Map<String,IndexableField> notermvector=new HashMap<>();
- public static Map<String,IndexableField> lazy= new HashMap<>();
- public static Map<String,IndexableField> noNorms=new HashMap<>();
- public static Map<String,IndexableField> noTf=new HashMap<>();
-
static {
//Initialize the large Lazy Field
StringBuilder buffer = new StringBuilder();
- for (int i = 0; i < 10000; i++)
- {
+ for (int i = 0; i < 10000; i++) {
buffer.append("Lazily loading lengths of language in lieu of laughing ");
}
- try {
- LAZY_FIELD_BINARY_BYTES = "These are some binary field bytes".getBytes("UTF8");
- } catch (UnsupportedEncodingException e) {
- }
- lazyFieldBinary = new StoredField(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES);
- fields[fields.length - 2] = lazyFieldBinary;
LARGE_LAZY_FIELD_TEXT = buffer.toString();
- largeLazyField = new Field(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT, customType);
- fields[fields.length - 1] = largeLazyField;
- for (int i=0; i<fields.length; i++) {
- IndexableField f = fields[i];
- add(all,f);
- if (f.fieldType().indexOptions() != null) add(indexed,f);
- else add(unindexed,f);
- if (f.fieldType().storeTermVectors()) add(termvector,f);
- if (f.fieldType().indexOptions() != null && !f.fieldType().storeTermVectors()) add(notermvector,f);
- if (f.fieldType().stored()) add(stored,f);
- else add(unstored,f);
- if (f.fieldType().indexOptions() == IndexOptions.DOCS_ONLY) add(noTf,f);
- if (f.fieldType().omitNorms()) add(noNorms,f);
- if (f.fieldType().indexOptions() == IndexOptions.DOCS_ONLY) add(noTf,f);
- //if (f.isLazy()) add(lazy, f);
- }
}
-
- private static void add(Map<String,IndexableField> map, IndexableField field) {
- map.put(field.name(), field);
- }
-
-
- static
- {
+ static {
nameValues = new HashMap<>();
nameValues.put(TEXT_FIELD_1_KEY, FIELD_1_TEXT);
nameValues.put(TEXT_FIELD_2_KEY, FIELD_2_TEXT);
@@ -240,26 +121,132 @@ class DocHelper {
nameValues.put(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT);
nameValues.put(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT);
nameValues.put(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT);
- }
-
+ }
+
/**
* Adds the fields above to a document
* @param doc The document to write
*/
- public static void setupDoc(Document doc) {
- for (int i=0; i<fields.length; i++) {
- doc.add(fields[i]);
- }
+ private static void setupDoc(FieldTypes fieldTypes, Document2 doc) {
+
+ fieldTypes.enableTermVectors(TEXT_FIELD_2_KEY);
+ fieldTypes.disableHighlighting(TEXT_FIELD_2_KEY);
+ fieldTypes.enableTermVectorPositions(TEXT_FIELD_2_KEY);
+ fieldTypes.enableTermVectorOffsets(TEXT_FIELD_2_KEY);
+
+ fieldTypes.enableTermVectors(TEXT_FIELD_UTF2_KEY);
+ fieldTypes.disableHighlighting(TEXT_FIELD_UTF2_KEY);
+ fieldTypes.enableTermVectorPositions(TEXT_FIELD_UTF2_KEY);
+ fieldTypes.enableTermVectorOffsets(TEXT_FIELD_UTF2_KEY);
+
+ fieldTypes.disableHighlighting(TEXT_FIELD_3_KEY);
+ fieldTypes.disableNorms(TEXT_FIELD_3_KEY);
+
+ fieldTypes.disableHighlighting(NO_NORMS_KEY);
+ fieldTypes.setIndexOptions(NO_NORMS_KEY, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ fieldTypes.disableNorms(NO_NORMS_KEY);
+
+ // Otherwise large text:
+ fieldTypes.disableHighlighting(NO_TF_KEY);
+ fieldTypes.setIndexOptions(NO_TF_KEY, IndexOptions.DOCS_ONLY);
+
+ // Otherwise large text:
+ fieldTypes.disableStored(UNSTORED_FIELD_1_KEY);
+
+ // Otherwise large text:
+ fieldTypes.enableTermVectors(UNSTORED_FIELD_2_KEY);
+ fieldTypes.disableStored(UNSTORED_FIELD_2_KEY);
+
+ doc.addLargeText(TEXT_FIELD_1_KEY, FIELD_1_TEXT);
+ doc.addLargeText(TEXT_FIELD_2_KEY, FIELD_2_TEXT);
+ doc.addLargeText(TEXT_FIELD_3_KEY, FIELD_3_TEXT);
+ doc.addAtom(KEYWORD_FIELD_KEY, KEYWORD_TEXT);
+ doc.addLargeText(NO_NORMS_KEY, NO_NORMS_TEXT);
+ doc.addLargeText(NO_TF_KEY, NO_TF_TEXT);
+ doc.addStored(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT);
+ doc.addLargeText(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT);
+ doc.addLargeText(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT);
+ doc.addLargeText(LAZY_FIELD_KEY, LAZY_FIELD_TEXT);
+ doc.addStored(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES);
+ doc.addLargeText(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT);
+ doc.addLargeText(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT);
+ doc.addLargeText(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT);
}
+ public static Set<String> getUnstored(FieldTypes fieldTypes) {
+ Set<String> unstored = new HashSet<>();
+ for(String fieldName : fieldTypes.getFieldNames()) {
+ if (fieldTypes.getStored(fieldName) == false) {
+ unstored.add(fieldName);
+ }
+ }
+ return unstored;
+ }
+
+ public static Set<String> getIndexed(FieldTypes fieldTypes) {
+ Set<String> indexed = new HashSet<>();
+ for(String fieldName : fieldTypes.getFieldNames()) {
+ if (fieldTypes.getIndexOptions(fieldName) != null) {
+ indexed.add(fieldName);
+ }
+ }
+ return indexed;
+ }
+
+ public static Set<String> getNotIndexed(FieldTypes fieldTypes) {
+ Set<String> notIndexed = new HashSet<>();
+ for(String fieldName : fieldTypes.getFieldNames()) {
+ if (fieldTypes.getIndexOptions(fieldName) == null) {
+ notIndexed.add(fieldName);
+ }
+ }
+ return notIndexed;
+ }
+
+ public static Set<String> getTermVectorFields(FieldTypes fieldTypes) {
+ Set<String> tvFields = new HashSet<>();
+ for(String fieldName : fieldTypes.getFieldNames()) {
+ if (fieldTypes.getTermVectors(fieldName)) {
+ tvFields.add(fieldName);
+ }
+ }
+ return tvFields;
+ }
+
+ public static Set<String> getNoTermVectorFields(FieldTypes fieldTypes) {
+ Set<String> noTVFields = new HashSet<>();
+ for(String fieldName : fieldTypes.getFieldNames()) {
+ if (fieldTypes.getIndexOptions(fieldName) != null && fieldTypes.getTermVectors(fieldName) == false) {
+ noTVFields.add(fieldName);
+ }
+ }
+ return noTVFields;
+ }
+
+ public static Set<String> getNoNorms(FieldTypes fieldTypes) {
+ Set<String> noNorms = new HashSet<>();
+ for(String fieldName : fieldTypes.getFieldNames()) {
+ if (fieldTypes.getNorms(fieldName) == false) {
+ noNorms.add(fieldName);
+ }
+ }
+ return noNorms;
+ }
+
+ public static Set<String> getAll(FieldTypes fieldTypes) {
+ Set<String> all = new HashSet<>();
+ for(String fieldName : fieldTypes.getFieldNames()) {
+ all.add(fieldName);
+ }
+ return all;
+ }
+
/**
- * Writes the document to the directory using a segment
- * named "test"; returns the SegmentInfo describing the new
- * segment
+ * Writes our test document to the directory and returns the SegmentCommitInfo
+ * describing the new segment
*/
- public static SegmentCommitInfo writeDoc(Random random, Directory dir, Document doc) throws IOException
- {
- return writeDoc(random, dir, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false), null, doc);
+ public static SegmentCommitInfo writeDoc(Random random, Directory dir) throws IOException {
+ return writeDoc(random, dir, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false), null);
}
/**
@@ -267,10 +254,11 @@ class DocHelper {
* and the similarity score; returns the SegmentInfo
* describing the new segment
*/
- public static SegmentCommitInfo writeDoc(Random random, Directory dir, Analyzer analyzer, Similarity similarity, Document doc) throws IOException {
+ public static SegmentCommitInfo writeDoc(Random random, Directory dir, Analyzer analyzer, Similarity similarity) throws IOException {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( /* LuceneTestCase.newIndexWriterConfig(random, */
analyzer).setSimilarity(similarity == null ? IndexSearcher.getDefaultSimilarity() : similarity));
- //writer.setNoCFSRatio(0.0);
+ Document2 doc = writer.newDocument();
+ setupDoc(writer.getFieldTypes(), doc);
writer.addDocument(doc);
writer.commit();
SegmentCommitInfo info = writer.newestSegment();
@@ -278,36 +266,37 @@ class DocHelper {
return info;
}
- public static int numFields(Document doc) {
- return doc.getFields().size();
+ public static int numFields() {
+ return 14;
}
public static int numFields(StoredDocument doc) {
return doc.getFields().size();
}
- public static Document createDocument(int n, String indexName, int numFields) {
+ public static Document2 createDocument(IndexWriter writer, int n, String indexName, int numFields) {
StringBuilder sb = new StringBuilder();
- FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setStoreTermVectors(true);
- customType.setStoreTermVectorPositions(true);
- customType.setStoreTermVectorOffsets(true);
-
- FieldType customType1 = new FieldType(StringField.TYPE_STORED);
- customType1.setStoreTermVectors(true);
- customType1.setStoreTermVectorPositions(true);
- customType1.setStoreTermVectorOffsets(true);
-
- final Document doc = new Document();
- doc.add(new Field("id", Integer.toString(n), customType1));
- doc.add(new Field("indexname", indexName, customType1));
+
+ FieldTypes fieldTypes = writer.getFieldTypes();
+ for(int i=0;i<numFields+1;i++) {
+ fieldTypes.enableTermVectors("field" + (i+1));
+ fieldTypes.enableTermVectorPositions("field" + (i+1));
+ fieldTypes.enableTermVectorOffsets("field" + (i+1));
+ }
+ fieldTypes.enableTermVectors("id");
+ fieldTypes.enableTermVectorPositions("id");
+ fieldTypes.enableTermVectorOffsets("id");
+
+ Document2 doc = writer.newDocument();
+ doc.addAtom("id", Integer.toString(n));
+ doc.addAtom("indexname", indexName);
sb.append("a");
sb.append(n);
- doc.add(new Field("field1", sb.toString(), customType));
+ doc.addLargeText("field1", sb.toString());
sb.append(" b");
sb.append(n);
for (int i = 1; i < numFields; i++) {
- doc.add(new Field("field" + (i + 1), sb.toString(), customType));
+ doc.addLargeText("field" + (i+1), sb.toString());
}
return doc;
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1634820&r1=1634819&r2=1634820&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Tue Oct 28 09:04:05 2014
@@ -25,7 +25,9 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
@@ -94,6 +96,14 @@ public class RandomIndexWriter implement
// any forced merges:
doRandomForceMerge = !(c.getMergePolicy() instanceof NoMergePolicy) && r.nextBoolean();
}
+
+ public FieldTypes getFieldTypes() {
+ return w.getFieldTypes();
+ }
+
+ public Document2 newDocument() {
+ return w.newDocument();
+ }
/**
* Adds a Document.
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java?rev=1634820&r1=1634819&r2=1634820&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java Tue Oct 28 09:04:05 2014
@@ -202,7 +202,7 @@ public class QueryUtils {
}
IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random)));
for (int i = 0; i < numDocs; i++) {
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
}
w.forceMerge(1);
w.close();
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1634820&r1=1634819&r2=1634820&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Tue Oct 28 09:04:05 2014
@@ -56,6 +56,7 @@ import java.util.concurrent.atomic.Atomi
import java.util.logging.Logger;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -857,6 +858,11 @@ public abstract class LuceneTestCase ext
dumpIterator(label, iter, stream);
}
+ /** create a new index writer config with random defaults, using MockAnalyzer */
+ public static IndexWriterConfig newIndexWriterConfig() {
+ return newIndexWriterConfig(random(), new MockAnalyzer(random()));
+ }
+
/** create a new index writer config with random defaults */
public static IndexWriterConfig newIndexWriterConfig(Analyzer a) {
return newIndexWriterConfig(random(), a);