You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/11/30 12:07:19 UTC
svn commit: r1642535 [2/19] - in /lucene/dev/branches/lucene6005/lucene:
analysis/common/src/java/org/apache/lucene/collation/
analysis/common/src/test/org/apache/lucene/analysis/core/
analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/ ...
Modified: lucene/dev/branches/lucene6005/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/lucene6005/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sun Nov 30 11:07:09 2014
@@ -37,19 +37,8 @@ import java.util.regex.Pattern;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleDocValuesField;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedNumericDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
@@ -170,18 +159,23 @@ public class TestBackwardsCompatibility
IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()))
.setUseCompoundFile(false).setMergePolicy(NoMergePolicy.INSTANCE);
IndexWriter writer = new IndexWriter(dir, conf);
+ FieldTypes fieldTypes = writer.getFieldTypes();
+ fieldTypes.disableSorting("bdv1");
+ fieldTypes.disableSorting("bdv1_c");
+ fieldTypes.disableSorting("bdv2");
+ fieldTypes.disableSorting("bdv2_c");
// create an index w/ few doc-values fields, some with updates and some without
for (int i = 0; i < 30; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", "" + i, Field.Store.NO));
- doc.add(new NumericDocValuesField("ndv1", i));
- doc.add(new NumericDocValuesField("ndv1_c", i*2));
- doc.add(new NumericDocValuesField("ndv2", i*3));
- doc.add(new NumericDocValuesField("ndv2_c", i*6));
- doc.add(new BinaryDocValuesField("bdv1", toBytes(i)));
- doc.add(new BinaryDocValuesField("bdv1_c", toBytes(i*2)));
- doc.add(new BinaryDocValuesField("bdv2", toBytes(i*3)));
- doc.add(new BinaryDocValuesField("bdv2_c", toBytes(i*6)));
+ Document doc = writer.newDocument();
+ doc.addAtom("id", "" + i);
+ doc.addInt("ndv1", i);
+ doc.addInt("ndv1_c", i*2);
+ doc.addInt("ndv2", i*3);
+ doc.addInt("ndv2_c", i*6);
+ doc.addBinary("bdv1", toBytes(i));
+ doc.addBinary("bdv1_c", toBytes(i*2));
+ doc.addBinary("bdv2", toBytes(i*3));
+ doc.addBinary("bdv2_c", toBytes(i*6));
writer.addDocument(doc);
if ((i+1) % 10 == 0) {
writer.commit(); // flush every 10 docs
@@ -624,7 +618,7 @@ public class TestBackwardsCompatibility
for(int i=0;i<35;i++) {
if (liveDocs.get(i)) {
- Document2 d = reader.document(i);
+ Document d = reader.document(i);
List<IndexableField> fields = d.getFields();
boolean isProxDoc = d.getField("content3") == null;
if (isProxDoc) {
@@ -726,7 +720,7 @@ public class TestBackwardsCompatibility
ScoreDoc[] hits = searcher.search(new TermQuery(new Term(new String("content"), "aaa")), null, 1000).scoreDocs;
// First document should be #0
- Document2 d = searcher.getIndexReader().document(hits[0].doc);
+ Document d = searcher.getIndexReader().document(hits[0].doc);
assertEquals("didn't get the right document first", "0", d.getString("id"));
doTestHits(hits, 34, searcher.getIndexReader());
@@ -770,7 +764,7 @@ public class TestBackwardsCompatibility
IndexReader reader = DirectoryReader.open(dir);
IndexSearcher searcher = newSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
- Document2 d = searcher.getIndexReader().document(hits[0].doc);
+ Document d = searcher.getIndexReader().document(hits[0].doc);
assertEquals("wrong first document", "0", d.getString("id"));
doTestHits(hits, 44, searcher.getIndexReader());
reader.close();
@@ -798,7 +792,7 @@ public class TestBackwardsCompatibility
IndexSearcher searcher = newSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
assertEquals("wrong number of hits", 34, hits.length);
- Document2 d = searcher.doc(hits[0].doc);
+ Document d = searcher.doc(hits[0].doc);
assertEquals("wrong first document", "0", d.getString("id"));
reader.close();
@@ -859,57 +853,56 @@ public class TestBackwardsCompatibility
dir.close();
}
- private void addDoc(IndexWriter writer, int id) throws IOException
- {
- Document doc = new Document();
- doc.add(new TextField("content", "aaa", Field.Store.NO));
- doc.add(new StringField("id", Integer.toString(id), Field.Store.YES));
- FieldType customType2 = new FieldType(TextField.TYPE_STORED);
- customType2.setStoreTermVectors(true);
- customType2.setStoreTermVectorPositions(true);
- customType2.setStoreTermVectorOffsets(true);
- doc.add(new Field("autf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));
- doc.add(new Field("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));
- doc.add(new Field("content2", "here is more content with aaa aaa aaa", customType2));
- doc.add(new Field("fie\u2C77ld", "field with non-ascii name", customType2));
+ private void addDoc(IndexWriter writer, int id) throws IOException {
+ FieldTypes fieldTypes = writer.getFieldTypes();
+ for(String fieldName : new String[] {"autf8", "utf8", "content2", "fie\u2C77ld", "content5", "content6"}) {
+ fieldTypes.enableTermVectors(fieldName);
+ fieldTypes.enableTermVectorPositions(fieldName);
+ fieldTypes.enableTermVectorOffsets(fieldName);
+ }
+ fieldTypes.setIndexOptions("content6", IndexOptions.DOCS_AND_FREQS);
+ fieldTypes.disableSorting("dvBytesDerefFixed");
+ fieldTypes.disableSorting("dvBytesDerefVar");
+ fieldTypes.disableSorting("dvBytesStraightFixed");
+ fieldTypes.disableSorting("dvBytesStraightVar");
+ fieldTypes.setMultiValued("dvSortedSet");
+ fieldTypes.setMultiValued("dvSortedNumeric");
+
+ Document doc = writer.newDocument();
+ doc.addLargeText("content", "aaa");
+ doc.addAtom("id", Integer.toString(id));
+ doc.addLargeText("autf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd");
+ doc.addLargeText("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd");
+ doc.addLargeText("content2", "here is more content with aaa aaa aaa");
+ doc.addLargeText("fie\u2C77ld", "field with non-ascii name");
// add numeric fields, to test if flex preserves encoding
// nocommit get these into back compat index
//doc.add(new IntField("trieInt", id, Field.Store.NO));
//doc.add(new LongField("trieLong", (long) id, Field.Store.NO));
// add docvalues fields
- doc.add(new NumericDocValuesField("dvByte", (byte) id));
+ doc.addInt("dvByte", (byte) id);
byte bytes[] = new byte[] {
(byte)(id >>> 24), (byte)(id >>> 16),(byte)(id >>> 8),(byte)id
};
BytesRef ref = new BytesRef(bytes);
- doc.add(new BinaryDocValuesField("dvBytesDerefFixed", ref));
- doc.add(new BinaryDocValuesField("dvBytesDerefVar", ref));
- doc.add(new SortedDocValuesField("dvBytesSortedFixed", ref));
- doc.add(new SortedDocValuesField("dvBytesSortedVar", ref));
- doc.add(new BinaryDocValuesField("dvBytesStraightFixed", ref));
- doc.add(new BinaryDocValuesField("dvBytesStraightVar", ref));
- doc.add(new DoubleDocValuesField("dvDouble", (double)id));
- doc.add(new FloatDocValuesField("dvFloat", (float)id));
- doc.add(new NumericDocValuesField("dvInt", id));
- doc.add(new NumericDocValuesField("dvLong", id));
- doc.add(new NumericDocValuesField("dvPacked", id));
- doc.add(new NumericDocValuesField("dvShort", (short)id));
- doc.add(new SortedSetDocValuesField("dvSortedSet", ref));
- doc.add(new SortedNumericDocValuesField("dvSortedNumeric", id));
+ doc.addBinary("dvBytesDerefFixed", ref);
+ doc.addBinary("dvBytesDerefVar", ref);
+ doc.addAtom("dvBytesSortedFixed", ref);
+ doc.addAtom("dvBytesSortedVar", ref);
+ doc.addBinary("dvBytesStraightFixed", ref);
+ doc.addBinary("dvBytesStraightVar", ref);
+ doc.addDouble("dvDouble", (double) id);
+ doc.addFloat("dvFloat", (float) id);
+ doc.addInt("dvInt", id);
+ doc.addLong("dvLong", id);
+ doc.addInt("dvPacked", id);
+ doc.addInt("dvShort", (short) id);
+ doc.addAtom("dvSortedSet", ref);
+ doc.addInt("dvSortedNumeric", id);
// a field with both offsets and term vectors for a cross-check
- FieldType customType3 = new FieldType(TextField.TYPE_STORED);
- customType3.setStoreTermVectors(true);
- customType3.setStoreTermVectorPositions(true);
- customType3.setStoreTermVectorOffsets(true);
- customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- doc.add(new Field("content5", "here is more content with aaa aaa aaa", customType3));
+ doc.addLargeText("content5", "here is more content with aaa aaa aaa");
// a field that omits only positions
- FieldType customType4 = new FieldType(TextField.TYPE_STORED);
- customType4.setStoreTermVectors(true);
- customType4.setStoreTermVectorPositions(false);
- customType4.setStoreTermVectorOffsets(true);
- customType4.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
- doc.add(new Field("content6", "here is more content with aaa aaa aaa", customType4));
+ doc.addLargeText("content6", "here is more content with aaa aaa aaa");
// TODO:
// index different norms types via similarity (we use a random one currently?!)
// remove any analyzer randomness, explicitly add payloads for certain fields.
@@ -917,16 +910,9 @@ public class TestBackwardsCompatibility
}
private void addNoProxDoc(IndexWriter writer) throws IOException {
- Document doc = new Document();
- FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setIndexOptions(IndexOptions.DOCS);
- Field f = new Field("content3", "aaa", customType);
- doc.add(f);
- FieldType customType2 = new FieldType();
- customType2.setStored(true);
- customType2.setIndexOptions(IndexOptions.DOCS);
- f = new Field("content4", "aaa", customType2);
- doc.add(f);
+ Document doc = writer.newDocument();
+ doc.addAtom("content3", "aaa");
+ doc.addAtom("content4", "aaa");
writer.addDocument(doc);
}
@@ -988,7 +974,7 @@ public class TestBackwardsCompatibility
// first create a little index with the current code and get the version
Directory currentDir = newDirectory();
RandomIndexWriter riw = new RandomIndexWriter(random(), currentDir);
- riw.addDocument(new Document());
+ riw.addDocument(riw.newDocument());
riw.close();
DirectoryReader ir = DirectoryReader.open(currentDir);
SegmentReader air = (SegmentReader)ir.leaves().get(0).reader();
@@ -1040,7 +1026,7 @@ public class TestBackwardsCompatibility
for (int id=10; id<15; id++) {
ScoreDoc[] hits = searcher.search(NumericRangeQuery.newIntRange("trieInt", NumericUtils.PRECISION_STEP_DEFAULT_32, Integer.valueOf(id), Integer.valueOf(id), true, true), 100).scoreDocs;
assertEquals("wrong number of hits", 1, hits.length);
- Document2 d = searcher.doc(hits[0].doc);
+ Document d = searcher.doc(hits[0].doc);
assertEquals(String.valueOf(id), d.getString("id"));
hits = searcher.search(NumericRangeQuery.newLongRange("trieLong", NumericUtils.PRECISION_STEP_DEFAULT, Long.valueOf(id), Long.valueOf(id), true, true), 100).scoreDocs;
@@ -1300,7 +1286,7 @@ public class TestBackwardsCompatibility
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setOpenMode(OpenMode.APPEND));
- writer.addDocument(new Document());
+ writer.addDocument(writer.newDocument());
DirectoryReader r = DirectoryReader.open(writer, true);
writer.commit();
r.close();
@@ -1319,9 +1305,9 @@ public class TestBackwardsCompatibility
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setOpenMode(OpenMode.APPEND));
- writer.addDocument(new Document());
+ writer.addDocument(writer.newDocument());
writer.commit();
- writer.addDocument(new Document());
+ writer.addDocument(writer.newDocument());
writer.commit();
writer.close();
dir.close();
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java Sun Nov 30 11:07:09 2014
@@ -34,14 +34,8 @@ import java.util.TimeZone;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.benchmark.byTask.utils.Config;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType.NumericType;
-import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FieldTypes;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
/**
@@ -138,11 +132,11 @@ public class DocMaker implements Closeab
// create a doc
// use only part of the body, modify it to keep the rest (or use all if size==0).
// reset the docdata properties so they are not added more than once.
- private Document2 createDocument(IndexWriter w, DocData docData, int size, int cnt) throws UnsupportedEncodingException {
+ private Document createDocument(IndexWriter w, DocData docData, int size, int cnt) throws UnsupportedEncodingException {
applySchema(w);
final DocState ds = getDocState();
- final Document2 doc = w.newDocument();
+ final Document doc = w.newDocument();
int id;
if (r != null) {
@@ -285,10 +279,10 @@ public class DocMaker implements Closeab
* <code>reuseFields</code> was set to true, it will reuse {@link Document}
* and {@link Field} instances.
*/
- public Document2 makeDocument(IndexWriter w) throws Exception {
+ public Document makeDocument(IndexWriter w) throws Exception {
resetLeftovers();
DocData docData = source.getNextDocData(getDocState().docData);
- Document2 doc = createDocument(w, docData, 0, -1);
+ Document doc = createDocument(w, docData, 0, -1);
return doc;
}
@@ -296,7 +290,7 @@ public class DocMaker implements Closeab
* Same as {@link #makeDocument()}, only this method creates a document of the
* given size input by <code>size</code>.
*/
- public Document2 makeDocument(IndexWriter w, int size) throws Exception {
+ public Document makeDocument(IndexWriter w, int size) throws Exception {
LeftOver lvr = leftover.get();
if (lvr == null || lvr.docdata == null || lvr.docdata.getBody() == null
|| lvr.docdata.getBody().length() == 0) {
@@ -311,7 +305,7 @@ public class DocMaker implements Closeab
cnt = 0;
dd.setBody(dd2.getBody() + dd.getBody());
}
- Document2 doc = createDocument(w, dd, size, cnt);
+ Document doc = createDocument(w, dd, size, cnt);
if (dd.getBody() == null || dd.getBody().length() == 0) {
resetLeftovers();
} else {
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialDocMaker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialDocMaker.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialDocMaker.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialDocMaker.java Sun Nov 30 11:07:09 2014
@@ -24,11 +24,8 @@ import java.util.Random;
import java.util.Set;
import org.apache.lucene.benchmark.byTask.utils.Config;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
@@ -165,11 +162,11 @@ public class SpatialDocMaker extends Doc
}
@Override
- public Document2 makeDocument(IndexWriter w) throws Exception {
+ public Document makeDocument(IndexWriter w) throws Exception {
DocState docState = getDocState();
- Document2 doc = super.makeDocument(w);
+ Document doc = super.makeDocument(w);
// Set SPATIAL_FIELD from body
DocData docData = docState.docData;
@@ -197,7 +194,7 @@ public class SpatialDocMaker extends Doc
}
@Override
- public Document2 makeDocument(IndexWriter w, int size) throws Exception {
+ public Document makeDocument(IndexWriter w, int size) throws Exception {
//TODO consider abusing the 'size' notion to number of shapes per document
throw new UnsupportedOperationException();
}
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java Sun Nov 30 11:07:09 2014
@@ -21,7 +21,6 @@ import java.util.Locale;
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
@@ -45,7 +44,7 @@ public class AddDocTask extends PerfTask
@Override
public int doLogic() throws Exception {
- Document2 doc;
+ Document doc;
DocMaker docMaker = getRunData().getDocMaker();
IndexWriter iw = getRunData().getIndexWriter();
if (docSize > 0) {
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/BenchmarkHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/BenchmarkHighlighter.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/BenchmarkHighlighter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/BenchmarkHighlighter.java Sun Nov 30 11:07:09 2014
@@ -18,7 +18,6 @@ package org.apache.lucene.benchmark.byTa
*/
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
@@ -27,5 +26,5 @@ import org.apache.lucene.index.IndexRead
*/
public abstract class BenchmarkHighlighter {
public abstract int doHighlight(IndexReader reader, int doc, String field,
- Document2 document, Analyzer analyzer, String text) throws Exception;
+ Document document, Analyzer analyzer, String text) throws Exception;
}
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java Sun Nov 30 11:07:09 2014
@@ -26,7 +26,6 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.feeds.QueryMaker;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@@ -34,7 +33,6 @@ import org.apache.lucene.index.Indexable
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
@@ -96,7 +94,7 @@ public abstract class ReadTask extends P
// optionally warm and add num docs traversed to count
if (withWarm()) {
- Document2 doc = null;
+ Document doc = null;
Bits liveDocs = MultiFields.getLiveDocs(reader);
for (int m = 0; m < reader.maxDoc(); m++) {
if (null == liveDocs || liveDocs.get(m)) {
@@ -142,7 +140,7 @@ public abstract class ReadTask extends P
System.out.println("numDocs() = " + reader.numDocs());
for(int i=0;i<hits.scoreDocs.length;i++) {
final int docID = hits.scoreDocs[i].doc;
- final Document2 doc = reader.document(docID);
+ final Document doc = reader.document(docID);
System.out.println(" " + i + ": doc=" + docID + " score=" + hits.scoreDocs[i].score + " " + printHitsField + " =" + doc.getString(printHitsField));
}
}
@@ -163,7 +161,7 @@ public abstract class ReadTask extends P
int id = scoreDocs[m].doc;
res++;
if (retrieve) {
- Document2 document = retrieveDoc(reader, id);
+ Document document = retrieveDoc(reader, id);
res += document != null ? 1 : 0;
if (numHighlight > 0 && m < numHighlight) {
Collection<String> fieldsToHighlight = getFieldsToHighlight(document);
@@ -193,7 +191,7 @@ public abstract class ReadTask extends P
}
- protected Document2 retrieveDoc(IndexReader ir, int id) throws IOException {
+ protected Document retrieveDoc(IndexReader ir, int id) throws IOException {
return ir.document(id);
}
@@ -296,7 +294,7 @@ public abstract class ReadTask extends P
* @param document The Document
* @return A Collection of Field names (Strings)
*/
- protected Collection<String> getFieldsToHighlight(Document2 document) {
+ protected Collection<String> getFieldsToHighlight(Document document) {
List<IndexableField> fields = document.getFields();
Set<String> result = new HashSet<>(fields.size());
for (final IndexableField f : fields) {
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java Sun Nov 30 11:07:09 2014
@@ -25,10 +25,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
@@ -69,7 +66,7 @@ public class ReadTokensTask extends Perf
if (iw == null) {
iw = getPrivateWriter();
}
- Document2 doc = docMaker.makeDocument(iw);
+ Document doc = docMaker.makeDocument(iw);
List<IndexableField> fields = doc.getFields();
Analyzer analyzer = iw.getFieldTypes().getIndexAnalyzer();
@@ -79,7 +76,7 @@ public class ReadTokensTask extends Perf
field.name().equals(DocMaker.DATE_FIELD) ||
field.name().equals(DocMaker.TITLE_FIELD)) {
- final TokenStream stream = field.tokenStream(analyzer, null);
+ final TokenStream stream = field.tokenStream(null);
// reset the TokenStream to the first token
stream.reset();
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java Sun Nov 30 11:07:09 2014
@@ -25,7 +25,6 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.benchmark.byTask.PerfRunData;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
@@ -102,7 +101,7 @@ public class SearchTravRetHighlightTask
return new BenchmarkHighlighter(){
@Override
public int doHighlight(IndexReader reader, int doc, String field,
- Document2 document, Analyzer analyzer, String text) throws Exception {
+ Document document, Analyzer analyzer, String text) throws Exception {
TokenStream ts = TokenSources.getAnyTokenStream(reader, doc, field, document, analyzer);
TextFragment[] frag = highlighter.getBestTextFragments(ts, text, mergeContiguous, maxFrags);
return frag != null ? frag.length : 0;
@@ -111,7 +110,7 @@ public class SearchTravRetHighlightTask
}
@Override
- protected Collection<String> getFieldsToHighlight(Document2 document) {
+ protected Collection<String> getFieldsToHighlight(Document document) {
Collection<String> result = super.getFieldsToHighlight(document);
//if stored is false, then result will be empty, in which case just get all the param fields
if (paramFields.isEmpty() == false && result.isEmpty() == false) {
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetLoadFieldSelectorTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetLoadFieldSelectorTask.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetLoadFieldSelectorTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetLoadFieldSelectorTask.java Sun Nov 30 11:07:09 2014
@@ -22,9 +22,8 @@ import java.util.Set;
import java.util.StringTokenizer;
import org.apache.lucene.benchmark.byTask.PerfRunData;
-import org.apache.lucene.document.Document2;
-import org.apache.lucene.document.Document2StoredFieldVisitor;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Document2StoredFieldVisitor;
import org.apache.lucene.index.IndexReader;
/**
@@ -55,7 +54,7 @@ public class SearchTravRetLoadFieldSelec
@Override
- protected Document2 retrieveDoc(IndexReader ir, int id) throws IOException {
+ protected Document retrieveDoc(IndexReader ir, int id) throws IOException {
if (fieldsToLoad == null) {
return ir.document(id);
} else {
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java Sun Nov 30 11:07:09 2014
@@ -24,7 +24,6 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.benchmark.byTask.PerfRunData;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
@@ -100,7 +99,7 @@ public class SearchTravRetVectorHighligh
return new BenchmarkHighlighter(){
@Override
public int doHighlight(IndexReader reader, int doc, String field,
- Document2 document, Analyzer analyzer, String text) throws Exception {
+ Document document, Analyzer analyzer, String text) throws Exception {
final FieldQuery fq = highlighter.getFieldQuery( myq, reader);
String[] fragments = highlighter.getBestFragments(fq, reader, doc, field, fragSize, maxFrags);
return fragments != null ? fragments.length : 0;
@@ -109,7 +108,7 @@ public class SearchTravRetVectorHighligh
}
@Override
- protected Collection<String> getFieldsToHighlight(Document2 document) {
+ protected Collection<String> getFieldsToHighlight(Document document) {
Collection<String> result = super.getFieldsToHighlight(document);
//if stored is false, then result will be empty, in which case just get all the param fields
if (paramFields.isEmpty() == false && result.isEmpty() == false) {
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/UpdateDocTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/UpdateDocTask.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/UpdateDocTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/UpdateDocTask.java Sun Nov 30 11:07:09 2014
@@ -19,7 +19,6 @@ package org.apache.lucene.benchmark.byTa
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
@@ -40,7 +39,7 @@ public class UpdateDocTask extends PerfT
@Override
public int doLogic() throws Exception {
- Document2 doc;
+ Document doc;
DocMaker docMaker = getRunData().getDocMaker();
IndexWriter iw = getRunData().getIndexWriter();
if (docSize > 0) {
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java Sun Nov 30 11:07:09 2014
@@ -12,7 +12,6 @@ import java.nio.file.Paths;
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
import org.apache.lucene.benchmark.byTask.utils.StreamUtils;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
@@ -63,7 +62,7 @@ public class WriteEnwikiLineDocTask exte
}
@Override
- protected PrintWriter lineFileOut(Document2 doc) {
+ protected PrintWriter lineFileOut(Document doc) {
IndexableField titleField = doc.getField(DocMaker.TITLE_FIELD);
if (titleField!=null && titleField.stringValue().startsWith("Category:")) {
return categoryLineFileOut;
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java Sun Nov 30 11:07:09 2014
@@ -32,7 +32,6 @@ import org.apache.lucene.benchmark.byTas
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.benchmark.byTask.utils.StreamUtils;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -176,7 +175,7 @@ public class WriteLineDocTask extends Pe
if (iw == null) {
iw = getPrivateWriter();
}
- Document2 doc = docSize > 0 ? docMaker.makeDocument(iw,docSize) : docMaker.makeDocument(iw);
+ Document doc = docSize > 0 ? docMaker.makeDocument(iw,docSize) : docMaker.makeDocument(iw);
Matcher matcher = threadNormalizer.get();
if (matcher == null) {
@@ -211,7 +210,7 @@ public class WriteLineDocTask extends Pe
* Selects output line file by written doc.
* Default: original output line file.
*/
- protected PrintWriter lineFileOut(Document2 doc) {
+ protected PrintWriter lineFileOut(Document doc) {
return lineFileOut;
}
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java Sun Nov 30 11:07:09 2014
@@ -30,7 +30,6 @@ import org.apache.lucene.benchmark.byTas
import org.apache.lucene.benchmark.byTask.feeds.EnwikiContentSource;
import org.apache.lucene.benchmark.byTask.feeds.NoMoreDataException;
import org.apache.lucene.benchmark.byTask.utils.Config;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -93,7 +92,7 @@ public class ExtractWikipedia {
}
public void extract() throws Exception {
- Document2 doc = null;
+ Document doc = null;
System.out.println("Starting Extraction");
long start = System.currentTimeMillis();
RAMDirectory dir = new RAMDirectory();
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/DocMakerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/DocMakerTest.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/DocMakerTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/DocMakerTest.java Sun Nov 30 11:07:09 2014
@@ -30,7 +30,6 @@ import org.apache.lucene.benchmark.byTas
import org.apache.lucene.benchmark.byTask.tasks.CreateIndexTask;
import org.apache.lucene.benchmark.byTask.tasks.TaskSequence;
import org.apache.lucene.benchmark.byTask.utils.Config;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@@ -102,7 +101,7 @@ public class DocMakerTest extends Benchm
reader.close();
}
- private Document2 createTestNormsDocument(boolean setNormsProp,
+ private Document createTestNormsDocument(boolean setNormsProp,
boolean normsPropVal, boolean setBodyNormsProp, boolean bodyNormsVal)
throws Exception {
Properties props = new Properties();
@@ -124,7 +123,7 @@ public class DocMakerTest extends Benchm
RAMDirectory dir = new RAMDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
dm.setConfig(config, new OneDocSource());
- Document2 doc = dm.makeDocument(w);
+ Document doc = dm.makeDocument(w);
w.close();
dir.close();
return doc;
@@ -145,7 +144,7 @@ public class DocMakerTest extends Benchm
/* Tests doc.tokenized.norms and doc.body.tokenized.norms properties. */
public void testNorms() throws Exception {
- Document2 doc;
+ Document doc;
// Don't set anything, use the defaults
doc = createTestNormsDocument(false, false, false, false);
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/AddIndexesTaskTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/AddIndexesTaskTest.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/AddIndexesTaskTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/AddIndexesTaskTest.java Sun Nov 30 11:07:09 2014
@@ -23,7 +23,6 @@ import java.util.Properties;
import org.apache.lucene.benchmark.BenchmarkTestCase;
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.utils.Config;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
@@ -48,7 +47,7 @@ public class AddIndexesTaskTest extends
try {
IndexWriter writer = new IndexWriter(tmpDir, new IndexWriterConfig(null));
for (int i = 0; i < 10; i++) {
- writer.addDocument(new Document());
+ writer.addDocument(writer.newDocument());
}
writer.close();
} finally {
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CountingHighlighterTestTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CountingHighlighterTestTask.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CountingHighlighterTestTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CountingHighlighterTestTask.java Sun Nov 30 11:07:09 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.benchmark.byTask.PerfRunData;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
@@ -45,8 +44,8 @@ public class CountingHighlighterTestTask
}
@Override
- protected Document2 retrieveDoc(IndexReader ir, int id) throws IOException {
- Document2 document = ir.document(id);
+ protected Document retrieveDoc(IndexReader ir, int id) throws IOException {
+ Document document = ir.document(id);
if (document != null) {
numDocsRetrieved++;
}
@@ -58,7 +57,7 @@ public class CountingHighlighterTestTask
highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(q));
return new BenchmarkHighlighter() {
@Override
- public int doHighlight(IndexReader reader, int doc, String field, Document2 document, Analyzer analyzer, String text) throws Exception {
+ public int doHighlight(IndexReader reader, int doc, String field, Document document, Analyzer analyzer, String text) throws Exception {
TokenStream ts = TokenSources.getAnyTokenStream(reader, doc, field, document, analyzer);
TextFragment[] frag = highlighter.getBestTextFragments(ts, text, mergeContiguous, maxFrags);
numHighlightedResults += frag != null ? frag.length : 0;
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTaskTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTaskTest.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTaskTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTaskTest.java Sun Nov 30 11:07:09 2014
@@ -32,10 +32,7 @@ import org.apache.lucene.benchmark.Bench
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
import org.apache.lucene.benchmark.byTask.utils.Config;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
/** Tests the functionality of {@link WriteEnwikiLineDocTask}. */
@@ -49,9 +46,9 @@ public class WriteEnwikiLineDocTaskTest
AtomicInteger flip = new AtomicInteger(0);
@Override
- public Document2 makeDocument(IndexWriter w) throws Exception {
+ public Document makeDocument(IndexWriter w) throws Exception {
boolean isCategory = (flip.incrementAndGet() % 2 == 0);
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
doc.addAtom(BODY_FIELD, "body text");
doc.addAtom(TITLE_FIELD, isCategory ? "Category:title text" : "title text");
doc.addAtom(DATE_FIELD, "date text");
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java Sun Nov 30 11:07:09 2014
@@ -33,10 +33,7 @@ import org.apache.lucene.benchmark.byTas
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.benchmark.byTask.utils.StreamUtils.Type;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
/** Tests the functionality of {@link WriteLineDocTask}. */
@@ -46,8 +43,8 @@ public class WriteLineDocTaskTest extend
public static final class WriteLineDocMaker extends DocMaker {
@Override
- public Document2 makeDocument(IndexWriter w) throws Exception {
- Document2 doc = w.newDocument();
+ public Document makeDocument(IndexWriter w) throws Exception {
+ Document doc = w.newDocument();
doc.addAtom(BODY_FIELD, "body");
doc.addAtom(TITLE_FIELD, "title");
doc.addAtom(DATE_FIELD, "date");
@@ -60,8 +57,8 @@ public class WriteLineDocTaskTest extend
public static final class NewLinesDocMaker extends DocMaker {
@Override
- public Document2 makeDocument(IndexWriter w) throws Exception {
- Document2 doc = w.newDocument();
+ public Document makeDocument(IndexWriter w) throws Exception {
+ Document doc = w.newDocument();
doc.addAtom(BODY_FIELD, "body\r\ntext\ttwo");
doc.addAtom(TITLE_FIELD, "title\r\ntext");
doc.addAtom(DATE_FIELD, "date\r\ntext");
@@ -73,8 +70,8 @@ public class WriteLineDocTaskTest extend
// class has to be public so that Class.forName.newInstance() will work
public static final class NoBodyDocMaker extends DocMaker {
@Override
- public Document2 makeDocument(IndexWriter w) throws Exception {
- Document2 doc = w.newDocument();
+ public Document makeDocument(IndexWriter w) throws Exception {
+ Document doc = w.newDocument();
doc.addAtom(TITLE_FIELD, "title");
doc.addAtom(DATE_FIELD, "date");
return doc;
@@ -84,8 +81,8 @@ public class WriteLineDocTaskTest extend
// class has to be public so that Class.forName.newInstance() will work
public static final class NoTitleDocMaker extends DocMaker {
@Override
- public Document2 makeDocument(IndexWriter w) throws Exception {
- Document2 doc = w.newDocument();
+ public Document makeDocument(IndexWriter w) throws Exception {
+ Document doc = w.newDocument();
doc.addAtom(BODY_FIELD, "body");
doc.addAtom(DATE_FIELD, "date");
return doc;
@@ -95,8 +92,8 @@ public class WriteLineDocTaskTest extend
// class has to be public so that Class.forName.newInstance() will work
public static final class JustDateDocMaker extends DocMaker {
@Override
- public Document2 makeDocument(IndexWriter w) throws Exception {
- Document2 doc = w.newDocument();
+ public Document makeDocument(IndexWriter w) throws Exception {
+ Document doc = w.newDocument();
doc.addAtom(DATE_FIELD, "date");
return doc;
}
@@ -106,8 +103,8 @@ public class WriteLineDocTaskTest extend
// same as JustDate just that this one is treated as legal
public static final class LegalJustDateDocMaker extends DocMaker {
@Override
- public Document2 makeDocument(IndexWriter w) throws Exception {
- Document2 doc = w.newDocument();
+ public Document makeDocument(IndexWriter w) throws Exception {
+ Document doc = w.newDocument();
doc.addAtom(DATE_FIELD, "date");
return doc;
}
@@ -116,7 +113,7 @@ public class WriteLineDocTaskTest extend
// class has to be public so that Class.forName.newInstance() will work
public static final class EmptyDocMaker extends DocMaker {
@Override
- public Document2 makeDocument(IndexWriter w) throws Exception {
+ public Document makeDocument(IndexWriter w) throws Exception {
return w.newDocument();
}
}
@@ -125,8 +122,8 @@ public class WriteLineDocTaskTest extend
public static final class ThreadingDocMaker extends DocMaker {
@Override
- public Document2 makeDocument(IndexWriter w) throws Exception {
- Document2 doc = w.newDocument();
+ public Document makeDocument(IndexWriter w) throws Exception {
+ Document doc = w.newDocument();
String name = Thread.currentThread().getName();
doc.addAtom(BODY_FIELD, "body_" + name);
doc.addAtom(TITLE_FIELD, "title_" + name);
Modified: lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java (original)
+++ lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java Sun Nov 30 11:07:09 2014
@@ -25,7 +25,7 @@ import java.util.concurrent.ConcurrentSk
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.document.Document2;
+import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.MultiFields;
@@ -171,7 +171,7 @@ public class BooleanPerceptronClassifier
// run the search and use stored field values
for (ScoreDoc scoreDoc : indexSearcher.search(q,
Integer.MAX_VALUE).scoreDocs) {
- Document2 doc = indexSearcher.doc(scoreDoc.doc);
+ Document doc = indexSearcher.doc(scoreDoc.doc);
// assign class to the doc
ClassificationResult<Boolean> classificationResult = assignClass(doc
Modified: lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java Sun Nov 30 11:07:09 2014
@@ -21,9 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
@@ -72,49 +70,48 @@ public class DatasetSplitter {
IndexWriter cvWriter = new IndexWriter(crossValidationIndex, new IndexWriterConfig(analyzer));
IndexWriter trainingWriter = new IndexWriter(trainingIndex, new IndexWriterConfig(analyzer));
+ for (IndexWriter w : new IndexWriter[] {testWriter, cvWriter, trainingWriter}) {
+ FieldTypes fieldTypes = w.getFieldTypes();
+ for (String fieldName : fieldNames) {
+ fieldTypes.enableTermVectors(fieldName);
+ fieldTypes.enableTermVectorPositions(fieldName);
+ fieldTypes.enableTermVectorOffsets(fieldName);
+ }
+ }
+
try {
int size = originalIndex.maxDoc();
IndexSearcher indexSearcher = new IndexSearcher(originalIndex);
TopDocs topDocs = indexSearcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE);
- // set the type to be indexed, stored, with term vectors
- FieldType ft = new FieldType(TextField.TYPE_STORED);
- ft.setStoreTermVectors(true);
- ft.setStoreTermVectorOffsets(true);
- ft.setStoreTermVectorPositions(true);
-
int b = 0;
// iterate over existing documents
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+ IndexWriter w;
+ if (b % 2 == 0 && testWriter.maxDoc() < size * testRatio) {
+ w = testWriter;
+ } else if (cvWriter.maxDoc() < size * crossValidationRatio) {
+ w = cvWriter;
+ } else {
+ w = trainingWriter;
+ }
// create a new document for indexing
- Document doc = new Document();
+ Document doc = w.newDocument();
if (fieldNames != null && fieldNames.length > 0) {
for (String fieldName : fieldNames) {
- doc.add(new Field(fieldName, originalIndex.document(scoreDoc.doc).getField(fieldName).stringValue(), ft));
+ doc.addLargeText(fieldName, originalIndex.document(scoreDoc.doc).getField(fieldName).stringValue());
}
} else {
for (IndexableField storableField : originalIndex.document(scoreDoc.doc).getFields()) {
- if (storableField.binaryValue() != null) {
- doc.add(new Field(storableField.name(), storableField.binaryValue(), ft));
- } else if (storableField.stringValue() != null) {
- doc.add(new Field(storableField.name(), storableField.stringValue(), ft));
- } else if (storableField.numericValue() != null) {
- doc.add(new Field(storableField.name(), storableField.numericValue().toString(), ft));
- }
+ doc.addLargeText(storableField.name(), storableField.stringValue());
}
}
// add it to one of the IDXs
- if (b % 2 == 0 && testWriter.maxDoc() < size * testRatio) {
- testWriter.addDocument(doc);
- } else if (cvWriter.maxDoc() < size * crossValidationRatio) {
- cvWriter.addDocument(doc);
- } else {
- trainingWriter.addDocument(doc);
- }
+ w.addDocument(doc);
b++;
}
} catch (Exception e) {
Modified: lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java Sun Nov 30 11:07:09 2014
@@ -21,9 +21,7 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.RandomIndexWriter;
@@ -50,7 +48,6 @@ public abstract class ClassificationTest
private RandomIndexWriter indexWriter;
private Directory dir;
- private FieldType ft;
String textFieldName;
String categoryFieldName;
@@ -65,10 +62,12 @@ public abstract class ClassificationTest
textFieldName = "text";
categoryFieldName = "cat";
booleanFieldName = "bool";
- ft = new FieldType(TextField.TYPE_STORED);
- ft.setStoreTermVectors(true);
- ft.setStoreTermVectorOffsets(true);
- ft.setStoreTermVectorPositions(true);
+ FieldTypes fieldTypes = indexWriter.getFieldTypes();
+ for(String fieldName : new String[] {textFieldName, categoryFieldName, booleanFieldName}) {
+ fieldTypes.enableTermVectors(fieldName);
+ fieldTypes.enableTermVectorOffsets(fieldName);
+ fieldTypes.enableTermVectorPositions(fieldName);
+ }
}
@Override
@@ -131,73 +130,80 @@ public abstract class ClassificationTest
indexWriter = new RandomIndexWriter(random(), dir, newIndexWriterConfig(analyzer).setOpenMode(IndexWriterConfig.OpenMode.CREATE));
indexWriter.commit();
+ FieldTypes fieldTypes = indexWriter.getFieldTypes();
+ for(String fieldName : new String[] {textFieldName, categoryFieldName, booleanFieldName}) {
+ fieldTypes.enableTermVectors(fieldName);
+ fieldTypes.enableTermVectorOffsets(fieldName);
+ fieldTypes.enableTermVectorPositions(fieldName);
+ }
+
String text;
- Document doc = new Document();
+ Document doc = indexWriter.newDocument();
text = "The traveling press secretary for Mitt Romney lost his cool and cursed at reporters " +
"who attempted to ask questions of the Republican presidential candidate in a public plaza near the Tomb of " +
"the Unknown Soldier in Warsaw Tuesday.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "politics", ft));
- doc.add(new Field(booleanFieldName, "true", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "politics");
+ doc.addLargeText(booleanFieldName, "true");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "Mitt Romney seeks to assure Israel and Iran, as well as Jewish voters in the United" +
" States, that he will be tougher against Iran's nuclear ambitions than President Barack Obama.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "politics", ft));
- doc.add(new Field(booleanFieldName, "true", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "politics");
+ doc.addLargeText(booleanFieldName, "true");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "And there's a threshold question that he has to answer for the American people and " +
"that's whether he is prepared to be commander-in-chief,\" she continued. \"As we look to the past events, we " +
"know that this raises some questions about his preparedness and we'll see how the rest of his trip goes.\"";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "politics", ft));
- doc.add(new Field(booleanFieldName, "true", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "politics");
+ doc.addLargeText(booleanFieldName, "true");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "Still, when it comes to gun policy, many congressional Democrats have \"decided to " +
"keep quiet and not go there,\" said Alan Lizotte, dean and professor at the State University of New York at " +
"Albany's School of Criminal Justice.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "politics", ft));
- doc.add(new Field(booleanFieldName, "true", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "politics");
+ doc.addLargeText(booleanFieldName, "true");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "Standing amongst the thousands of people at the state Capitol, Jorstad, director of " +
"technology at the University of Wisconsin-La Crosse, documented the historic moment and shared it with the " +
"world through the Internet.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "technology", ft));
- doc.add(new Field(booleanFieldName, "false", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "technology");
+ doc.addLargeText(booleanFieldName, "false");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "So, about all those experts and analysts who've spent the past year or so saying " +
"Facebook was going to make a phone. A new expert has stepped forward to say it's not going to happen.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "technology", ft));
- doc.add(new Field(booleanFieldName, "false", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "technology");
+ doc.addLargeText(booleanFieldName, "false");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "More than 400 million people trust Google with their e-mail, and 50 million store files" +
" in the cloud using the Dropbox service. People manage their bank accounts, pay bills, trade stocks and " +
"generally transfer or store huge volumes of personal data online.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "technology", ft));
- doc.add(new Field(booleanFieldName, "false", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "technology");
+ doc.addLargeText(booleanFieldName, "false");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "unlabeled doc";
- doc.add(new Field(textFieldName, text, ft));
+ doc.addLargeText(textFieldName, text);
indexWriter.addDocument(doc);
indexWriter.commit();
@@ -224,18 +230,21 @@ public abstract class ClassificationTest
indexWriter = new RandomIndexWriter(random(), dir, newIndexWriterConfig(analyzer).setOpenMode(IndexWriterConfig.OpenMode.CREATE));
indexWriter.commit();
- FieldType ft = new FieldType(TextField.TYPE_STORED);
- ft.setStoreTermVectors(true);
- ft.setStoreTermVectorOffsets(true);
- ft.setStoreTermVectorPositions(true);
+ FieldTypes fieldTypes = indexWriter.getFieldTypes();
+ for(String fieldName : new String[] {textFieldName, categoryFieldName, booleanFieldName}) {
+ fieldTypes.enableTermVectors(fieldName);
+ fieldTypes.enableTermVectorOffsets(fieldName);
+ fieldTypes.enableTermVectorPositions(fieldName);
+ }
+
int docs = 1000;
Random random = random();
for (int i = 0; i < docs; i++) {
boolean b = random.nextBoolean();
- Document doc = new Document();
- doc.add(new Field(textFieldName, createRandomString(random), ft));
- doc.add(new Field(categoryFieldName, b ? "technology" : "politics", ft));
- doc.add(new Field(booleanFieldName, String.valueOf(b), ft));
+ Document doc = indexWriter.newDocument();
+ doc.addLargeText(textFieldName, createRandomString(random));
+ doc.addLargeText(categoryFieldName, b ? "technology" : "politics");
+ doc.addLargeText(booleanFieldName, String.valueOf(b));
indexWriter.addDocument(doc);
}
indexWriter.commit();
@@ -254,59 +263,59 @@ public abstract class ClassificationTest
String text;
- Document doc = new Document();
+ Document doc = indexWriter.newDocument();
text = "Warren Bennis says John F. Kennedy grasped a key lesson about the presidency that few have followed.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "politics", ft));
- doc.add(new Field(booleanFieldName, "true", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "politics");
+ doc.addLargeText(booleanFieldName, "true");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "Julian Zelizer says Bill Clinton is still trying to shape his party, years after the White House, while George W. Bush opts for a much more passive role.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "politics", ft));
- doc.add(new Field(booleanFieldName, "true", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "politics");
+ doc.addLargeText(booleanFieldName, "true");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "Crossfire: Sen. Tim Scott passes on Sen. Lindsey Graham endorsement";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "politics", ft));
- doc.add(new Field(booleanFieldName, "true", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "politics");
+ doc.addLargeText(booleanFieldName, "true");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "Illinois becomes 16th state to allow same-sex marriage.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "politics", ft));
- doc.add(new Field(booleanFieldName, "true", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "politics");
+ doc.addLargeText(booleanFieldName, "true");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "Apple is developing iPhones with curved-glass screens and enhanced sensors that detect different levels of pressure, according to a new report.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "technology", ft));
- doc.add(new Field(booleanFieldName, "false", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "technology");
+ doc.addLargeText(booleanFieldName, "false");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "The Xbox One is Microsoft's first new gaming console in eight years. It's a quality piece of hardware but it's also noteworthy because Microsoft is using it to make a statement.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "technology", ft));
- doc.add(new Field(booleanFieldName, "false", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "technology");
+ doc.addLargeText(booleanFieldName, "false");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "Google says it will replace a Google Maps image after a California father complained it shows the body of his teen-age son, who was shot to death in 2009.";
- doc.add(new Field(textFieldName, text, ft));
- doc.add(new Field(categoryFieldName, "technology", ft));
- doc.add(new Field(booleanFieldName, "false", ft));
+ doc.addLargeText(textFieldName, text);
+ doc.addLargeText(categoryFieldName, "technology");
+ doc.addLargeText(booleanFieldName, "false");
indexWriter.addDocument(doc);
- doc = new Document();
+ doc = indexWriter.newDocument();
text = "second unlabeled doc";
- doc.add(new Field(textFieldName, text, ft));
+ doc.addLargeText(textFieldName, text);
indexWriter.addDocument(doc);
indexWriter.commit();
Modified: lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java Sun Nov 30 11:07:09 2014
@@ -20,14 +20,9 @@ package org.apache.lucene.classification
import java.io.IOException;
import java.util.Random;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FieldTypes;
-import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
@@ -76,7 +71,7 @@ public class DataSplitterTest extends Lu
Random rnd = random();
for (int i = 0; i < 100; i++) {
- Document2 doc = indexWriter.newDocument();
+ Document doc = indexWriter.newDocument();
doc.addAtom(idFieldName, Integer.toString(i));
doc.addLargeText(textFieldName, TestUtil.randomUnicodeString(rnd, 1024));
doc.addLargeText(classFieldName, TestUtil.randomUnicodeString(rnd, 10));
Modified: lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/utils/DocToDoubleVectorUtilsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/utils/DocToDoubleVectorUtilsTest.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/utils/DocToDoubleVectorUtilsTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/utils/DocToDoubleVectorUtilsTest.java Sun Nov 30 11:07:09 2014
@@ -16,14 +16,8 @@
*/
package org.apache.lucene.classification.utils;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FieldTypes;
-import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.RandomIndexWriter;
@@ -61,7 +55,7 @@ public class DocToDoubleVectorUtilsTest
fieldTypes.enableTermVectorOffsets("text");
for (int i = 0; i < 10; i++) {
- Document2 doc = indexWriter.newDocument();
+ Document doc = indexWriter.newDocument();
doc.addAtom("id", Integer.toString(i));
doc.addLargeText("text", random().nextInt(10) + " " + random().nextInt(10) + " " + random().nextInt(10));
indexWriter.addDocument(doc);
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesFormat.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesFormat.java Sun Nov 30 11:07:09 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.document.SortedSetDocValuesField; // javadocs
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.ArrayUtil;
@@ -45,7 +44,7 @@ import org.apache.lucene.util.ArrayUtil;
* document's set of values cannot exceed about 2.1 B
* values (see #MAX_SORTED_SET_ORDS). For example,
* if every document has 10 values (10 instances of
- * {@link SortedSetDocValuesField}) added, then no
+ * {@link Document2#addAtom}) added, then no
* more than ~210 M documents can be added to one
* segment. </li>
* </ul> */
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/TestOrdsBlockTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/TestOrdsBlockTree.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/TestOrdsBlockTree.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/TestOrdsBlockTree.java Sun Nov 30 11:07:09 2014
@@ -24,9 +24,7 @@ import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.index.BasePostingsFormatTestCase;
import org.apache.lucene.index.DirectoryReader;
@@ -51,7 +49,7 @@ public class TestOrdsBlockTree extends B
public void testBasic() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
doc.addLargeText("field", "a b c");
w.addDocument(doc);
IndexReader r = w.getReader();
@@ -92,7 +90,7 @@ public class TestOrdsBlockTree extends B
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
List<String> terms = new ArrayList<>();
for(int i=0;i<36;i++) {
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
String term = "" + (char) (97+i);
terms.add(term);
if (VERBOSE) {
@@ -102,7 +100,7 @@ public class TestOrdsBlockTree extends B
w.addDocument(doc);
}
for(int i=0;i<36;i++) {
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
String term = "m" + (char) (97+i);
terms.add(term);
if (VERBOSE) {
@@ -161,7 +159,7 @@ public class TestOrdsBlockTree extends B
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
List<String> terms = new ArrayList<>();
for(int i=0;i<36;i++) {
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
String term = "" + (char) (97+i);
terms.add(term);
if (VERBOSE) {
@@ -171,7 +169,7 @@ public class TestOrdsBlockTree extends B
w.addDocument(doc);
}
for(int i=0;i<36;i++) {
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
String term = "m" + (char) (97+i);
terms.add(term);
if (VERBOSE) {
@@ -181,7 +179,7 @@ public class TestOrdsBlockTree extends B
w.addDocument(doc);
}
for(int i=0;i<36;i++) {
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
String term = "mo" + (char) (97+i);
terms.add(term);
if (VERBOSE) {
@@ -242,7 +240,7 @@ public class TestOrdsBlockTree extends B
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
IndexWriter w = new IndexWriter(dir, iwc);
for(int i=0;i<128;i++) {
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
String term = "" + (char) i;
if (VERBOSE) {
System.out.println("i=" + i + " term=" + term + " bytes=" + new BytesRef(term));
@@ -281,7 +279,7 @@ public class TestOrdsBlockTree extends B
IndexWriter w = new IndexWriter(dir, iwc);
List<String> terms = new ArrayList<>();
for(int i=0;i<36;i++) {
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
String term = "" + (char) (97+i);
terms.add(term);
if (VERBOSE) {
@@ -291,7 +289,7 @@ public class TestOrdsBlockTree extends B
w.addDocument(doc);
}
for(int i=0;i<128;i++) {
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
String term = "m" + (char) i;
terms.add(term);
if (VERBOSE) {
@@ -328,7 +326,7 @@ public class TestOrdsBlockTree extends B
List<String> terms = new ArrayList<>();
for(int i=0;i<30;i++) {
for(int j=0;j<30;j++) {
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
String term = "" + (char) (97+i) + (char) (97+j);
terms.add(term);
if (VERBOSE) {
@@ -368,7 +366,7 @@ public class TestOrdsBlockTree extends B
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
FieldTypes fieldTypes = w.getFieldTypes();
fieldTypes.setMultiValued("field");
- Document2 doc = w.newDocument();
+ Document doc = w.newDocument();
// Get empty string in there!
doc.addAtom("field", "");
w.addDocument(doc);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java Sun Nov 30 11:07:09 2014
@@ -23,8 +23,6 @@ import java.lang.reflect.Modifier;
import org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeFactory;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java Sun Nov 30 11:07:09 2014
@@ -19,7 +19,7 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
-import org.apache.lucene.document.Document2;
+import org.apache.lucene.document.Document;
import org.apache.lucene.document.Document2StoredFieldVisitor;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
@@ -100,7 +100,7 @@ public abstract class StoredFieldsWriter
// fieldsWriter.addDocument; see LUCENE-1282
Document2StoredFieldVisitor visitor = new Document2StoredFieldVisitor(null);
storedFieldsReader.visitDocument(docID, visitor);
- Document2 doc = visitor.getDocument();
+ Document doc = visitor.getDocument();
addDocument(doc, mergeState.mergeFieldInfos);
docCount++;
mergeState.checkAbort.work(300);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Sun Nov 30 11:07:09 2014
@@ -251,8 +251,12 @@ public final class CompressingStoredFiel
final BytesRef bytes;
final String string;
+ // nocommit stored fields writer shouldn't have to do per-doc schema like this?
Number number = field.numericValue();
if (number != null) {
+ // nocommit this is sort of evil? it should be more low-schema, i.e. store a long encoding of the number and let high-schema decode
+ // appropriately, like how DVs work. e.g. DATE, BIG_INT, IP_ADDRESS work this way already ... we can't expect this impl to have to
+ // know about all future types
if (number instanceof Byte || number instanceof Short || number instanceof Integer) {
bits = NUMERIC_INT;
} else if (number instanceof Long) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/DateTools.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/DateTools.java?rev=1642535&r1=1642534&r2=1642535&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/DateTools.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/DateTools.java Sun Nov 30 11:07:09 2014
@@ -20,7 +20,6 @@ package org.apache.lucene.document;
import org.apache.lucene.search.NumericRangeQuery; // for javadocs
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.TermRangeQuery;
-import org.apache.lucene.util.NumericUtils; // for javadocs
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -40,9 +39,6 @@ import java.util.TimeZone;
* {@link TermRangeQuery} and {@link PrefixQuery} will require more memory and become slower.
*
* <P>
- * Another approach is {@link NumericUtils}, which provides
- * a sortable binary representation (prefix encoded) of numeric values, which
- * date/time are.
* For indexing a {@link Date} or {@link Calendar}, just get the unix timestamp as
* <code>long</code> using {@link Date#getTime} or {@link Calendar#getTimeInMillis} and
* index this as a numeric value with {@link LongField}