You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/07/05 00:26:55 UTC
svn commit: r1142831 [1/2] - in /lucene/dev/branches/fieldtype/lucene:
contrib/highlighter/src/test/org/apache/lucene/search/highlight/
contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/
contrib/instantiated/src/java/org/apache/luce...
Author: mikemccand
Date: Mon Jul 4 22:26:54 2011
New Revision: 1142831
URL: http://svn.apache.org/viewvc?rev=1142831&view=rev
Log:
LUCENE-2308: initial cutover of some tests to document2
Added:
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/BinaryField.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Document.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Field.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelector.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelectorResult.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldType.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/NumericField.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/StringField.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/TextField.java (with props)
Modified:
lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocument.java
lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java
lucene/dev/branches/fieldtype/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/TestDemo.java
Modified: lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java?rev=1142831&r1=1142830&r2=1142831&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Mon Jul 4 22:26:54 2011
@@ -26,11 +26,10 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document2.Document;
+import org.apache.lucene.document2.Field;
+import org.apache.lucene.document2.FieldType;
+import org.apache.lucene.document2.TextField;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
@@ -61,8 +60,11 @@ public class HighlighterPhraseTest exten
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamConcurrent(),
- TermVector.WITH_POSITIONS_OFFSETS));
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setStoreTermVectors(true);
+ document.add(new Field(FIELD, customType, new TokenStreamConcurrent()));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
@@ -105,8 +107,12 @@ public class HighlighterPhraseTest exten
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamConcurrent(),
- TermVector.WITH_POSITIONS_OFFSETS));
+
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setStoreTermVectors(true);
+ document.add(new Field(FIELD, customType, new TokenStreamConcurrent()));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
@@ -175,8 +181,12 @@ public class HighlighterPhraseTest exten
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamSparse(),
- TermVector.WITH_POSITIONS_OFFSETS));
+
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setStoreTermVectors(true);
+ document.add(new Field(FIELD, customType, new TokenStreamSparse()));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
@@ -218,8 +228,12 @@ public class HighlighterPhraseTest exten
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
- document.add(new Field(FIELD, TEXT, Store.YES, Index.ANALYZED,
- TermVector.WITH_OFFSETS));
+
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStored(true);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectors(true);
+ document.add(new Field(FIELD, customType, TEXT));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
@@ -259,8 +273,11 @@ public class HighlighterPhraseTest exten
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamSparse(),
- TermVector.WITH_POSITIONS_OFFSETS));
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setStoreTermVectors(true);
+ document.add(new Field(FIELD, customType, new TokenStreamSparse()));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
Modified: lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1142831&r1=1142830&r2=1142831&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Mon Jul 4 22:26:54 2011
@@ -42,11 +42,11 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.NumericField;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document2.Document;
+import org.apache.lucene.document2.Field;
+import org.apache.lucene.document2.FieldType;
+import org.apache.lucene.document2.NumericField;
+import org.apache.lucene.document2.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -112,7 +112,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document doc = searcher.doc(hits.scoreDocs[i].doc);
+ org.apache.lucene.document.Document doc = searcher.doc(hits.scoreDocs[i].doc);
String storedField = doc.get(FIELD_NAME);
TokenStream stream = TokenSources.getAnyTokenStream(searcher
@@ -1531,7 +1531,9 @@ public class HighlighterTest extends Bas
private Document doc( String f, String v ){
Document doc = new Document();
- doc.add( new Field( f, v, Store.YES, Index.ANALYZED ) );
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStored(true);
+ doc.add( new Field( f, customType, v));
return doc;
}
@@ -1566,7 +1568,7 @@ public class HighlighterTest extends Bas
TopDocs hits = searcher.search(query, null, 10);
for( int i = 0; i < hits.totalHits; i++ ){
- Document doc = searcher.doc( hits.scoreDocs[i].doc );
+ org.apache.lucene.document.Document doc = searcher.doc( hits.scoreDocs[i].doc );
String result = h.getBestFragment( a, "t_text1", doc.get( "t_text1" ));
if (VERBOSE) System.out.println("result:" + result);
assertEquals("more <B>random</B> words for second field", result);
@@ -1656,21 +1658,23 @@ public class HighlighterTest extends Bas
addDoc(writer, text);
}
Document doc = new Document();
- NumericField nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
+ FieldType storedNumericType = new FieldType(NumericField.DEFAULT_TYPE);
+ storedNumericType.setStored(true);
+ NumericField nfield = new NumericField(NUMERIC_FIELD_NAME, storedNumericType);
nfield.setIntValue(1);
doc.add(nfield);
writer.addDocument(doc, analyzer);
- nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
+ nfield = new NumericField(NUMERIC_FIELD_NAME, storedNumericType);
nfield.setIntValue(3);
doc = new Document();
doc.add(nfield);
writer.addDocument(doc, analyzer);
- nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
+ nfield = new NumericField(NUMERIC_FIELD_NAME, storedNumericType);
nfield.setIntValue(5);
doc = new Document();
doc.add(nfield);
writer.addDocument(doc, analyzer);
- nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
+ nfield = new NumericField(NUMERIC_FIELD_NAME, storedNumericType);
nfield.setIntValue(7);
doc = new Document();
doc.add(nfield);
@@ -1691,7 +1695,10 @@ public class HighlighterTest extends Bas
}
private void addDoc(IndexWriter writer, String text) throws IOException {
Document d = new Document();
- Field f = new Field(FIELD_NAME, text, Field.Store.YES, Field.Index.ANALYZED);
+
+ FieldType storedType = new FieldType(TextField.DEFAULT_TYPE);
+ storedType.setStored(true);
+ Field f = new Field(FIELD_NAME, storedType, text);
d.add(f);
writer.addDocument(d);
Modified: lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java?rev=1142831&r1=1142830&r2=1142831&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java Mon Jul 4 22:26:54 2011
@@ -26,9 +26,10 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document2.Document;
+import org.apache.lucene.document2.Field;
+import org.apache.lucene.document2.FieldType;
+import org.apache.lucene.document2.TextField;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
@@ -107,8 +108,10 @@ public class TokenSourcesTest extends Lu
newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer()));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamOverlap(),
- TermVector.WITH_OFFSETS));
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorOffsets(true);
+ document.add(new Field(FIELD, customType, new TokenStreamOverlap()));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
@@ -153,8 +156,11 @@ public class TokenSourcesTest extends Lu
newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer()));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamOverlap(),
- TermVector.WITH_POSITIONS_OFFSETS));
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectorPositions(true);
+ document.add(new Field(FIELD, customType, new TokenStreamOverlap()));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
@@ -199,8 +205,10 @@ public class TokenSourcesTest extends Lu
newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer()));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamOverlap(),
- TermVector.WITH_OFFSETS));
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorOffsets(true);
+ document.add(new Field(FIELD, customType, new TokenStreamOverlap()));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
@@ -246,8 +254,10 @@ public class TokenSourcesTest extends Lu
newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer()));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamOverlap(),
- TermVector.WITH_POSITIONS_OFFSETS));
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorOffsets(true);
+ document.add(new Field(FIELD, customType, new TokenStreamOverlap()));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
Modified: lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1142831&r1=1142830&r2=1142831&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Mon Jul 4 22:26:54 2011
@@ -28,11 +28,10 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document2.Document;
+import org.apache.lucene.document2.Field;
+import org.apache.lucene.document2.FieldType;
+import org.apache.lucene.document2.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -332,8 +331,14 @@ public abstract class AbstractTestCase e
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, analyzer).setOpenMode(OpenMode.CREATE));
Document doc = new Document();
- for( String value: values )
- doc.add( new Field( F, value, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setStored(true);
+ for( String value: values ) {
+ doc.add( new Field( F, customType, value ) );
+ }
writer.addDocument( doc );
writer.close();
if (reader != null) reader.close();
@@ -345,8 +350,16 @@ public abstract class AbstractTestCase e
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, analyzerK).setOpenMode(OpenMode.CREATE));
Document doc = new Document();
- for( String value: values )
- doc.add( new Field( F, value, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setTokenized(false);
+ customType.setStored(true);
+ for( String value: values ) {
+ doc.add( new Field( F, customType, value ));
+ //doc.add( new Field( F, value, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
+ }
writer.addDocument( doc );
writer.close();
if (reader != null) reader.close();
Modified: lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java?rev=1142831&r1=1142830&r2=1142831&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java Mon Jul 4 22:26:54 2011
@@ -17,11 +17,10 @@ package org.apache.lucene.search.vectorh
* limitations under the License.
*/
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document2.Document;
+import org.apache.lucene.document2.Field;
+import org.apache.lucene.document2.FieldType;
+import org.apache.lucene.document2.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -132,7 +131,12 @@ public class SimpleFragmentsBuilderTest
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, analyzerW).setOpenMode(OpenMode.CREATE));
Document doc = new Document();
- doc.add( new Field( F, "aaa", Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectorPositions(true);
+ doc.add( new Field( F, customType, "aaa" ) );
+ //doc.add( new Field( F, "aaa", Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
writer.addDocument( doc );
writer.close();
if (reader != null) reader.close();
Modified: lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocument.java?rev=1142831&r1=1142830&r2=1142831&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocument.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocument.java Mon Jul 4 22:26:54 2011
@@ -16,7 +16,7 @@ package org.apache.lucene.store.instanti
* limitations under the License.
*/
-import org.apache.lucene.document.Document;
+import org.apache.lucene.document2.Document;
import java.util.List;
import java.util.Map;
@@ -64,10 +64,14 @@ public class InstantiatedDocument {
this.vectorSpace = vectorSpace;
}
- public Document getDocument() {
+ public Document getDocument2() {
return document;
}
-
+
+ //nocommit this code is temporary!!
+ public org.apache.lucene.document.Document getDocument() {
+ return null;
+ }
@Override
public String toString() {
Modified: lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java?rev=1142831&r1=1142830&r2=1142831&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java Mon Jul 4 22:26:54 2011
@@ -26,9 +26,9 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document2.Document;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MultiNorms;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
@@ -189,17 +189,17 @@ public class InstantiatedIndex
} else {
InstantiatedDocument document = new InstantiatedDocument();
// copy stored fields from source reader
- Document sourceDocument = sourceIndexReader.document(i);
- for (Fieldable field : sourceDocument.getFields()) {
+ org.apache.lucene.document.Document sourceDocument = sourceIndexReader.document(i);
+ for (IndexableField field : sourceDocument) {
if (fields == null || fields.contains(field.name())) {
- document.getDocument().add(field);
+ document.getDocument2().add(field);
}
}
document.setDocumentNumber(i);
documentsByNumber[i] = document;
- for (Fieldable field : document.getDocument().getFields()) {
+ for (IndexableField field : document.getDocument2()) {
if (fields == null || fields.contains(field.name())) {
- if (field.isTermVectorStored()) {
+ if (field.storeTermVectors()) {
if (document.getVectorSpace() == null) {
document.setVectorSpace(new HashMap<String, List<InstantiatedTermDocumentInformation>>());
}
@@ -290,8 +290,8 @@ public class InstantiatedIndex
if (document == null) {
continue; // deleted
}
- for (Fieldable field : document.getDocument().getFields()) {
- if (field.isTermVectorStored() && field.isStoreOffsetWithTermVector()) {
+ for (IndexableField field : document.getDocument2()) {
+ if (field.storeTermVectors() && field.storeTermVectorOffsets()) {
TermPositionVector termPositionVector = (TermPositionVector) sourceIndexReader.getTermFreqVector(document.getDocumentNumber(), field.name());
if (termPositionVector != null) {
for (int i = 0; i < termPositionVector.getTerms().length; i++) {
Modified: lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=1142831&r1=1142830&r2=1142831&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Mon Jul 4 22:26:54 2011
@@ -36,10 +36,10 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document2.Document;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermVectorOffsetInfo;
import org.apache.lucene.search.IndexSearcher;
@@ -239,7 +239,7 @@ public class InstantiatedIndexWriter imp
if (eFieldTermDocInfoFactoriesByTermText.getKey().indexed && !eFieldTermDocInfoFactoriesByTermText.getKey().omitNorms) {
final String fieldName = eFieldTermDocInfoFactoriesByTermText.getKey().fieldName;
final FieldInvertState invertState = new FieldInvertState();
- invertState.setBoost(eFieldTermDocInfoFactoriesByTermText.getKey().boost * document.getDocument().getBoost());
+ invertState.setBoost(eFieldTermDocInfoFactoriesByTermText.getKey().boost);
invertState.setLength(eFieldTermDocInfoFactoriesByTermText.getKey().fieldLength);
final float norm = similarityProvider.get(fieldName).computeNorm(invertState);
normsByFieldNameAndDocumentNumber.get(fieldName)[document.getDocumentNumber()] = similarityProvider.get(fieldName).encodeNormValue(norm);
@@ -471,7 +471,7 @@ public class InstantiatedIndexWriter imp
// normalize settings per field name in document
Map<String /* field name */, FieldSetting> fieldSettingsByFieldName = new HashMap<String, FieldSetting>();
- for (Fieldable field : document.getDocument().getFields()) {
+ for (IndexableField field : document.getDocument2()) {
FieldSetting fieldSetting = fieldSettingsByFieldName.get(field.name());
if (fieldSetting == null) {
fieldSetting = new FieldSetting();
@@ -481,52 +481,52 @@ public class InstantiatedIndexWriter imp
}
// todo: fixme: multiple fields with the same name does not mean field boost += more boost.
- fieldSetting.boost *= field.getBoost();
+ fieldSetting.boost *= field.boost();
//fieldSettings.dimensions++;
// once fieldSettings, always fieldSettings.
- if (field.getOmitNorms()) {
+ if (field.omitNorms()) {
fieldSetting.omitNorms = true;
}
- if (field.isIndexed() ) {
+ if (field.indexed() ) {
fieldSetting.indexed = true;
}
- if (field.isTokenized()) {
+ if (field.tokenized()) {
fieldSetting.tokenized = true;
}
- if (field.isStored()) {
+ if (field.stored()) {
fieldSetting.stored = true;
}
- if (field.isBinary()) {
+ if (field.binaryValue(null) != null) {
fieldSetting.isBinary = true;
}
- if (field.isTermVectorStored()) {
+ if (field.storeTermVectors()) {
fieldSetting.storeTermVector = true;
}
- if (field.isStorePositionWithTermVector()) {
+ if (field.storeTermVectorPositions()) {
fieldSetting.storePositionWithTermVector = true;
}
- if (field.isStoreOffsetWithTermVector()) {
+ if (field.storeTermVectorOffsets()) {
fieldSetting.storeOffsetWithTermVector = true;
}
}
- Map<Fieldable, LinkedList<Token>> tokensByField = new LinkedHashMap<Fieldable, LinkedList<Token>>(20);
+ Map<IndexableField, LinkedList<Token>> tokensByField = new LinkedHashMap<IndexableField, LinkedList<Token>>(20);
// tokenize indexed fields.
- for (Iterator<Fieldable> it = document.getDocument().getFields().iterator(); it.hasNext();) {
+ for (Iterator<IndexableField> it = document.getDocument2().iterator(); it.hasNext();) {
- Fieldable field = it.next();
+ IndexableField field = it.next();
FieldSetting fieldSetting = fieldSettingsByFieldName.get(field.name());
- if (field.isIndexed()) {
+ if (field.indexed()) {
LinkedList<Token> tokens = new LinkedList<Token>();
tokensByField.put(field, tokens);
- if (field.isTokenized()) {
+ if (field.tokenized()) {
final TokenStream tokenStream;
// todo readerValue(), binaryValue()
if (field.tokenStreamValue() != null) {
@@ -566,8 +566,8 @@ public class InstantiatedIndexWriter imp
}
}
- if (!field.isStored()) {
- it.remove();
+ if (!field.stored()) {
+ //it.remove();
}
}
@@ -576,7 +576,7 @@ public class InstantiatedIndexWriter imp
termDocumentInformationFactoryByDocument.put(document, termDocumentInformationFactoryByTermTextAndFieldSetting);
// build term vector, term positions and term offsets
- for (Map.Entry<Fieldable, LinkedList<Token>> eField_Tokens : tokensByField.entrySet()) {
+ for (Map.Entry<IndexableField, LinkedList<Token>> eField_Tokens : tokensByField.entrySet()) {
FieldSetting fieldSetting = fieldSettingsByFieldName.get(eField_Tokens.getKey().name());
Map<String, TermDocumentInformationFactory> termDocumentInformationFactoryByTermText = termDocumentInformationFactoryByTermTextAndFieldSetting.get(fieldSettingsByFieldName.get(eField_Tokens.getKey().name()));
@@ -612,7 +612,7 @@ public class InstantiatedIndexWriter imp
termDocumentInformationFactory.payloads.add(null);
}
- if (eField_Tokens.getKey().isStoreOffsetWithTermVector()) {
+ if (eField_Tokens.getKey().storeTermVectorOffsets()) {
termDocumentInformationFactory.termOffsets.add(new TermVectorOffsetInfo(fieldSetting.offset + token.startOffset(), fieldSetting.offset + token.endOffset()));
lastOffset = fieldSetting.offset + token.endOffset();
@@ -621,7 +621,7 @@ public class InstantiatedIndexWriter imp
}
- if (eField_Tokens.getKey().isStoreOffsetWithTermVector()) {
+ if (eField_Tokens.getKey().storeTermVectorOffsets()) {
fieldSetting.offset = lastOffset + 1;
}
Modified: lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=1142831&r1=1142830&r2=1142831&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Mon Jul 4 22:26:54 2011
@@ -27,8 +27,10 @@ import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document2.Document;
+import org.apache.lucene.document2.Field;
+import org.apache.lucene.document2.FieldType;
+import org.apache.lucene.document2.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.MultiNorms;
@@ -204,19 +206,47 @@ public class TestIndicesEquals extends L
private void assembleDocument(Document document, int i) {
- document.add(new Field("a", i + " Do you really want to go and live in that house all winter?", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ FieldType customType = new FieldType(TextField.DEFAULT_TYPE);
+ customType.setStored(true);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectorPositions(true);
+ //document.add(new Field("a", i + " Do you really want to go and live in that house all winter?", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ document.add(new Field("a", customType, i + " Do you really want to go and live in that house all winter?"));
if (i > 0) {
- document.add(new Field("b0", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- document.add(new Field("b1", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO));
- document.add(new Field("b2", i + " All work and no play makes Jack a dull boy", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.NO));
- document.add(new Field("b3", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
+ //document.add(new Field("b0", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ document.add(new Field("b0", customType, i + " All work and no play makes Jack a dull boy"));
+
+ //document.add(new Field("b1", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO));
+ FieldType customType2 = new FieldType(TextField.DEFAULT_TYPE);
+ customType2.setStored(true);
+ customType2.setTokenized(false);
+ customType2.setOmitNorms(true);
+ document.add(new Field("b1", customType2, i + " All work and no play makes Jack a dull boy"));
+
+ //document.add(new Field("b2", i + " All work and no play makes Jack a dull boy", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.NO));
+ FieldType customType3 = new FieldType(TextField.DEFAULT_TYPE);
+ customType3.setTokenized(false);
+ document.add(new Field("b1", customType3, i + " All work and no play makes Jack a dull boy"));
+
+ //document.add(new Field("b3", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
+ FieldType customType4 = new FieldType(TextField.DEFAULT_TYPE);
+ customType4.setStored(true);
+ customType4.setIndexed(false);
+ customType4.setTokenized(false);
+ document.add(new Field("b1", customType4, i + " All work and no play makes Jack a dull boy"));
if (i > 1) {
- document.add(new Field("c", i + " Redrum redrum", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ //document.add(new Field("c", i + " Redrum redrum", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ document.add(new Field("c", customType, i + " Redrum redrum"));
if (i > 2) {
- document.add(new Field("d", i + " Hello Danny, come and play with us... forever and ever. and ever.", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ //document.add(new Field("d", i + " Hello Danny, come and play with us... forever and ever. and ever.", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ document.add(new Field("d", customType, i + " Hello Danny, come and play with us... forever and ever. and ever."));
if (i > 3) {
- Field f = new Field("e", i + " Heres Johnny!", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- f.setOmitNorms(true);
+ //Field f = new Field("e", i + " Heres Johnny!", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
+ //f.setOmitNorms(true);
+ FieldType customType5 = new FieldType(TextField.DEFAULT_TYPE);
+ customType5.setOmitNorms(true);
+ Field f = new Field("e", customType5, i + " Heres Johnny!");
document.add(f);
if (i > 4) {
final List<Token> tokens = new ArrayList<Token>(2);
@@ -247,7 +277,8 @@ public class TestIndicesEquals extends L
}
};
- document.add(new Field("f", ts));
+ //document.add(new Field("f", ts));
+ document.add(new TextField("f", ts));
}
}
}
Modified: lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java?rev=1142831&r1=1142830&r2=1142831&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java Mon Jul 4 22:26:54 2011
@@ -18,8 +18,9 @@ package org.apache.lucene.store.instanti
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document2.Document;
+import org.apache.lucene.document2.Field;
+import org.apache.lucene.document2.StringField;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.LuceneTestCase;
@@ -43,7 +44,7 @@ public class TestRealTime extends Lucene
Collector collector;
doc = new Document();
- doc.add(new Field("f", "a", Field.Store.NO, Field.Index.NOT_ANALYZED));
+ doc.add(new StringField("f", "a"));
writer.addDocument(doc);
writer.commit();
@@ -52,7 +53,7 @@ public class TestRealTime extends Lucene
assertEquals(1, collector.hits);
doc = new Document();
- doc.add(new Field("f", "a", Field.Store.NO, Field.Index.NOT_ANALYZED));
+ doc.add(new StringField("f", "a"));
writer.addDocument(doc);
writer.commit();
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/BinaryField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/BinaryField.java?rev=1142831&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/BinaryField.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/BinaryField.java Mon Jul 4 22:26:54 2011
@@ -0,0 +1,36 @@
+package org.apache.lucene.document2;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public final class BinaryField extends Field {
+
+ public static final FieldType DEFAULT_TYPE = new FieldType();
+ static {
+ DEFAULT_TYPE.setStored(true);
+ DEFAULT_TYPE.freeze();
+ }
+
+ public BinaryField(String name, byte[] value) {
+ super(name, BinaryField.DEFAULT_TYPE, value);
+ this.isBinary = true;
+ }
+
+ public boolean isNumeric() {
+ return false;
+ }
+}
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Document.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Document.java?rev=1142831&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Document.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Document.java Mon Jul 4 22:26:54 2011
@@ -0,0 +1,180 @@
+package org.apache.lucene.document2;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.*;
+
+import org.apache.lucene.index.IndexReader; // for javadoc
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.IndexSearcher; // for javadoc
+import org.apache.lucene.search.ScoreDoc; // for javadoc
+
+/** Documents are the unit of indexing and search.
+ *
+ * A Document is a set of fields. Each field has a name and a textual value.
+ * A field may be {@link Fieldable#isStored() stored} with the document, in which
+ * case it is returned with search hits on the document. Thus each document
+ * should typically contain one or more stored fields which uniquely identify
+ * it.
+ *
+ * <p>Note that fields which are <i>not</i> {@link Fieldable#isStored() stored} are
+ * <i>not</i> available in documents retrieved from the index, e.g. with {@link
+ * ScoreDoc#doc} or {@link IndexReader#document(int)}.
+ */
+
+public final class Document implements Iterable<IndexableField> {
+
+ List<IndexableField> fields = new ArrayList<IndexableField>();
+
+ /** Constructs a new document with no fields. */
+ public Document() {}
+
+ // @Override not until Java 1.6
+ public Iterator<IndexableField> iterator() {
+
+ return new Iterator<IndexableField>() {
+ private int fieldUpto = 0;
+
+ public boolean hasNext() {
+ return fieldUpto < fields.size();
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ public IndexableField next() {
+ return fields.get(fieldUpto++);
+ }
+ };
+ }
+
+ /**
+ * <p>Adds a field to a document. Several fields may be added with
+ * the same name. In this case, if the fields are indexed, their text is
+ * treated as though appended for the purposes of search.</p>
+ * <p> Note that add like the removeField(s) methods only makes sense
+ * prior to adding a document to an index. These methods cannot
+ * be used to change the content of an existing index! In order to achieve this,
+ * a document has to be deleted from an index and a new changed version of that
+ * document has to be added.</p>
+ */
+ public final void add(IndexableField field) {
+ fields.add(field);
+ }
+
+ /**
+ * <p>Removes field with the specified name from the document.
+ * If multiple fields exist with this name, this method removes the first field that has been added.
+ * If there is no field with the specified name, the document remains unchanged.</p>
+ * <p> Note that the removeField(s) methods like the add method only make sense
+ * prior to adding a document to an index. These methods cannot
+ * be used to change the content of an existing index! In order to achieve this,
+ * a document has to be deleted from an index and a new changed version of that
+ * document has to be added.</p>
+ */
+ public final void removeField(String name) {
+ Iterator<IndexableField> it = fields.iterator();
+ while (it.hasNext()) {
+ IndexableField field = it.next();
+ if (field.name().equals(name)) {
+ it.remove();
+ return;
+ }
+ }
+ }
+
+ /**
+ * <p>Removes all fields with the given name from the document.
+ * If there is no field with the specified name, the document remains unchanged.</p>
+ * <p> Note that the removeField(s) methods like the add method only make sense
+ * prior to adding a document to an index. These methods cannot
+ * be used to change the content of an existing index! In order to achieve this,
+ * a document has to be deleted from an index and a new changed version of that
+ * document has to be added.</p>
+ */
+ public final void removeFields(String name) {
+ Iterator<IndexableField> it = fields.iterator();
+ while (it.hasNext()) {
+ IndexableField field = it.next();
+ if (field.name().equals(name)) {
+ it.remove();
+ }
+ }
+ }
+
+ private final static byte[][] NO_BYTES = new byte[0][];
+
+ /**
+ * Returns an array of byte arrays for of the fields that have the name specified
+ * as the method parameter. This method returns an empty
+ * array when there are no matching fields. It never
+ * returns null.
+ *
+ * @param name the name of the field
+ * @return a <code>byte[][]</code> of binary field values
+ */
+ public final byte[][] getBinaryValues(String name) {
+ List<byte[]> result = new ArrayList<byte[]>();
+ for (IndexableField field : fields) {
+ if (field.name().equals(name) && ((Field) field).isBinary())
+ result.add(field.binaryValue(null).bytes);
+ }
+
+ if (result.size() == 0)
+ return NO_BYTES;
+
+ return result.toArray(new byte[result.size()][]);
+ }
+
+ /**
+ * Returns an array of bytes for the first (or only) field that has the name
+ * specified as the method parameter. This method will return <code>null</code>
+ * if no binary fields with the specified name are available.
+ * There may be non-binary fields with the same name.
+ *
+ * @param name the name of the field.
+ * @return a <code>byte[]</code> containing the binary field value or <code>null</code>
+ */
+ public final byte[] getBinaryValue(String name) {
+ for (IndexableField field : fields) {
+ if (field.name().equals(name) && ((Field) field).isBinary())
+ return field.binaryValue(null).bytes;
+ }
+ return null;
+ }
+
+ public Integer size() {
+ return fields.size();
+ }
+
+ /** Prints the fields of a document for human consumption. */
+ @Override
+ public final String toString() {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append("Document<");
+ for (int i = 0; i < fields.size(); i++) {
+ IndexableField field = fields.get(i);
+ buffer.append(field.toString());
+ if (i != fields.size()-1)
+ buffer.append(" ");
+ }
+ buffer.append(">");
+ return buffer.toString();
+ }
+}
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Field.java?rev=1142831&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Field.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Field.java Mon Jul 4 22:26:54 2011
@@ -0,0 +1,373 @@
+package org.apache.lucene.document2;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.StringHelper;
+
+/**
+ * A field is a section of a Document. Each field has two parts, a name and a
+ * value. Values may be free text, provided as a String or as a Reader, or they
+ * may be atomic keywords, which are not further processed. Such keywords may be
+ * used to represent dates, urls, etc. Fields are optionally stored in the
+ * index, so that they may be returned with hits on the document.
+ */
+
+public class Field implements IndexableField {
+
+ protected FieldType type;
+ protected String name = "body";
+ // the data object for all different kind of field values
+ protected Object fieldsData = null;
+ // pre-analyzed tokenStream for indexed fields
+ protected TokenStream tokenStream;
+ protected boolean isBinary = false;
+ // length/offset for all primitive types
+ protected int binaryLength;
+ protected int binaryOffset;
+
+ protected float boost = 1.0f;
+
+ public Field(String name, FieldType type) {
+ this.name = name;
+ this.type = type;
+ }
+
+ public Field(String name, FieldType type, Reader reader) {
+ if (name == null)
+ throw new NullPointerException("name cannot be null");
+ if (reader == null)
+ throw new NullPointerException("reader cannot be null");
+
+ this.name = StringHelper.intern(name); // field names are interned
+ this.fieldsData = reader;
+ this.type = type;
+ }
+
+ public Field(String name, FieldType type, TokenStream tokenStream) {
+ if (name == null)
+ throw new NullPointerException("name cannot be null");
+ if (tokenStream == null)
+ throw new NullPointerException("tokenStream cannot be null");
+
+ this.name = StringHelper.intern(name); // field names are interned
+ this.fieldsData = null;
+ this.tokenStream = tokenStream;
+ this.type = type;
+ }
+
+ public Field(String name, FieldType type, byte[] value) {
+ this(name, type, value, 0, value.length);
+ }
+
+ public Field(String name, FieldType type, byte[] value, int offset, int length) {
+ this.isBinary = true;
+ this.fieldsData = value;
+ this.type = type;
+ this.binaryOffset = offset;
+ this.binaryLength = length;
+ this.name = StringHelper.intern(name);
+ }
+
+ public Field(String name, FieldType type, String value) {
+ this(name, true, type, value);
+ }
+
+ public Field(String name, boolean internName, FieldType type, String value) {
+ if (name == null)
+ throw new IllegalArgumentException("name cannot be null");
+ if (value == null)
+ throw new IllegalArgumentException("value cannot be null");
+
+ this.type = type;
+ this.name = name;
+ this.fieldsData = value;
+
+ if (internName) // field names are optionally interned
+ name = StringHelper.intern(name);
+ }
+
+ /**
+ * The value of the field as a String, or null. If null, the Reader value or
+ * binary value is used. Exactly one of stringValue(), readerValue(), and
+ * getBinaryValue() must be set.
+ */
+ public String stringValue() {
+ return fieldsData instanceof String ? (String) fieldsData : null;
+ }
+
+ /**
+ * The value of the field as a Reader, or null. If null, the String value or
+ * binary value is used. Exactly one of stringValue(), readerValue(), and
+ * getBinaryValue() must be set.
+ */
+ public Reader readerValue() {
+ return fieldsData instanceof Reader ? (Reader) fieldsData : null;
+ }
+
+ /**
+ * The TokesStream for this field to be used when indexing, or null. If null,
+ * the Reader value or String value is analyzed to produce the indexed tokens.
+ */
+ public TokenStream tokenStreamValue() {
+ return tokenStream;
+ }
+
+ public Number getNumericValue() {
+ return null;
+ }
+
+ /**
+ * <p>
+ * Expert: change the value of this field. This can be used during indexing to
+ * re-use a single Field instance to improve indexing speed by avoiding GC
+ * cost of new'ing and reclaiming Field instances. Typically a single
+ * {@link Document} instance is re-used as well. This helps most on small
+ * documents.
+ * </p>
+ *
+ * <p>
+ * Each Field instance should only be used once within a single
+ * {@link Document} instance. See <a
+ * href="http://wiki.apache.org/lucene-java/ImproveIndexingSpeed"
+ * >ImproveIndexingSpeed</a> for details.
+ * </p>
+ */
+ public void setValue(String value) {
+ if (isBinary) {
+ throw new IllegalArgumentException(
+ "cannot set a String value on a binary field");
+ }
+ fieldsData = value;
+ }
+
+ /**
+ * Expert: change the value of this field. See <a
+ * href="#setValue(java.lang.String)">setValue(String)</a>.
+ */
+ public void setValue(Reader value) {
+ if (isBinary) {
+ throw new IllegalArgumentException(
+ "cannot set a Reader value on a binary field");
+ }
+ if (stored()) {
+ throw new IllegalArgumentException(
+ "cannot set a Reader value on a stored field");
+ }
+ fieldsData = value;
+ }
+
+ /**
+ * Expert: change the value of this field. See <a
+ * href="#setValue(java.lang.String)">setValue(String)</a>.
+ */
+ public void setValue(byte[] value) {
+ if (!isBinary) {
+ throw new IllegalArgumentException(
+ "cannot set a byte[] value on a non-binary field");
+ }
+ fieldsData = value;
+ binaryLength = value.length;
+ binaryOffset = 0;
+ }
+
+ /**
+ * Expert: change the value of this field. See <a
+ * href="#setValue(java.lang.String)">setValue(String)</a>.
+ */
+ public void setValue(byte[] value, int offset, int length) {
+ if (!isBinary) {
+ throw new IllegalArgumentException(
+ "cannot set a byte[] value on a non-binary field");
+ }
+ fieldsData = value;
+ binaryLength = length;
+ binaryOffset = offset;
+ }
+
+ /**
+ * Expert: sets the token stream to be used for indexing and causes
+ * isIndexed() and isTokenized() to return true. May be combined with stored
+ * values from stringValue() or getBinaryValue()
+ */
+ public void setTokenStream(TokenStream tokenStream) {
+ if (!indexed() || !tokenized()) {
+ throw new IllegalArgumentException(
+ "cannot set token stream on non indexed and tokenized field");
+ }
+ this.tokenStream = tokenStream;
+ }
+
+ public String name() {
+ return name;
+ }
+
+ public float boost() {
+ return boost;
+ }
+
+ /**
+ * Sets the boost factor hits on this field. This value will be multiplied
+ * into the score of all hits on this this field of this document.
+ *
+ * <p>
+ * The boost is multiplied by
+ * {@link org.apache.lucene.document.Document#getBoost()} of the document
+ * containing this field. If a document has multiple fields with the same
+ * name, all such values are multiplied together. This product is then used to
+ * compute the norm factor for the field. By default, in the
+ * {@link org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)}
+ * method, the boost value is multiplied by the length normalization factor
+ * and then rounded by
+ * {@link org.apache.lucene.search.Similarity#encodeNormValue(float)} before
+ * it is stored in the index. One should attempt to ensure that this product
+ * does not overflow the range of that encoding.
+ *
+ * @see org.apache.lucene.document.Document#setBoost(float)
+ * @see org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)
+ * @see org.apache.lucene.search.Similarity#encodeNormValue(float)
+ */
+ public void setBoost(float boost) {
+ this.boost = boost;
+ }
+
+ public boolean numeric() {
+ return false;
+ }
+
+ public Number numericValue() {
+ return null;
+ }
+
+ public NumericField.DataType numericDataType() {
+ return null;
+ }
+
+ private byte[] getBinaryValue(byte[] result /* unused */) {
+ if (isBinary || fieldsData instanceof byte[]) return (byte[]) fieldsData;
+ else return null;
+ }
+
+ private byte[] getBinaryValue() {
+ return getBinaryValue(null);
+ }
+
+ public BytesRef binaryValue(BytesRef reuse) {
+ final byte[] bytes = getBinaryValue();
+ if (bytes != null) {
+ if (reuse == null) {
+ return new BytesRef(bytes, getBinaryOffset(), getBinaryLength());
+ } else {
+ reuse.bytes = bytes;
+ reuse.offset = getBinaryOffset();
+ reuse.length = getBinaryLength();
+ return reuse;
+ }
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns length of byte[] segment that is used as value, if Field is not
+ * binary returned value is undefined
+ *
+ * @return length of byte[] segment that represents this Field value
+ */
+ private int getBinaryLength() {
+ if (isBinary) {
+ return binaryLength;
+ } else if (fieldsData instanceof byte[]) return ((byte[]) fieldsData).length;
+ else return 0;
+ }
+
+ /**
+ * Returns offset into byte[] segment that is used as value, if Field is not
+ * binary returned value is undefined
+ *
+ * @return index of the first character in byte[] segment that represents this
+ * Field value
+ */
+ public int getBinaryOffset() {
+ return binaryOffset;
+ }
+
+ public boolean isBinary() {
+ return isBinary;
+ }
+
+ /** methods from inner FieldType */
+
+ public boolean stored() {
+ return type.stored();
+ }
+
+ public boolean indexed() {
+ return type.indexed();
+ }
+
+ public boolean tokenized() {
+ return type.tokenized();
+ }
+
+ public boolean omitNorms() {
+ return type.omitNorms();
+ }
+
+ public boolean omitTermFreqAndPositions() {
+ return type.omitTermFreqAndPositions();
+ }
+
+ public boolean storeTermVectors() {
+ return type.storeTermVectors();
+ }
+
+ public boolean storeTermVectorOffsets() {
+ return type.storeTermVectorOffsets();
+ }
+
+ public boolean storeTermVectorPositions() {
+ return type.storeTermVectorPositions();
+ }
+
+ public boolean lazy() {
+ return type.lazy();
+ }
+
+ /** Prints a Field for human consumption. */
+ @Override
+ public final String toString() {
+ StringBuilder result = new StringBuilder();
+ result.append(type.toString());
+ result.append('<');
+ result.append(name);
+ result.append(':');
+
+ if (fieldsData != null && type.lazy() == false) {
+ result.append(fieldsData);
+ }
+
+ result.append('>');
+ return result.toString();
+ }
+}
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelector.java?rev=1142831&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelector.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelector.java Mon Jul 4 22:26:54 2011
@@ -0,0 +1,33 @@
+package org.apache.lucene.document2;
+
+/**
+ * Copyright 2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Similar to a {@link java.io.FileFilter}, the FieldSelector allows one to make decisions about
+ * what Fields get loaded on a {@link Document} by {@link org.apache.lucene.index.IndexReader#document(int,org.apache.lucene.document.FieldSelector)}
+ *
+ **/
+public interface FieldSelector {
+
+ /**
+ *
+ * @param fieldName the field to accept or reject
+ * @return an instance of {@link FieldSelectorResult}
+ * if the {@link Field} named <code>fieldName</code> should be loaded.
+ */
+ FieldSelectorResult accept(String fieldName);
+}
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelectorResult.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelectorResult.java?rev=1142831&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelectorResult.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelectorResult.java Mon Jul 4 22:26:54 2011
@@ -0,0 +1,76 @@
+package org.apache.lucene.document2;
+
+/**
+ * Copyright 2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Provides information about what should be done with this Field
+ *
+ **/
+public enum FieldSelectorResult {
+
+ /**
+ * Load this {@link Field} every time the {@link Document} is loaded, reading in the data as it is encountered.
+ * {@link Document#getField(String)} and {@link Document#getFieldable(String)} should not return null.
+ *<p/>
+ * {@link Document#add(Fieldable)} should be called by the Reader.
+ */
+ LOAD,
+
+ /**
+ * Lazily load this {@link Field}. This means the {@link Field} is valid, but it may not actually contain its data until
+ * invoked. {@link Document#getField(String)} SHOULD NOT BE USED. {@link Document#getFieldable(String)} is safe to use and should
+ * return a valid instance of a {@link Fieldable}.
+ *<p/>
+ * {@link Document#add(Fieldable)} should be called by the Reader.
+ */
+ LAZY_LOAD,
+
+ /**
+ * Do not load the {@link Field}. {@link Document#getField(String)} and {@link Document#getFieldable(String)} should return null.
+ * {@link Document#add(Fieldable)} is not called.
+ * <p/>
+ * {@link Document#add(Fieldable)} should not be called by the Reader.
+ */
+ NO_LOAD,
+
+ /**
+ * Load this field as in the {@link #LOAD} case, but immediately return from {@link Field} loading for the {@link Document}. Thus, the
+ * Document may not have its complete set of Fields. {@link Document#getField(String)} and {@link Document#getFieldable(String)} should
+ * both be valid for this {@link Field}
+ * <p/>
+ * {@link Document#add(Fieldable)} should be called by the Reader.
+ */
+ LOAD_AND_BREAK,
+
+ /** Expert: Load the size of this {@link Field} rather than its value.
+ * Size is measured as number of bytes required to store the field == bytes for a binary or any compressed value, and 2*chars for a String value.
+ * The size is stored as a binary value, represented as an int in a byte[], with the higher order byte first in [0]
+ */
+ SIZE,
+
+ /** Expert: Like {@link #SIZE} but immediately break from the field loading loop, i.e., stop loading further fields, after the size is loaded */
+ SIZE_AND_BREAK,
+
+ /**
+ * Lazily load this {@link Field}, but do not cache the result. This means the {@link Field} is valid, but it may not actually contain its data until
+ * invoked. {@link Document#getField(String)} SHOULD NOT BE USED. {@link Document#getFieldable(String)} is safe to use and should
+ * return a valid instance of a {@link Fieldable}.
+ *<p/>
+ * {@link Document#add(Fieldable)} should be called by the Reader.
+ */
+ LATENT
+}
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldType.java?rev=1142831&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldType.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldType.java Mon Jul 4 22:26:54 2011
@@ -0,0 +1,183 @@
+package org.apache.lucene.document2;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class FieldType {
+
+ private boolean indexed;
+ private boolean stored;
+ private boolean tokenized;
+ private boolean storeTermVectors;
+ private boolean storeTermVectorOffsets;
+ private boolean storeTermVectorPositions;
+ private boolean omitNorms;
+ private boolean omitTermFreqsAndPositions;
+ private boolean lazy;
+ private boolean frozen;
+
+ public FieldType(FieldType ref) {
+ this.indexed = ref.indexed();
+ this.stored = ref.stored();
+ this.tokenized = ref.tokenized();
+ this.storeTermVectors = ref.storeTermVectors();
+ this.storeTermVectorOffsets = ref.storeTermVectorOffsets();
+ this.storeTermVectorPositions = ref.storeTermVectorPositions();
+ this.omitNorms = ref.omitNorms();
+ this.omitTermFreqsAndPositions = ref.omitTermFreqAndPositions();
+ this.lazy = ref.lazy();
+ }
+
+ public FieldType() {
+ }
+
+ private void checkIfFrozen() {
+ if (frozen) {
+ throw new IllegalStateException();
+ }
+ }
+
+ public void freeze() {
+ this.frozen = true;
+ }
+
+ public boolean indexed() {
+ return this.indexed;
+ }
+
+ public void setIndexed(boolean value) {
+ checkIfFrozen();
+ this.indexed = value;
+ }
+
+ public boolean stored() {
+ return this.stored;
+ }
+
+ public void setStored(boolean value) {
+ checkIfFrozen();
+ this.stored = value;
+ }
+
+ public boolean tokenized() {
+ return this.tokenized;
+ }
+
+ public void setTokenized(boolean value) {
+ checkIfFrozen();
+ this.tokenized = value;
+ }
+
+ public boolean storeTermVectors() {
+ return this.storeTermVectors;
+ }
+
+ public void setStoreTermVectors(boolean value) {
+ checkIfFrozen();
+ this.storeTermVectors = value;
+ }
+
+ public boolean storeTermVectorOffsets() {
+ return this.storeTermVectorOffsets;
+ }
+
+ public void setStoreTermVectorOffsets(boolean value) {
+ checkIfFrozen();
+ this.storeTermVectorOffsets = value;
+ }
+
+ public boolean storeTermVectorPositions() {
+ return this.storeTermVectorPositions;
+ }
+
+ public void setStoreTermVectorPositions(boolean value) {
+ checkIfFrozen();
+ this.storeTermVectorPositions = value;
+ }
+
+ public boolean omitNorms() {
+ return this.omitNorms;
+ }
+
+ public void setOmitNorms(boolean value) {
+ checkIfFrozen();
+ this.omitNorms = value;
+ }
+
+ public boolean omitTermFreqAndPositions() {
+ return this.omitTermFreqsAndPositions;
+ }
+
+ public void setOmitTermFreqAndPositions(boolean value) {
+ checkIfFrozen();
+ this.omitTermFreqsAndPositions = value;
+ }
+
+ public boolean lazy() {
+ return this.lazy;
+ }
+
+ public void setLazy(boolean value) {
+ checkIfFrozen();
+ this.lazy = value;
+ }
+
+ /** Prints a Field for human consumption. */
+ @Override
+ public final String toString() {
+ StringBuilder result = new StringBuilder();
+ if (stored()) {
+ result.append("stored");
+ }
+ if (indexed()) {
+ if (result.length() > 0)
+ result.append(",");
+ result.append("indexed");
+ }
+ if (tokenized()) {
+ if (result.length() > 0)
+ result.append(",");
+ result.append("tokenized");
+ }
+ if (storeTermVectors()) {
+ if (result.length() > 0)
+ result.append(",");
+ result.append("termVector");
+ }
+ if (storeTermVectorOffsets()) {
+ if (result.length() > 0)
+ result.append(",");
+ result.append("termVectorOffsets");
+ }
+ if (storeTermVectorPositions()) {
+ if (result.length() > 0)
+ result.append(",");
+ result.append("termVectorPosition");
+ }
+ if (omitNorms()) {
+ result.append(",omitNorms");
+ }
+ if (omitTermFreqAndPositions()) {
+ result.append(",omitTermFreqAndPositions");
+ }
+ if (lazy()){
+ result.append(",lazy");
+ }
+
+ return result.toString();
+ }
+}
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/NumericField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/NumericField.java?rev=1142831&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/NumericField.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/NumericField.java Mon Jul 4 22:26:54 2011
@@ -0,0 +1,372 @@
+package org.apache.lucene.document2;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+
+import org.apache.lucene.document.NumericField.DataType;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.search.NumericRangeQuery; // javadocs
+import org.apache.lucene.search.NumericRangeFilter; // javadocs
+import org.apache.lucene.search.SortField; // javadocs
+import org.apache.lucene.search.FieldCache; // javadocs
+
+/**
+ * <p>
+ * This class provides a {@link Field} that enables indexing of numeric values
+ * for efficient range filtering and sorting. Here's an example usage, adding an
+ * int value:
+ *
+ * <pre>
+ * document.add(new NumericField(name).setIntValue(value));
+ * </pre>
+ *
+ * For optimal performance, re-use the <code>NumericField</code> and
+ * {@link Document} instance for more than one document:
+ *
+ * <pre>
+ * NumericField field = new NumericField(name);
+ * Document document = new Document();
+ * document.add(field);
+ *
+ * for(all documents) {
+ * ...
+ * field.setIntValue(value)
+ * writer.addDocument(document);
+ * ...
+ * }
+ * </pre>
+ *
+ * <p>
+ * The java native types <code>int</code>, <code>long</code>, <code>float</code>
+ * and <code>double</code> are directly supported. However, any value that can
+ * be converted into these native types can also be indexed. For example,
+ * date/time values represented by a {@link java.util.Date} can be translated
+ * into a long value using the {@link java.util.Date#getTime} method. If you
+ * don't need millisecond precision, you can quantize the value, either by
+ * dividing the result of {@link java.util.Date#getTime} or using the separate
+ * getters (for year, month, etc.) to construct an <code>int</code> or
+ * <code>long</code> value.
+ * </p>
+ *
+ * <p>
+ * To perform range querying or filtering against a <code>NumericField</code>,
+ * use {@link NumericRangeQuery} or {@link NumericRangeFilter}. To sort
+ * according to a <code>NumericField</code>, use the normal numeric sort types,
+ * eg {@link SortField#INT}. <code>NumericField</code> values can also be loaded
+ * directly from {@link FieldCache}.
+ * </p>
+ *
+ * <p>
+ * By default, a <code>NumericField</code>'s value is not stored but is indexed
+ * for range filtering and sorting. You can use the
+ * {@link #NumericField(String,Field.Store,boolean)} constructor if you need to
+ * change these defaults.
+ * </p>
+ *
+ * <p>
+ * You may add the same field name as a <code>NumericField</code> to the same
+ * document more than once. Range querying and filtering will be the logical OR
+ * of all values; so a range query will hit all documents that have at least one
+ * value in the range. However sort behavior is not defined. If you need to
+ * sort, you should separately index a single-valued <code>NumericField</code>.
+ * </p>
+ *
+ * <p>
+ * A <code>NumericField</code> will consume somewhat more disk space in the
+ * index than an ordinary single-valued field. However, for a typical index that
+ * includes substantial textual content per document, this increase will likely
+ * be in the noise.
+ * </p>
+ *
+ * <p>
+ * Within Lucene, each numeric value is indexed as a <em>trie</em> structure,
+ * where each term is logically assigned to larger and larger pre-defined
+ * brackets (which are simply lower-precision representations of the value). The
+ * step size between each successive bracket is called the
+ * <code>precisionStep</code>, measured in bits. Smaller
+ * <code>precisionStep</code> values result in larger number of brackets, which
+ * consumes more disk space in the index but may result in faster range search
+ * performance. The default value, 4, was selected for a reasonable tradeoff of
+ * disk space consumption versus performance. You can use the expert constructor
+ * {@link #NumericField(String,int,Field.Store,boolean)} if you'd like to change
+ * the value. Note that you must also specify a congruent value when creating
+ * {@link NumericRangeQuery} or {@link NumericRangeFilter}. For low cardinality
+ * fields larger precision steps are good. If the cardinality is < 100, it is
+ * fair to use {@link Integer#MAX_VALUE}, which produces one term per value.
+ *
+ * <p>
+ * For more information on the internals of numeric trie indexing, including the
+ * <a href="../search/NumericRangeQuery.html#precisionStepDesc">
+ * <code>precisionStep</code></a> configuration, see {@link NumericRangeQuery}.
+ * The format of indexed values is described in {@link NumericUtils}.
+ *
+ * <p>
+ * If you only need to sort by numeric value, and never run range
+ * querying/filtering, you can index using a <code>precisionStep</code> of
+ * {@link Integer#MAX_VALUE}. This will minimize disk space consumed.
+ * </p>
+ *
+ * <p>
+ * More advanced users can instead use {@link NumericTokenStream} directly, when
+ * indexing numbers. This class is a wrapper around this token stream type for
+ * easier, more intuitive usage.
+ * </p>
+ *
+ * @since 2.9
+ */
+public final class NumericField extends Field {
+
+ /**
+ * Data type of the value in {@link NumericField}.
+ *
+ * @since 3.2
+ */
+
+ /*
+ public static enum DataType {
+ INT, LONG, FLOAT, DOUBLE
+ }
+ */
+
+ public static final FieldType DEFAULT_TYPE = new FieldType();
+ static {
+ DEFAULT_TYPE.setIndexed(true);
+ DEFAULT_TYPE.setOmitNorms(true);
+ DEFAULT_TYPE.setOmitTermFreqAndPositions(true);
+ DEFAULT_TYPE.freeze();
+ }
+
+ private DataType dataType;
+ private transient NumericTokenStream numericTS;
+ private final int precisionStep;
+
+ /**
+ * Creates a field for numeric values using the default
+ * <code>precisionStep</code> {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * The instance is not yet initialized with a numeric value, before indexing a
+ * document containing this field, set a value using the various set
+ * <em>???</em>Value() methods. This constructor creates an indexed, but not
+ * stored field.
+ *
+ * @param name
+ * the field name
+ */
+ public NumericField(String name) {
+ this(name, NumericUtils.PRECISION_STEP_DEFAULT, NumericField.DEFAULT_TYPE);
+ }
+
+ /**
+ * Creates a field for numeric values using the default
+ * <code>precisionStep</code> {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * The instance is not yet initialized with a numeric value, before indexing a
+ * document containing this field, set a value using the various set
+ * <em>???</em>Value() methods.
+ *
+ * @param name
+ * the field name
+ * @param store
+ * if the field should be stored, {@link Document#getFieldable} then
+ * returns {@code NumericField} instances on search results.
+ * @param index
+ * if the field should be indexed using {@link NumericTokenStream}
+ */
+ public NumericField(String name, FieldType type) {
+ this(name, NumericUtils.PRECISION_STEP_DEFAULT, type);
+ }
+
+ /**
+ * Creates a field for numeric values with the specified
+ * <code>precisionStep</code>. The instance is not yet initialized with a
+ * numeric value, before indexing a document containing this field, set a
+ * value using the various set<em>???</em>Value() methods. This constructor
+ * creates an indexed, but not stored field.
+ *
+ * @param name
+ * the field name
+ * @param precisionStep
+ * the used <a
+ * href="../search/NumericRangeQuery.html#precisionStepDesc"
+ * >precision step</a>
+ */
+ public NumericField(String name, int precisionStep) {
+ this(name, precisionStep, NumericField.DEFAULT_TYPE);
+ }
+
+ /**
+ * Creates a field for numeric values with the specified
+ * <code>precisionStep</code>. The instance is not yet initialized with a
+ * numeric value, before indexing a document containing this field, set a
+ * value using the various set<em>???</em>Value() methods.
+ *
+ * @param name
+ * the field name
+ * @param precisionStep
+ * the used <a
+ * href="../search/NumericRangeQuery.html#precisionStepDesc"
+ * >precision step</a>
+ * @param store
+ * if the field should be stored, {@link Document#getFieldable} then
+ * returns {@code NumericField} instances on search results.
+ * @param index
+ * if the field should be indexed using {@link NumericTokenStream}
+ */
+ public NumericField(String name, int precisionStep, FieldType type) {
+ super(name, type);
+ this.precisionStep = precisionStep;
+ }
+
+ /** Returns a {@link NumericTokenStream} for indexing the numeric value. */
+ public TokenStream tokenStreamValue() {
+ if (!indexed()) return null;
+ if (numericTS == null) {
+ // lazy init the TokenStream as it is heavy to instantiate
+ // (attributes,...),
+ // if not needed (stored field loading)
+ numericTS = new NumericTokenStream(precisionStep);
+ // initialize value in TokenStream
+ if (fieldsData != null) {
+ assert dataType != null;
+ final Number val = (Number) fieldsData;
+ switch (dataType) {
+ case INT:
+ numericTS.setIntValue(val.intValue());
+ break;
+ case LONG:
+ numericTS.setLongValue(val.longValue());
+ break;
+ case FLOAT:
+ numericTS.setFloatValue(val.floatValue());
+ break;
+ case DOUBLE:
+ numericTS.setDoubleValue(val.doubleValue());
+ break;
+ default:
+ assert false : "Should never get here";
+ }
+ }
+ }
+ return numericTS;
+ }
+
+ /** Returns always <code>null</code> for numeric fields */
+ public Reader readerValue() {
+ return null;
+ }
+
+ /**
+ * Returns the numeric value as a string. This format is also returned if you
+ * call {@link Document#get(String)} on search results. It is recommended to
+ * use {@link Document#getFieldable} instead that returns {@code NumericField}
+ * instances. You can then use {@link #getNumericValue} to return the stored
+ * value.
+ */
+ public String stringValue() {
+ return (fieldsData == null) ? null : fieldsData.toString();
+ }
+
+ /**
+ * Returns the current numeric value as a subclass of {@link Number},
+ * <code>null</code> if not yet initialized.
+ */
+ public Number getNumericValue() {
+ return (Number) fieldsData;
+ }
+
+ /** Returns the precision step. */
+ public int getPrecisionStep() {
+ return precisionStep;
+ }
+
+ /**
+ * Returns the data type of the current value, {@code null} if not yet set.
+ *
+ * @since 3.2
+ */
+ public DataType getNumericDataType() {
+ return dataType;
+ }
+
+ public boolean isNumeric() {
+ return true;
+ }
+
+ /**
+ * Initializes the field with the supplied <code>long</code> value.
+ *
+ * @param value
+ * the numeric value
+ * @return this instance, because of this you can use it the following way:
+ * <code>document.add(new NumericField(name, precisionStep).setLongValue(value))</code>
+ */
+ public NumericField setLongValue(final long value) {
+ if (numericTS != null) numericTS.setLongValue(value);
+ fieldsData = Long.valueOf(value);
+ dataType = DataType.LONG;
+ return this;
+ }
+
+ /**
+ * Initializes the field with the supplied <code>int</code> value.
+ *
+ * @param value
+ * the numeric value
+ * @return this instance, because of this you can use it the following way:
+ * <code>document.add(new NumericField(name, precisionStep).setIntValue(value))</code>
+ */
+ public NumericField setIntValue(final int value) {
+ if (numericTS != null) numericTS.setIntValue(value);
+ fieldsData = Integer.valueOf(value);
+ dataType = DataType.INT;
+ return this;
+ }
+
+ /**
+ * Initializes the field with the supplied <code>double</code> value.
+ *
+ * @param value
+ * the numeric value
+ * @return this instance, because of this you can use it the following way:
+ * <code>document.add(new NumericField(name, precisionStep).setDoubleValue(value))</code>
+ */
+ public NumericField setDoubleValue(final double value) {
+ if (numericTS != null) numericTS.setDoubleValue(value);
+ fieldsData = Double.valueOf(value);
+ dataType = DataType.DOUBLE;
+ return this;
+ }
+
+ /**
+ * Initializes the field with the supplied <code>float</code> value.
+ *
+ * @param value
+ * the numeric value
+ * @return this instance, because of this you can use it the following way:
+ * <code>document.add(new NumericField(name, precisionStep).setFloatValue(value))</code>
+ */
+ public NumericField setFloatValue(final float value) {
+ if (numericTS != null) numericTS.setFloatValue(value);
+ fieldsData = Float.valueOf(value);
+ dataType = DataType.FLOAT;
+ return this;
+ }
+
+}