You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by yo...@apache.org on 2005/10/30 06:42:14 UTC
svn commit: r329526 - in
/lucene/java/trunk/src/test/org/apache/lucene/index: DocHelper.java
TestDocumentWriter.java TestFieldInfos.java TestFieldsReader.java
TestMultiReader.java TestSegmentMerger.java TestSegmentReader.java
Author: yonik
Date: Sat Oct 29 22:42:09 2005
New Revision: 329526
URL: http://svn.apache.org/viewcvs?rev=329526&view=rev
Log:
omitNorms tests, make some tests more generic
Modified:
lucene/java/trunk/src/test/org/apache/lucene/index/DocHelper.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldInfos.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldsReader.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentMerger.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentReader.java
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/DocHelper.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/DocHelper.java Sat Oct 29 22:42:09 2005
@@ -36,17 +36,28 @@
public static final int [] FIELD_2_FREQS = {3, 1, 1};
public static final String TEXT_FIELD_2_KEY = "textField2";
public static Field textField2 = new Field(TEXT_FIELD_2_KEY, FIELD_2_TEXT, Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
-
+
+ public static final String FIELD_3_TEXT = "aaaNoNorms aaaNoNorms bbbNoNorms";
+ public static final String TEXT_FIELD_3_KEY = "textField3";
+ public static Field textField3 = new Field(TEXT_FIELD_3_KEY, FIELD_3_TEXT, Field.Store.YES, Field.Index.TOKENIZED);
+ static { textField3.setOmitNorms(true); }
+
public static final String KEYWORD_TEXT = "Keyword";
public static final String KEYWORD_FIELD_KEY = "keyField";
public static Field keyField = new Field(KEYWORD_FIELD_KEY, KEYWORD_TEXT,
Field.Store.YES, Field.Index.UN_TOKENIZED);
-
+
+ public static final String NO_NORMS_TEXT = "omitNormsText";
+ public static final String NO_NORMS_KEY = "omitNorms";
+ public static Field noNormsField = new Field(NO_NORMS_KEY, NO_NORMS_TEXT,
+ Field.Store.YES, Field.Index.NO_NORMS);
+
public static final String UNINDEXED_FIELD_TEXT = "unindexed field text";
public static final String UNINDEXED_FIELD_KEY = "unIndField";
public static Field unIndField = new Field(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT,
Field.Store.YES, Field.Index.NO);
-
+
+
public static final String UNSTORED_1_FIELD_TEXT = "unstored field text";
public static final String UNSTORED_FIELD_1_KEY = "unStoredField1";
public static Field unStoredField1 = new Field(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT,
@@ -58,13 +69,58 @@
Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES);
public static Map nameValues = null;
-
+
+ // ordered list of all the fields...
+ // could use LinkedHashMap for this purpose if Java1.4 is OK
+ public static Field[] fields = new Field[] {
+ textField1,
+ textField2,
+ textField3,
+ keyField,
+ noNormsField,
+ unIndField,
+ unStoredField1,
+ unStoredField2,
+ };
+
+ // Map<String fieldName, Field field>
+ public static Map all=new HashMap();
+ public static Map indexed=new HashMap();
+ public static Map stored=new HashMap();
+ public static Map unstored=new HashMap();
+ public static Map unindexed=new HashMap();
+ public static Map termvector=new HashMap();
+ public static Map notermvector=new HashMap();
+ public static Map noNorms=new HashMap();
+
+ static {
+ for (int i=0; i<fields.length; i++) {
+ Field f = fields[i];
+ add(all,f);
+ if (f.isIndexed()) add(indexed,f);
+ else add(unindexed,f);
+ if (f.isTermVectorStored()) add(termvector,f);
+ if (f.isIndexed() && !f.isTermVectorStored()) add(notermvector,f);
+ if (f.isStored()) add(stored,f);
+ else add(unstored,f);
+ if (f.getOmitNorms()) add(noNorms,f);
+ }
+ }
+
+
+ private static void add(Map map, Field field) {
+ map.put(field.name(), field);
+ }
+
+
static
{
nameValues = new HashMap();
nameValues.put(TEXT_FIELD_1_KEY, FIELD_1_TEXT);
nameValues.put(TEXT_FIELD_2_KEY, FIELD_2_TEXT);
+ nameValues.put(TEXT_FIELD_3_KEY, FIELD_3_TEXT);
nameValues.put(KEYWORD_FIELD_KEY, KEYWORD_TEXT);
+ nameValues.put(NO_NORMS_KEY, NO_NORMS_TEXT);
nameValues.put(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT);
nameValues.put(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT);
nameValues.put(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT);
@@ -75,12 +131,9 @@
* @param doc The document to write
*/
public static void setupDoc(Document doc) {
- doc.add(textField1);
- doc.add(textField2);
- doc.add(keyField);
- doc.add(unIndField);
- doc.add(unStoredField1);
- doc.add(unStoredField2);
+ for (int i=0; i<fields.length; i++) {
+ doc.add(fields[i]);
+ }
}
/**
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java Sat Oct 29 22:42:09 2005
@@ -75,6 +75,15 @@
fields = doc.getFields("keyField");
assertTrue(fields != null && fields.length == 1);
assertTrue(fields[0].stringValue().equals(DocHelper.KEYWORD_TEXT));
+
+ fields = doc.getFields(DocHelper.NO_NORMS_KEY);
+ assertTrue(fields != null && fields.length == 1);
+ assertTrue(fields[0].stringValue().equals(DocHelper.NO_NORMS_TEXT));
+
+ fields = doc.getFields(DocHelper.TEXT_FIELD_3_KEY);
+ assertTrue(fields != null && fields.length == 1);
+ assertTrue(fields[0].stringValue().equals(DocHelper.FIELD_3_TEXT));
+
} catch (IOException e) {
e.printStackTrace();
assertTrue(false);
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldInfos.java Sat Oct 29 22:42:09 2005
@@ -31,7 +31,7 @@
FieldInfos fieldInfos = new FieldInfos();
fieldInfos.add(testDoc);
//Since the complement is stored as well in the fields map
- assertTrue(fieldInfos.size() == 6); //this is 6 b/c we are using the no-arg constructor
+ assertTrue(fieldInfos.size() == DocHelper.all.size()); //this is all b/c we are using the no-arg constructor
RAMDirectory dir = new RAMDirectory();
String name = "testFile";
IndexOutput output = dir.createOutput(name);
@@ -47,11 +47,23 @@
FieldInfo info = readIn.fieldInfo("textField1");
assertTrue(info != null);
assertTrue(info.storeTermVector == false);
-
+ assertTrue(info.omitNorms == false);
+
info = readIn.fieldInfo("textField2");
assertTrue(info != null);
assertTrue(info.storeTermVector == true);
-
+ assertTrue(info.omitNorms == false);
+
+ info = readIn.fieldInfo("textField3");
+ assertTrue(info != null);
+ assertTrue(info.storeTermVector == false);
+ assertTrue(info.omitNorms == true);
+
+ info = readIn.fieldInfo("omitNorms");
+ assertTrue(info != null);
+ assertTrue(info.storeTermVector == false);
+ assertTrue(info.omitNorms == true);
+
dir.close();
} catch (IOException e) {
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestFieldsReader.java Sat Oct 29 22:42:09 2005
@@ -54,14 +54,22 @@
assertTrue(doc != null);
assertTrue(doc.getField("textField1") != null);
-
Field field = doc.getField("textField2");
assertTrue(field != null);
assertTrue(field.isTermVectorStored() == true);
assertTrue(field.isStoreOffsetWithTermVector() == true);
assertTrue(field.isStorePositionWithTermVector() == true);
-
+ assertTrue(field.getOmitNorms() == false);
+
+ field = doc.getField("textField3");
+ assertTrue(field != null);
+ assertTrue(field.isTermVectorStored() == false);
+ assertTrue(field.isStoreOffsetWithTermVector() == false);
+ assertTrue(field.isStorePositionWithTermVector() == false);
+ assertTrue(field.getOmitNorms() == true);
+
+
reader.close();
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java Sat Oct 29 22:42:09 2005
@@ -61,12 +61,13 @@
assertTrue(reader != null);
Document newDoc1 = reader.document(0);
assertTrue(newDoc1 != null);
- assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - 2);
+ assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size());
Document newDoc2 = reader.document(1);
assertTrue(newDoc2 != null);
- assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - 2);
+ assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());
TermFreqVector vector = reader.getTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY);
assertTrue(vector != null);
+ TestSegmentReader.checkNorms(reader);
}
public void testTermVectors() {
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentMerger.java Sat Oct 29 22:42:09 2005
@@ -75,10 +75,10 @@
Document newDoc1 = mergedReader.document(0);
assertTrue(newDoc1 != null);
//There are 2 unstored fields on the document
- assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - 2);
+ assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size());
Document newDoc2 = mergedReader.document(1);
assertTrue(newDoc2 != null);
- assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - 2);
+ assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());
TermDocs termDocs = mergedReader.termDocs(new Term(DocHelper.TEXT_FIELD_2_KEY, "field"));
assertTrue(termDocs != null);
@@ -106,6 +106,8 @@
//System.out.println("Term: " + term + " Freq: " + freq);
assertTrue(DocHelper.FIELD_2_TEXT.indexOf(term) != -1);
assertTrue(DocHelper.FIELD_2_FREQS[i] == freq);
- }
+ }
+
+ TestSegmentReader.checkNorms(mergedReader);
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=329526&r1=329525&r2=329526&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestSegmentReader.java Sat Oct 29 22:42:09 2005
@@ -20,6 +20,7 @@
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.search.DefaultSimilarity;
import java.io.IOException;
import java.util.Collection;
@@ -50,7 +51,7 @@
assertTrue(dir != null);
assertTrue(reader != null);
assertTrue(DocHelper.nameValues.size() > 0);
- assertTrue(DocHelper.numFields(testDoc) == 6);
+ assertTrue(DocHelper.numFields(testDoc) == DocHelper.all.size());
}
public void testDocument() throws IOException {
@@ -59,7 +60,7 @@
Document result = reader.document(0);
assertTrue(result != null);
//There are 2 unstored fields on the document that are not preserved across writing
- assertTrue(DocHelper.numFields(result) == DocHelper.numFields(testDoc) - 2);
+ assertTrue(DocHelper.numFields(result) == DocHelper.numFields(testDoc) - DocHelper.unstored.size());
Enumeration fields = result.fields();
while (fields.hasMoreElements()) {
@@ -91,7 +92,7 @@
public void testGetFieldNameVariations() {
Collection result = reader.getFieldNames(IndexReader.FieldOption.ALL);
assertTrue(result != null);
- assertTrue(result.size() == 6);
+ assertTrue(result.size() == DocHelper.all.size());
for (Iterator iter = result.iterator(); iter.hasNext();) {
String s = (String) iter.next();
//System.out.println("Name: " + s);
@@ -99,23 +100,23 @@
}
result = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
assertTrue(result != null);
- assertTrue(result.size() == 5);
+ assertTrue(result.size() == DocHelper.indexed.size());
for (Iterator iter = result.iterator(); iter.hasNext();) {
String s = (String) iter.next();
- assertTrue(DocHelper.nameValues.containsKey(s) == true || s.equals(""));
+ assertTrue(DocHelper.indexed.containsKey(s) == true || s.equals(""));
}
result = reader.getFieldNames(IndexReader.FieldOption.UNINDEXED);
assertTrue(result != null);
- assertTrue(result.size() == 1);
+ assertTrue(result.size() == DocHelper.unindexed.size());
//Get all indexed fields that are storing term vectors
result = reader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR);
assertTrue(result != null);
- assertTrue(result.size() == 2);
+ assertTrue(result.size() == DocHelper.termvector.size());
result = reader.getFieldNames(IndexReader.FieldOption.INDEXED_NO_TERMVECTOR);
assertTrue(result != null);
- assertTrue(result.size() == 3);
+ assertTrue(result.size() == DocHelper.notermvector.size());
}
public void testTerms() throws IOException {
@@ -134,6 +135,10 @@
assertTrue(termDocs != null);
termDocs.seek(new Term(DocHelper.TEXT_FIELD_1_KEY, "field"));
assertTrue(termDocs.next() == true);
+
+ termDocs.seek(new Term(DocHelper.NO_NORMS_KEY, DocHelper.NO_NORMS_TEXT));
+ assertTrue(termDocs.next() == true);
+
TermPositions positions = reader.termPositions();
positions.seek(new Term(DocHelper.TEXT_FIELD_1_KEY, "field"));
@@ -142,7 +147,7 @@
assertTrue(positions.nextPosition() >= 0);
}
- public void testNorms() {
+ public void testNorms() throws IOException {
//TODO: Not sure how these work/should be tested
/*
try {
@@ -155,7 +160,32 @@
}
*/
- }
+ checkNorms(reader);
+ }
+
+ public static void checkNorms(IndexReader reader) throws IOException {
+ // test omit norms
+ for (int i=0; i<DocHelper.fields.length; i++) {
+ Field f = DocHelper.fields[i];
+ if (f.isIndexed()) {
+ assertEquals(reader.hasNorms(f.name()), !f.getOmitNorms());
+ assertEquals(reader.hasNorms(f.name()), !DocHelper.noNorms.containsKey(f.name()));
+ if (!reader.hasNorms(f.name())) {
+ // test for fake norms of 1.0
+ byte [] norms = reader.norms(f.name());
+ assertEquals(norms.length,reader.maxDoc());
+ for (int j=0; j<reader.maxDoc(); j++) {
+ assertEquals(norms[j], DefaultSimilarity.encodeNorm(1.0f));
+ }
+ norms = new byte[reader.maxDoc()];
+ reader.norms(f.name(),norms, 0);
+ for (int j=0; j<reader.maxDoc(); j++) {
+ assertEquals(norms[j], DefaultSimilarity.encodeNorm(1.0f));
+ }
+ }
+ }
+ }
+ }
public void testTermVectors() throws IOException {
TermFreqVector result = reader.getTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY);