You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/07/21 15:31:16 UTC
svn commit: r1364085 - in /lucene/dev/branches/pforcodec_3892/lucene:
core/src/test/org/apache/lucene/index/
test-framework/src/java/org/apache/lucene/analysis/
test-framework/src/java/org/apache/lucene/util/
Author: mikemccand
Date: Sat Jul 21 13:31:16 2012
New Revision: 1364085
URL: http://svn.apache.org/viewvc?rev=1364085&view=rev
Log:
LUCENE-3892: turn on all options testing in TestPostingsFormat; fix it to skip offsets if the PF can't index them
Modified:
lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java?rev=1364085&r1=1364084&r2=1364085&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java Sat Jul 21 13:31:16 2012
@@ -330,15 +330,25 @@ public class TestPostingsFormat extends
System.out.println("\nTEST: now build index");
}
+ int maxIndexOptionNoOffsets = Arrays.asList(IndexOptions.values()).indexOf(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+
// nocommit use allowPayloads
FieldInfo[] newFieldInfoArray = new FieldInfo[fields.size()];
for(int fieldUpto=0;fieldUpto<fields.size();fieldUpto++) {
FieldInfo oldFieldInfo = fieldInfos.fieldInfo(fieldUpto);
+ String pf = _TestUtil.getPostingsFormat(oldFieldInfo.name);
+ int fieldMaxIndexOption;
+ if (doesntSupportOffsets.contains(pf)) {
+ fieldMaxIndexOption = Math.min(maxIndexOptionNoOffsets, maxIndexOption);
+ } else {
+ fieldMaxIndexOption = maxIndexOption;
+ }
+
// Randomly picked the IndexOptions to index this
// field with:
- IndexOptions indexOptions = IndexOptions.values()[random().nextInt(1+maxIndexOption)];
+ IndexOptions indexOptions = IndexOptions.values()[random().nextInt(1+fieldMaxIndexOption)];
boolean doPayloads = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 && allowPayloads;
newFieldInfoArray[fieldUpto] = new FieldInfo(oldFieldInfo.name,
@@ -366,12 +376,13 @@ public class TestPostingsFormat extends
Map<BytesRef,List<Posting>> terms = fieldEnt.getValue();
FieldInfo fieldInfo = newFieldInfos.fieldInfo(field);
- if (VERBOSE) {
- System.out.println("field=" + field);
- }
IndexOptions indexOptions = fieldInfo.getIndexOptions();
+ if (VERBOSE) {
+ System.out.println("field=" + field + " indexOtions=" + indexOptions);
+ }
+
boolean doFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
boolean doPos = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
boolean doPayloads = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 && allowPayloads;
@@ -413,6 +424,7 @@ public class TestPostingsFormat extends
} else {
totalTF++;
}
+ postingsConsumer.finishDoc();
docCount++;
}
termsConsumer.finishTerm(term, new TermStats(postings.size(), totalTF));
@@ -539,9 +551,9 @@ public class TestPostingsFormat extends
docsAndPositionsEnum = threadState.reuseDocsAndPositionsEnum;
}
- assertNotNull(docsEnum);
+ assertNotNull("null DocsEnum", docsEnum);
int initialDocID = docsEnum.docID();
- assertTrue(initialDocID == -1 || initialDocID == DocsEnum.NO_MORE_DOCS);
+ assertTrue("inital docID should be -1 or NO_MORE_DOCS", initialDocID == -1 || initialDocID == DocsEnum.NO_MORE_DOCS);
if (VERBOSE) {
if (prevDocsEnum == null) {
@@ -600,10 +612,10 @@ public class TestPostingsFormat extends
while (nextPosting <= stopAt) {
if (nextPosting == stopAt) {
if (stopAt == expected.size()) {
- assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc());
+ assertEquals("DocsEnum should have ended but didn't", DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc());
// Common bug is to forget to set this.doc=NO_MORE_DOCS in the enum!:
- assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.docID());
+ assertEquals("DocsEnum should have ended but didn't", DocsEnum.NO_MORE_DOCS, docsEnum.docID());
}
break;
}
@@ -631,7 +643,7 @@ public class TestPostingsFormat extends
if (VERBOSE) {
System.out.println(" now advance to end (target=" + target + ")");
}
- assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.advance(target));
+ assertEquals("DocsEnum should have ended but didn't", DocsEnum.NO_MORE_DOCS, docsEnum.advance(target));
break;
} else {
posting = expected.get(nextPosting++);
@@ -643,7 +655,7 @@ public class TestPostingsFormat extends
}
}
int docID = docsEnum.advance(targetDocID != -1 ? targetDocID : posting.docID);
- assertEquals(posting.docID, docID);
+ assertEquals("docID is wrong", posting.docID, docID);
}
} else {
posting = expected.get(nextPosting++);
@@ -651,7 +663,7 @@ public class TestPostingsFormat extends
System.out.println(" now nextDoc to " + posting.docID + " (" + nextPosting + " of " + stopAt + ")");
}
int docID = docsEnum.nextDoc();
- assertEquals(posting.docID, docID);
+ assertEquals("docID is wrong", posting.docID, docID);
}
if (doCheckFreqs && random().nextDouble() <= freqAskChance) {
@@ -659,7 +671,7 @@ public class TestPostingsFormat extends
System.out.println(" now freq()=" + posting.positions.size());
}
int freq = docsEnum.freq();
- assertEquals(posting.positions.size(), freq);
+ assertEquals("freq is wrong", posting.positions.size(), freq);
}
if (doCheckPositions) {
@@ -676,7 +688,7 @@ public class TestPostingsFormat extends
if (VERBOSE) {
System.out.println(" now nextPosition to " + position.position);
}
- assertEquals(position.position, docsAndPositionsEnum.nextPosition());
+ assertEquals("position is wrong", position.position, docsAndPositionsEnum.nextPosition());
// nocommit sometimes don't pull the payload even
// though we pulled the position
@@ -687,17 +699,18 @@ public class TestPostingsFormat extends
System.out.println(" now check payload length=" + (position.payload == null ? 0 : position.payload.length));
}
if (position.payload == null || position.payload.length == 0) {
- assertFalse(docsAndPositionsEnum.hasPayload());
+ assertFalse("should not have payload", docsAndPositionsEnum.hasPayload());
} else {
- assertTrue(docsAndPositionsEnum.hasPayload());
+ assertTrue("should have payload but doesn't", docsAndPositionsEnum.hasPayload());
BytesRef payload = docsAndPositionsEnum.getPayload();
- assertFalse(docsAndPositionsEnum.hasPayload());
+ assertFalse("2nd call to hasPayload should be false", docsAndPositionsEnum.hasPayload());
- assertNotNull(payload);
- assertEquals(position.payload.length, payload.length);
+ assertNotNull("payload should not be null", payload);
+ assertEquals("payload length is wrong", position.payload.length, payload.length);
for(int byteUpto=0;byteUpto<position.payload.length;byteUpto++) {
- assertEquals(position.payload[byteUpto],
+ assertEquals("payload bytes are wrong",
+ position.payload[byteUpto],
payload.bytes[payload.offset+byteUpto]);
}
}
@@ -713,16 +726,19 @@ public class TestPostingsFormat extends
if (VERBOSE) {
System.out.println(" now check offsets: startOff=" + position.startOffset + " endOffset=" + position.endOffset);
}
- assertEquals(position.startOffset, docsAndPositionsEnum.startOffset());
- assertEquals(position.endOffset, docsAndPositionsEnum.endOffset());
+ assertEquals("startOffset is wrong", position.startOffset, docsAndPositionsEnum.startOffset());
+ assertEquals("endOffset is wrong", position.endOffset, docsAndPositionsEnum.endOffset());
} else {
if (VERBOSE) {
System.out.println(" skip check offsets");
}
}
} else {
- assertEquals(-1, docsAndPositionsEnum.startOffset());
- assertEquals(-1, docsAndPositionsEnum.endOffset());
+ if (VERBOSE) {
+ System.out.println(" now check offsets are -1");
+ }
+ assertEquals("startOffset isn't -1", -1, docsAndPositionsEnum.startOffset());
+ assertEquals("endOffset isn't -1", -1, docsAndPositionsEnum.endOffset());
}
}
}
@@ -846,6 +862,7 @@ public class TestPostingsFormat extends
boolean indexPayloads = random().nextBoolean();
// nocommit test thread safety of buildIndex too
+
FieldsProducer fieldsProducer = buildIndex(dir, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, indexPayloads);
//testTerms(fieldsProducer, EnumSet.noneOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
@@ -855,7 +872,11 @@ public class TestPostingsFormat extends
//testTerms(fieldsProducer, EnumSet.of(Option.SKIPPING), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
//testTerms(fieldsProducer, EnumSet.of(Option.THREADS, Option.TERM_STATE, Option.SKIPPING, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
//testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.SKIPPING, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.PAYLOADS, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME, Option.SKIPPING), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ //testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.PAYLOADS, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME, Option.SKIPPING), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+
+ // NOTE: you can also test "weaker" index options than
+ // you indexed with:
+ testTerms(fieldsProducer, EnumSet.allOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
fieldsProducer.close();
dir.close();
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1364085&r1=1364084&r2=1364085&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Sat Jul 21 13:31:16 2012
@@ -473,16 +473,6 @@ public abstract class BaseTokenStreamTes
}
}
- static final Set<String> doesntSupportOffsets = new HashSet<String>(Arrays.asList(
- "Lucene3x",
- "MockFixedIntBlock",
- "MockVariableIntBlock",
- "MockSep",
- "MockRandom",
- "For",
- "PFor"
- ));
-
private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect, RandomIndexWriter iw) throws IOException {
final LineFileDocs docs = new LineFileDocs(random);
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1364085&r1=1364084&r2=1364085&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Sat Jul 21 13:31:16 2012
@@ -277,6 +277,15 @@ public abstract class LuceneTestCase ext
CORE_DIRECTORIES.add("RAMDirectory");
};
+ protected static final Set<String> doesntSupportOffsets = new HashSet<String>(Arrays.asList(
+ "Lucene3x",
+ "MockFixedIntBlock",
+ "MockVariableIntBlock",
+ "MockSep",
+ "MockRandom",
+ "For",
+ "PFor"
+ ));
// -----------------------------------------------------------------
// Fields initialized in class or instance rules.