You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2013/08/11 14:19:39 UTC
svn commit: r1512909 [21/38] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/suggest/ dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/core/src/test/ dev-too...
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java Sun Aug 11 12:19:13 2013
@@ -45,9 +45,15 @@ import org.apache.lucene.util.BytesRef;
* {@link FixedGapTermsIndexWriter}.
*/
public final class Lucene41WithOrds extends PostingsFormat {
-
+ final int termIndexInterval;
+
public Lucene41WithOrds() {
+ this(FixedGapTermsIndexWriter.DEFAULT_TERM_INDEX_INTERVAL);
+ }
+
+ public Lucene41WithOrds(int termIndexInterval) {
super("Lucene41WithOrds");
+ this.termIndexInterval = termIndexInterval;
}
@Override
@@ -61,7 +67,7 @@ public final class Lucene41WithOrds exte
TermsIndexWriterBase indexWriter;
boolean success = false;
try {
- indexWriter = new FixedGapTermsIndexWriter(state);
+ indexWriter = new FixedGapTermsIndexWriter(state, termIndexInterval);
success = true;
} finally {
if (!success) {
@@ -87,8 +93,6 @@ public final class Lucene41WithOrds exte
}
}
- public final static int TERMS_CACHE_SIZE = 1024;
-
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postings = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
@@ -99,7 +103,6 @@ public final class Lucene41WithOrds exte
indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
- state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator(),
state.segmentSuffix, state.context);
success = true;
@@ -117,7 +120,6 @@ public final class Lucene41WithOrds exte
state.segmentInfo,
postings,
state.context,
- TERMS_CACHE_SIZE,
state.segmentSuffix);
success = true;
return ret;
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java Sun Aug 11 12:19:13 2013
@@ -169,7 +169,6 @@ public final class MockFixedIntBlockPost
indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
- state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator(), state.segmentSuffix,
IOContext.DEFAULT);
success = true;
@@ -187,7 +186,6 @@ public final class MockFixedIntBlockPost
state.segmentInfo,
postingsReader,
state.context,
- 1024,
state.segmentSuffix);
success = true;
return ret;
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java Sun Aug 11 12:19:13 2013
@@ -194,7 +194,6 @@ public final class MockVariableIntBlockP
indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
- state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator(),
state.segmentSuffix, state.context);
success = true;
@@ -212,7 +211,6 @@ public final class MockVariableIntBlockP
state.segmentInfo,
postingsReader,
state.context,
- 1024,
state.segmentSuffix);
success = true;
return ret;
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Sun Aug 11 12:19:13 2013
@@ -220,11 +220,11 @@ public final class MockRandomPostingsFor
final TermsIndexWriterBase indexWriter;
try {
if (random.nextBoolean()) {
- state.termIndexInterval = _TestUtil.nextInt(random, 1, 100);
+ int termIndexInterval = _TestUtil.nextInt(random, 1, 100);
if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: fixed-gap terms index (tii=" + state.termIndexInterval + ")");
+ System.out.println("MockRandomCodec: fixed-gap terms index (tii=" + termIndexInterval + ")");
}
- indexWriter = new FixedGapTermsIndexWriter(state);
+ indexWriter = new FixedGapTermsIndexWriter(state, termIndexInterval);
} else {
final VariableGapTermsIndexWriter.IndexTermSelector selector;
final int n2 = random.nextInt(3);
@@ -340,8 +340,7 @@ public final class MockRandomPostingsFor
state.segmentInfo,
postingsReader,
state.context,
- state.segmentSuffix,
- state.termsIndexDivisor);
+ state.segmentSuffix);
success = true;
} finally {
if (!success) {
@@ -359,20 +358,14 @@ public final class MockRandomPostingsFor
final boolean doFixedGap = random.nextBoolean();
// randomness diverges from writer, here:
- if (state.termsIndexDivisor != -1) {
- state.termsIndexDivisor = _TestUtil.nextInt(random, 1, 10);
- }
if (doFixedGap) {
- // if termsIndexDivisor is set to -1, we should not touch it. It means a
- // test explicitly instructed not to load the terms index.
if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: fixed-gap terms index (divisor=" + state.termsIndexDivisor + ")");
+ System.out.println("MockRandomCodec: fixed-gap terms index");
}
indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
- state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator(),
state.segmentSuffix, state.context);
} else {
@@ -383,12 +376,11 @@ public final class MockRandomPostingsFor
random.nextLong();
}
if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: variable-gap terms index (divisor=" + state.termsIndexDivisor + ")");
+ System.out.println("MockRandomCodec: variable-gap terms index");
}
indexReader = new VariableGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
- state.termsIndexDivisor,
state.segmentSuffix, state.context);
}
@@ -400,8 +392,6 @@ public final class MockRandomPostingsFor
}
}
- final int termsCacheSize = _TestUtil.nextInt(random, 1, 1024);
-
success = false;
try {
fields = new BlockTermsReader(indexReader,
@@ -410,7 +400,6 @@ public final class MockRandomPostingsFor
state.segmentInfo,
postingsReader,
state.context,
- termsCacheSize,
state.segmentSuffix);
success = true;
} finally {
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java Sun Aug 11 12:19:13 2013
@@ -92,7 +92,6 @@ public final class MockSepPostingsFormat
indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
- state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator(),
state.segmentSuffix, state.context);
success = true;
@@ -110,7 +109,6 @@ public final class MockSepPostingsFormat
state.segmentInfo,
postingsReader,
state.context,
- 1024,
state.segmentSuffix);
success = true;
return ret;
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java Sun Aug 11 12:19:13 2013
@@ -84,8 +84,7 @@ public final class NestedPulsingPostings
state.directory, state.fieldInfos, state.segmentInfo,
pulsingReader,
state.context,
- state.segmentSuffix,
- state.termsIndexDivisor);
+ state.segmentSuffix);
success = true;
return ret;
} finally {
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java Sun Aug 11 12:19:13 2013
@@ -336,7 +336,7 @@ public final class RAMOnlyPostingsFormat
}
@Override
- public SeekStatus seekCeil(BytesRef term, boolean useCache) {
+ public SeekStatus seekCeil(BytesRef term) {
current = term.utf8ToString();
it = null;
if (ramField.termToDocs.containsKey(current)) {
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java Sun Aug 11 12:19:13 2013
@@ -185,9 +185,9 @@ public class AssertingAtomicReader exten
}
@Override
- public SeekStatus seekCeil(BytesRef term, boolean useCache) throws IOException {
+ public SeekStatus seekCeil(BytesRef term) throws IOException {
assert term.isValid();
- SeekStatus result = super.seekCeil(term, useCache);
+ SeekStatus result = super.seekCeil(term);
if (result == SeekStatus.END) {
state = State.UNPOSITIONED;
} else {
@@ -197,9 +197,9 @@ public class AssertingAtomicReader exten
}
@Override
- public boolean seekExact(BytesRef text, boolean useCache) throws IOException {
+ public boolean seekExact(BytesRef text) throws IOException {
assert text.isValid();
- if (super.seekExact(text, useCache)) {
+ if (super.seekExact(text)) {
state = State.POSITIONED;
return true;
} else {
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Sun Aug 11 12:19:13 2013
@@ -746,16 +746,16 @@ public abstract class BaseDocValuesForma
assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("zzz")));
// seekExact()
- assertTrue(termsEnum.seekExact(new BytesRef("beer"), true));
+ assertTrue(termsEnum.seekExact(new BytesRef("beer")));
assertEquals("beer", termsEnum.term().utf8ToString());
assertEquals(0, termsEnum.ord());
- assertTrue(termsEnum.seekExact(new BytesRef("hello"), true));
+ assertTrue(termsEnum.seekExact(new BytesRef("hello")));
assertEquals(Codec.getDefault().toString(), "hello", termsEnum.term().utf8ToString());
assertEquals(1, termsEnum.ord());
- assertTrue(termsEnum.seekExact(new BytesRef("world"), true));
+ assertTrue(termsEnum.seekExact(new BytesRef("world")));
assertEquals("world", termsEnum.term().utf8ToString());
assertEquals(2, termsEnum.ord());
- assertFalse(termsEnum.seekExact(new BytesRef("bogus"), true));
+ assertFalse(termsEnum.seekExact(new BytesRef("bogus")));
// seek(ord)
termsEnum.seekExact(0);
@@ -1022,7 +1022,7 @@ public abstract class BaseDocValuesForma
writer.close(true);
- DirectoryReader reader = DirectoryReader.open(dir, 1);
+ DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
IndexSearcher searcher = new IndexSearcher(reader);
@@ -1350,6 +1350,57 @@ public abstract class BaseDocValuesForma
dir.close();
}
+ private void doTestSortedVsFieldCache(int minLength, int maxLength) throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
+ Document doc = new Document();
+ Field idField = new StringField("id", "", Field.Store.NO);
+ Field indexedField = new StringField("indexed", "", Field.Store.NO);
+ Field dvField = new SortedDocValuesField("dv", new BytesRef());
+ doc.add(idField);
+ doc.add(indexedField);
+ doc.add(dvField);
+
+ // index some docs
+ int numDocs = atLeast(300);
+ for (int i = 0; i < numDocs; i++) {
+ idField.setStringValue(Integer.toString(i));
+ final int length;
+ if (minLength == maxLength) {
+ length = minLength; // fixed length
+ } else {
+ length = _TestUtil.nextInt(random(), minLength, maxLength);
+ }
+ String value = _TestUtil.randomSimpleString(random(), length);
+ indexedField.setStringValue(value);
+ dvField.setBytesValue(new BytesRef(value));
+ writer.addDocument(doc);
+ if (random().nextInt(31) == 0) {
+ writer.commit();
+ }
+ }
+
+ // delete some docs
+ int numDeletions = random().nextInt(numDocs/10);
+ for (int i = 0; i < numDeletions; i++) {
+ int id = random().nextInt(numDocs);
+ writer.deleteDocuments(new Term("id", Integer.toString(id)));
+ }
+ writer.close();
+
+ // compare
+ DirectoryReader ir = DirectoryReader.open(dir);
+ for (AtomicReaderContext context : ir.leaves()) {
+ AtomicReader r = context.reader();
+ SortedDocValues expected = FieldCache.DEFAULT.getTermsIndex(r, "indexed");
+ SortedDocValues actual = r.getSortedDocValues("dv");
+ assertEquals(r.maxDoc(), expected, actual);
+ }
+ ir.close();
+ dir.close();
+ }
+
public void testSortedFixedLengthVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1358,6 +1409,21 @@ public abstract class BaseDocValuesForma
}
}
+ public void testSortedFixedLengthVsFieldCache() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ int fixedLength = _TestUtil.nextInt(random(), 1, 10);
+ doTestSortedVsFieldCache(fixedLength, fixedLength);
+ }
+ }
+
+ public void testSortedVariableLengthVsFieldCache() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestSortedVsFieldCache(1, 10);
+ }
+ }
+
public void testSortedVariableLengthVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1788,16 +1854,16 @@ public abstract class BaseDocValuesForma
assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("zzz")));
// seekExact()
- assertTrue(termsEnum.seekExact(new BytesRef("beer"), true));
+ assertTrue(termsEnum.seekExact(new BytesRef("beer")));
assertEquals("beer", termsEnum.term().utf8ToString());
assertEquals(0, termsEnum.ord());
- assertTrue(termsEnum.seekExact(new BytesRef("hello"), true));
+ assertTrue(termsEnum.seekExact(new BytesRef("hello")));
assertEquals("hello", termsEnum.term().utf8ToString());
assertEquals(1, termsEnum.ord());
- assertTrue(termsEnum.seekExact(new BytesRef("world"), true));
+ assertTrue(termsEnum.seekExact(new BytesRef("world")));
assertEquals("world", termsEnum.term().utf8ToString());
assertEquals(2, termsEnum.ord());
- assertFalse(termsEnum.seekExact(new BytesRef("bogus"), true));
+ assertFalse(termsEnum.seekExact(new BytesRef("bogus")));
// seek(ord)
termsEnum.seekExact(0);
@@ -1905,6 +1971,10 @@ public abstract class BaseDocValuesForma
}
}
+ private void assertEquals(int maxDoc, SortedDocValues expected, SortedDocValues actual) throws Exception {
+ assertEquals(maxDoc, new SingletonSortedSetDocValues(expected), new SingletonSortedSetDocValues(actual));
+ }
+
private void assertEquals(int maxDoc, SortedSetDocValues expected, SortedSetDocValues actual) throws Exception {
// can be null for the segment if no docs actually had any SortedDocValues
// in this case FC.getDocTermsOrds returns EMPTY
@@ -1932,6 +2002,74 @@ public abstract class BaseDocValuesForma
actual.lookupTerm(actualBytes);
assertEquals(expectedBytes, actualBytes);
}
+
+ // compare termsenum
+ assertEquals(expected.getValueCount(), expected.termsEnum(), actual.termsEnum());
+ }
+
+ private void assertEquals(long numOrds, TermsEnum expected, TermsEnum actual) throws Exception {
+ BytesRef ref;
+
+ // sequential next() through all terms
+ while ((ref = expected.next()) != null) {
+ assertEquals(ref, actual.next());
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+ assertNull(actual.next());
+
+ // sequential seekExact(ord) through all terms
+ for (long i = 0; i < numOrds; i++) {
+ expected.seekExact(i);
+ actual.seekExact(i);
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+
+ // sequential seekExact(BytesRef) through all terms
+ for (long i = 0; i < numOrds; i++) {
+ expected.seekExact(i);
+ assertTrue(actual.seekExact(expected.term()));
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+
+ // sequential seekCeil(BytesRef) through all terms
+ for (long i = 0; i < numOrds; i++) {
+ expected.seekExact(i);
+ assertEquals(SeekStatus.FOUND, actual.seekCeil(expected.term()));
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+
+ // random seekExact(ord)
+ for (long i = 0; i < numOrds; i++) {
+ long randomOrd = _TestUtil.nextLong(random(), 0, numOrds-1);
+ expected.seekExact(randomOrd);
+ actual.seekExact(randomOrd);
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+
+ // random seekExact(BytesRef)
+ for (long i = 0; i < numOrds; i++) {
+ long randomOrd = _TestUtil.nextLong(random(), 0, numOrds-1);
+ expected.seekExact(randomOrd);
+ actual.seekExact(expected.term());
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+
+ // random seekCeil(BytesRef)
+ for (long i = 0; i < numOrds; i++) {
+ BytesRef target = new BytesRef(_TestUtil.randomUnicodeString(random()));
+ SeekStatus expectedStatus = expected.seekCeil(target);
+ assertEquals(expectedStatus, actual.seekCeil(target));
+ if (expectedStatus != SeekStatus.END) {
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+ }
}
private void doTestSortedSetVsUninvertedField(int minLength, int maxLength) throws Exception {
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java Sun Aug 11 12:19:13 2013
@@ -38,6 +38,8 @@ import org.apache.lucene.codecs.FieldsPr
import org.apache.lucene.codecs.PostingsConsumer;
import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.codecs.TermsConsumer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
@@ -353,9 +355,12 @@ public abstract class BasePostingsFormat
fields.put(field, postings);
Set<String> seenTerms = new HashSet<String>();
- // TODO:
- //final int numTerms = atLeast(10);
- final int numTerms = 4;
+ int numTerms;
+ if (random().nextInt(10) == 7) {
+ numTerms = atLeast(50);
+ } else {
+ numTerms = _TestUtil.nextInt(random(), 2, 20);
+ }
for(int termUpto=0;termUpto<numTerms;termUpto++) {
String term = _TestUtil.randomSimpleString(random());
@@ -483,7 +488,7 @@ public abstract class BasePostingsFormat
SegmentWriteState writeState = new SegmentWriteState(null, dir,
segmentInfo, newFieldInfos,
- 32, null, new IOContext(new FlushInfo(maxDoc, bytes)));
+ null, new IOContext(new FlushInfo(maxDoc, bytes)));
FieldsConsumer fieldsConsumer = codec.postingsFormat().fieldsConsumer(writeState);
for(Map.Entry<String,Map<BytesRef,Long>> fieldEnt : fields.entrySet()) {
@@ -567,7 +572,7 @@ public abstract class BasePostingsFormat
currentFieldInfos = newFieldInfos;
- SegmentReadState readState = new SegmentReadState(dir, segmentInfo, newFieldInfos, IOContext.DEFAULT, 1);
+ SegmentReadState readState = new SegmentReadState(dir, segmentInfo, newFieldInfos, IOContext.DEFAULT);
return codec.postingsFormat().fieldsProducer(readState);
}
@@ -595,6 +600,10 @@ public abstract class BasePostingsFormat
System.out.println(" verifyEnum: options=" + options + " maxTestOptions=" + maxTestOptions);
}
+ // Make sure TermsEnum really is positioned on the
+ // expected term:
+ assertEquals(term, termsEnum.term());
+
// 50% of the time time pass liveDocs:
boolean useLiveDocs = options.contains(Option.LIVE_DOCS) && random().nextBoolean();
Bits liveDocs;
@@ -983,7 +992,7 @@ public abstract class BasePostingsFormat
termsEnum = terms.iterator(null);
if (!useTermState) {
- assertTrue(termsEnum.seekExact(fieldAndTerm.term, true));
+ assertTrue(termsEnum.seekExact(fieldAndTerm.term));
} else {
termsEnum.seekExact(fieldAndTerm.term, termState);
}
@@ -1130,4 +1139,89 @@ public abstract class BasePostingsFormat
_TestUtil.rmDir(path);
}
}
+
+ public void testEmptyField() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ iwc.setCodec(getCodec());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ doc.add(newStringField("", "something", Field.Store.NO));
+ iw.addDocument(doc);
+ DirectoryReader ir = iw.getReader();
+ AtomicReader ar = getOnlySegmentReader(ir);
+ Fields fields = ar.fields();
+ int fieldCount = fields.size();
+ // -1 is allowed, if the codec doesn't implement fields.size():
+ assertTrue(fieldCount == 1 || fieldCount == -1);
+ Terms terms = ar.terms("");
+ assertNotNull(terms);
+ TermsEnum termsEnum = terms.iterator(null);
+ assertNotNull(termsEnum.next());
+ assertEquals(termsEnum.term(), new BytesRef("something"));
+ assertNull(termsEnum.next());
+ ir.close();
+ iw.close();
+ dir.close();
+ }
+
+ public void testEmptyFieldAndEmptyTerm() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ iwc.setCodec(getCodec());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ doc.add(newStringField("", "", Field.Store.NO));
+ iw.addDocument(doc);
+ DirectoryReader ir = iw.getReader();
+ AtomicReader ar = getOnlySegmentReader(ir);
+ Fields fields = ar.fields();
+ int fieldCount = fields.size();
+ // -1 is allowed, if the codec doesn't implement fields.size():
+ assertTrue(fieldCount == 1 || fieldCount == -1);
+ Terms terms = ar.terms("");
+ assertNotNull(terms);
+ TermsEnum termsEnum = terms.iterator(null);
+ assertNotNull(termsEnum.next());
+ assertEquals(termsEnum.term(), new BytesRef(""));
+ assertNull(termsEnum.next());
+ ir.close();
+ iw.close();
+ dir.close();
+ }
+
+ // tests that ghost fields still work
+ // TODO: can this be improved?
+ public void testGhosts() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ iwc.setCodec(getCodec());
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ iw.addDocument(doc);
+ doc.add(newStringField("ghostField", "something", Field.Store.NO));
+ iw.addDocument(doc);
+ iw.forceMerge(1);
+ iw.deleteDocuments(new Term("ghostField", "something")); // delete the only term for the field
+ iw.forceMerge(1);
+ DirectoryReader ir = iw.getReader();
+ AtomicReader ar = getOnlySegmentReader(ir);
+ Fields fields = ar.fields();
+ // Ghost busting terms dict impls will have
+ // fields.size() == 0; all others must be == 1:
+ assertTrue(fields.size() <= 1);
+ Terms terms = fields.terms("ghostField");
+ if (terms != null) {
+ TermsEnum termsEnum = terms.iterator(null);
+ BytesRef term = termsEnum.next();
+ if (term != null) {
+ DocsEnum docsEnum = termsEnum.docs(null, null);
+ assertTrue(docsEnum.nextDoc() == DocsEnum.NO_MORE_DOCS);
+ }
+ }
+ ir.close();
+ iw.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java Sun Aug 11 12:19:13 2013
@@ -94,7 +94,7 @@ public abstract class BaseStoredFieldsFo
Directory dir = newDirectory();
Random rand = random();
RandomIndexWriter w = new RandomIndexWriter(rand, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(_TestUtil.nextInt(rand, 5, 20)));
- //w.w.setUseCompoundFile(false);
+ //w.w.setNoCFSRatio(0.0);
final int docCount = atLeast(200);
final int fieldCount = _TestUtil.nextInt(rand, 1, 5);
@@ -509,7 +509,7 @@ public abstract class BaseStoredFieldsFo
Directory dir = newDirectory();
IndexWriterConfig iwConf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf.clone());
final int docCount = atLeast(200);
final byte[][][] data = new byte [docCount][][];
@@ -548,7 +548,7 @@ public abstract class BaseStoredFieldsFo
} else {
iwConf.setCodec(otherCodec);
}
- iw = new RandomIndexWriter(random(), dir, iwConf);
+ iw = new RandomIndexWriter(random(), dir, iwConf.clone());
}
}
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java Sun Aug 11 12:19:13 2013
@@ -500,9 +500,9 @@ public abstract class BaseTermVectorsFor
assertNull(termsEnum.next());
for (int i = 0; i < 5; ++i) {
if (random().nextBoolean()) {
- assertTrue(termsEnum.seekExact(RandomPicks.randomFrom(random(), tk.termBytes), random().nextBoolean()));
+ assertTrue(termsEnum.seekExact(RandomPicks.randomFrom(random(), tk.termBytes)));
} else {
- assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(RandomPicks.randomFrom(random(), tk.termBytes), random().nextBoolean()));
+ assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(RandomPicks.randomFrom(random(), tk.termBytes)));
}
}
}
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java Sun Aug 11 12:19:13 2013
@@ -272,7 +272,7 @@ class DocHelper {
public static SegmentInfoPerCommit writeDoc(Random random, Directory dir, Analyzer analyzer, Similarity similarity, Document doc) throws IOException {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( /* LuceneTestCase.newIndexWriterConfig(random, */
TEST_VERSION_CURRENT, analyzer).setSimilarity(similarity == null ? IndexSearcher.getDefaultSimilarity() : similarity));
- //writer.setUseCompoundFile(false);
+ //writer.setNoCFSRatio(0.0);
writer.addDocument(doc);
writer.commit();
SegmentInfoPerCommit info = writer.newestSegment();
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Sun Aug 11 12:19:13 2013
@@ -33,6 +33,8 @@ import org.apache.lucene.codecs.assertin
import org.apache.lucene.codecs.asserting.AssertingPostingsFormat;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds;
+import org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapDocFreqInterval;
+import org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapFixedInterval;
import org.apache.lucene.codecs.lucene42.Lucene42Codec;
import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene41Postings;
@@ -137,7 +139,9 @@ public class RandomCodec extends Lucene4
new MockVariableIntBlockPostingsFormat( _TestUtil.nextInt(random, 1, 127)),
new MockRandomPostingsFormat(random),
new NestedPulsingPostingsFormat(),
- new Lucene41WithOrds(),
+ new Lucene41WithOrds(_TestUtil.nextInt(random, 1, 1000)),
+ new Lucene41VarGapFixedInterval(_TestUtil.nextInt(random, 1, 1000)),
+ new Lucene41VarGapDocFreqInterval(_TestUtil.nextInt(random, 1, 100), _TestUtil.nextInt(random, 1, 1000)),
new SimpleTextPostingsFormat(),
new AssertingPostingsFormat(),
new MemoryPostingsFormat(true, random.nextFloat()),
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Sun Aug 11 12:19:13 2013
@@ -25,17 +25,12 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.IndexWriter; // javadoc
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NullInfoStream;
import org.apache.lucene.util.Version;
import org.apache.lucene.util._TestUtil;
@@ -55,23 +50,22 @@ public class RandomIndexWriter implement
private boolean getReaderCalled;
private final Codec codec; // sugar
- // Randomly calls Thread.yield so we mixup thread scheduling
- private static final class MockIndexWriter extends IndexWriter {
-
- private final Random r;
-
- public MockIndexWriter(Random r, Directory dir, IndexWriterConfig conf) throws IOException {
- super(dir, conf);
- // TODO: this should be solved in a different way; Random should not be shared (!).
- this.r = new Random(r.nextLong());
- }
-
- @Override
- boolean testPoint(String name) {
- if (r.nextInt(4) == 2)
- Thread.yield();
- return true;
- }
+
+ public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException {
+ // Randomly calls Thread.yield so we mixup thread scheduling
+ final Random random = new Random(r.nextLong());
+ return mockIndexWriter(dir, conf, new TestPoint() {
+ @Override
+ public void apply(String message) {
+ if (random.nextInt(4) == 2)
+ Thread.yield();
+ }
+ });
+ }
+
+ public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, TestPoint testPoint) throws IOException {
+ conf.setInfoStream(new TestPointInfoStream(conf.getInfoStream(), testPoint));
+ return new IndexWriter(dir, conf);
}
/** create a RandomIndexWriter with a random config: Uses TEST_VERSION_CURRENT and MockAnalyzer */
@@ -93,7 +87,7 @@ public class RandomIndexWriter implement
public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c) throws IOException {
// TODO: this should be solved in a different way; Random should not be shared (!).
this.r = new Random(r.nextLong());
- w = new MockIndexWriter(r, dir, c);
+ w = mockIndexWriter(dir, c, r);
flushAt = _TestUtil.nextInt(r, 10, 1000);
codec = w.getConfig().getCodec();
if (LuceneTestCase.VERBOSE) {
@@ -314,7 +308,7 @@ public class RandomIndexWriter implement
}
w.commit();
if (r.nextBoolean()) {
- return DirectoryReader.open(w.getDirectory(), _TestUtil.nextInt(r, 1, 10));
+ return DirectoryReader.open(w.getDirectory());
} else {
return w.getReader(applyDeletions);
}
@@ -345,4 +339,42 @@ public class RandomIndexWriter implement
public void forceMerge(int maxSegmentCount) throws IOException {
w.forceMerge(maxSegmentCount);
}
+
+ private static final class TestPointInfoStream extends InfoStream {
+ private final InfoStream delegate;
+ private final TestPoint testPoint;
+
+ public TestPointInfoStream(InfoStream delegate, TestPoint testPoint) {
+ this.delegate = delegate == null ? new NullInfoStream(): delegate;
+ this.testPoint = testPoint;
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+ @Override
+ public void message(String component, String message) {
+ if ("TP".equals(component)) {
+ testPoint.apply(message);
+ }
+ if (delegate.isEnabled(component)) {
+ delegate.message(component, message);
+ }
+ }
+
+ @Override
+ public boolean isEnabled(String component) {
+ return "TP".equals(component) || delegate.isEnabled(component);
+ }
+ }
+
+ /**
+ * Simple interface that is executed for each <tt>TP</tt> {@link InfoStream} component
+ * message. See also {@link RandomIndexWriter#mockIndexWriter(Directory, IndexWriterConfig, TestPoint)}
+ */
+ public static interface TestPoint {
+ public abstract void apply(String message);
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java Sun Aug 11 12:19:13 2013
@@ -185,7 +185,7 @@ public abstract class ShardSearchingTest
}
try {
for(Term term : terms) {
- final TermContext termContext = TermContext.build(s.getIndexReader().getContext(), term, false);
+ final TermContext termContext = TermContext.build(s.getIndexReader().getContext(), term);
stats.put(term, s.termStatistics(term, termContext));
}
} finally {
@@ -370,20 +370,35 @@ public abstract class ShardSearchingTest
@Override
public TopDocs searchAfter(ScoreDoc after, Query query, int numHits) throws IOException {
final TopDocs[] shardHits = new TopDocs[nodeVersions.length];
+ // results are merged in that order: score, shardIndex, doc. therefore we set
+ // after to after.score and depending on the nodeID we set doc to either:
+ // - not collect any more documents with that score (only with worse score)
+ // - collect more documents with that score (and worse) following the last collected document
+ // - collect all documents with that score (and worse)
ScoreDoc shardAfter = new ScoreDoc(after.doc, after.score);
- for(int nodeID=0;nodeID<nodeVersions.length;nodeID++) {
+ for (int nodeID = 0; nodeID < nodeVersions.length; nodeID++) {
if (nodeID < after.shardIndex) {
- // If score is tied then no docs in this shard
- // should be collected:
- shardAfter.doc = Integer.MAX_VALUE;
+ // all documents with after.score were already collected, so collect
+ // only documents with worse scores.
+ final NodeState.ShardIndexSearcher s = nodes[nodeID].acquire(nodeVersions);
+ try {
+ // Setting after.doc to reader.maxDoc-1 is a way to tell
+ // TopScoreDocCollector that no more docs with that score should
+ // be collected. note that in practice the shard which sends the
+ // request to a remote shard won't have reader.maxDoc at hand, so
+ // it will send some arbitrary value which will be fixed on the
+ // other end.
+ shardAfter.doc = s.getIndexReader().maxDoc() - 1;
+ } finally {
+ nodes[nodeID].release(s);
+ }
} else if (nodeID == after.shardIndex) {
- // If score is tied then we break according to
- // docID (like normal):
+ // collect all documents following the last collected doc with
+ // after.score + documents with worse scores.
shardAfter.doc = after.doc;
} else {
- // If score is tied then all docs in this shard
- // should be collected, because they come after
- // the previous bottom:
+ // all documents with after.score (and worse) should be collected
+ // because they didn't make it to top-N in the previous round.
shardAfter.doc = -1;
}
if (nodeID == myNodeID) {
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Sun Aug 11 12:19:13 2013
@@ -20,6 +20,7 @@ package org.apache.lucene.store;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -385,7 +386,7 @@ public class MockDirectoryWrapper extend
if (randomState.nextBoolean()) {
throw new IOException("a random IOException (" + name + ")");
} else {
- throw new FileNotFoundException("a random IOException (" + name + ")");
+ throw randomState.nextBoolean() ? new FileNotFoundException("a random IOException (" + name + ")") : new NoSuchFileException("a random IOException (" + name + ")");
}
}
}
@@ -544,7 +545,7 @@ public class MockDirectoryWrapper extend
maybeThrowDeterministicException();
}
if (!delegate.fileExists(name)) {
- throw new FileNotFoundException(name + " in dir=" + delegate);
+ throw randomState.nextBoolean() ? new FileNotFoundException(name + " in dir=" + delegate) : new NoSuchFileException(name + " in dir=" + delegate);
}
// cannot open a file for input if it's still open for
@@ -920,7 +921,7 @@ public class MockDirectoryWrapper extend
throws IOException {
maybeYield();
if (!delegate.fileExists(name)) {
- throw new FileNotFoundException(name);
+ throw randomState.nextBoolean() ? new FileNotFoundException(name) : new NoSuchFileException(name);
}
// cannot open a file for input if it's still open for
// output, except for segments.gen and segments_N
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java Sun Aug 11 12:19:13 2013
@@ -36,6 +36,7 @@ import java.util.zip.GZIPInputStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.IntField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
@@ -158,6 +159,7 @@ public class LineFileDocs implements Clo
final Field titleDV;
final Field body;
final Field id;
+ final Field idNum;
final Field date;
public DocState(boolean useDocValues) {
@@ -180,6 +182,9 @@ public class LineFileDocs implements Clo
id = new StringField("docid", "", Field.Store.YES);
doc.add(id);
+ idNum = new IntField("docid_int", 0, Field.Store.NO);
+ doc.add(idNum);
+
date = new StringField("date", "", Field.Store.YES);
doc.add(date);
@@ -233,7 +238,9 @@ public class LineFileDocs implements Clo
}
docState.titleTokenized.setStringValue(title);
docState.date.setStringValue(line.substring(1+spot, spot2));
- docState.id.setStringValue(Integer.toString(id.getAndIncrement()));
+ final int i = id.getAndIncrement();
+ docState.id.setStringValue(Integer.toString(i));
+ docState.idNum.setIntValue(i);
return docState.doc;
}
}
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Sun Aug 11 12:19:13 2013
@@ -737,6 +737,12 @@ public abstract class LuceneTestCase ext
if (r.nextBoolean()) {
c.setMergeScheduler(new SerialMergeScheduler());
+ } else if (rarely(r)) {
+ int maxThreadCount = _TestUtil.nextInt(random(), 1, 4);
+ int maxMergeCount = _TestUtil.nextInt(random(), maxThreadCount, maxThreadCount+4);
+ ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+ cms.setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
+ c.setMergeScheduler(cms);
}
if (r.nextBoolean()) {
if (rarely(r)) {
@@ -748,15 +754,6 @@ public abstract class LuceneTestCase ext
}
}
if (r.nextBoolean()) {
- if (rarely(r)) {
- // crazy value
- c.setTermIndexInterval(r.nextBoolean() ? _TestUtil.nextInt(r, 1, 31) : _TestUtil.nextInt(r, 129, 1000));
- } else {
- // reasonable value
- c.setTermIndexInterval(_TestUtil.nextInt(r, 32, 128));
- }
- }
- if (r.nextBoolean()) {
int maxNumThreadStates = rarely(r) ? _TestUtil.nextInt(r, 5, 20) // crazy value
: _TestUtil.nextInt(r, 1, 4); // reasonable value
@@ -796,23 +793,31 @@ public abstract class LuceneTestCase ext
}
}
- if (rarely(r)) {
- c.setMergePolicy(new MockRandomMergePolicy(r));
- } else if (r.nextBoolean()) {
- c.setMergePolicy(newTieredMergePolicy());
- } else if (r.nextInt(5) == 0) {
- c.setMergePolicy(newAlcoholicMergePolicy());
- } else {
- c.setMergePolicy(newLogMergePolicy());
- }
+ c.setMergePolicy(newMergePolicy(r));
+
if (rarely(r)) {
c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream()));
}
+ c.setUseCompoundFile(r.nextBoolean());
c.setReaderPooling(r.nextBoolean());
- c.setReaderTermsIndexDivisor(_TestUtil.nextInt(r, 1, 4));
return c;
}
+ public static MergePolicy newMergePolicy(Random r) {
+ if (rarely(r)) {
+ return new MockRandomMergePolicy(r);
+ } else if (r.nextBoolean()) {
+ return newTieredMergePolicy(r);
+ } else if (r.nextInt(5) == 0) {
+ return newAlcoholicMergePolicy(r, classEnvRule.timeZone);
+ }
+ return newLogMergePolicy(r);
+ }
+
+ public static MergePolicy newMergePolicy() {
+ return newMergePolicy(random());
+ }
+
public static LogMergePolicy newLogMergePolicy() {
return newLogMergePolicy(random());
}
@@ -831,19 +836,28 @@ public abstract class LuceneTestCase ext
public static LogMergePolicy newLogMergePolicy(Random r) {
LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy();
- logmp.setUseCompoundFile(r.nextBoolean());
logmp.setCalibrateSizeByDeletes(r.nextBoolean());
if (rarely(r)) {
logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 9));
} else {
logmp.setMergeFactor(_TestUtil.nextInt(r, 10, 50));
}
- logmp.setUseCompoundFile(r.nextBoolean());
- logmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
+ configureRandom(r, logmp);
+ return logmp;
+ }
+
+ private static void configureRandom(Random r, MergePolicy mergePolicy) {
+ if (r.nextBoolean()) {
+ mergePolicy.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
+ } else {
+ mergePolicy.setNoCFSRatio(r.nextBoolean() ? 1.0 : 0.0);
+ }
+
if (rarely()) {
- logmp.setMaxCFSSegmentSizeMB(0.2 + r.nextDouble() * 2.0);
+ mergePolicy.setMaxCFSSegmentSizeMB(0.2 + r.nextDouble() * 2.0);
+ } else {
+ mergePolicy.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
}
- return logmp;
}
public static TieredMergePolicy newTieredMergePolicy(Random r) {
@@ -867,29 +881,25 @@ public abstract class LuceneTestCase ext
} else {
tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 10, 50));
}
- tmp.setUseCompoundFile(r.nextBoolean());
- tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
- if (rarely()) {
- tmp.setMaxCFSSegmentSizeMB(0.2 + r.nextDouble() * 2.0);
- }
+ configureRandom(r, tmp);
tmp.setReclaimDeletesWeight(r.nextDouble()*4);
return tmp;
}
- public static LogMergePolicy newLogMergePolicy(boolean useCFS) {
- LogMergePolicy logmp = newLogMergePolicy();
- logmp.setUseCompoundFile(useCFS);
+ public static MergePolicy newLogMergePolicy(boolean useCFS) {
+ MergePolicy logmp = newLogMergePolicy();
+ logmp.setNoCFSRatio(useCFS ? 1.0 : 0.0);
return logmp;
}
- public static LogMergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) {
+ public static MergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) {
LogMergePolicy logmp = newLogMergePolicy();
- logmp.setUseCompoundFile(useCFS);
+ logmp.setNoCFSRatio(useCFS ? 1.0 : 0.0);
logmp.setMergeFactor(mergeFactor);
return logmp;
}
- public static LogMergePolicy newLogMergePolicy(int mergeFactor) {
+ public static MergePolicy newLogMergePolicy(int mergeFactor) {
LogMergePolicy logmp = newLogMergePolicy();
logmp.setMergeFactor(mergeFactor);
return logmp;
@@ -1739,14 +1749,13 @@ public abstract class LuceneTestCase ext
rightEnum = rightTerms.iterator(rightEnum);
}
- final boolean useCache = random().nextBoolean();
final boolean seekExact = random().nextBoolean();
if (seekExact) {
- assertEquals(info, leftEnum.seekExact(b, useCache), rightEnum.seekExact(b, useCache));
+ assertEquals(info, leftEnum.seekExact(b), rightEnum.seekExact(b));
} else {
- SeekStatus leftStatus = leftEnum.seekCeil(b, useCache);
- SeekStatus rightStatus = rightEnum.seekCeil(b, useCache);
+ SeekStatus leftStatus = leftEnum.seekCeil(b);
+ SeekStatus rightStatus = rightEnum.seekCeil(b);
assertEquals(info, leftStatus, rightStatus);
if (leftStatus != SeekStatus.END) {
assertEquals(info, leftEnum.term(), rightEnum.term());
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java Sun Aug 11 12:19:13 2013
@@ -88,12 +88,27 @@ public final class RunListenerPrintRepro
@Override
public void testFinished(Description description) throws Exception {
if (testFailed) {
- reportAdditionalFailureInfo(description.getMethodName());
+ reportAdditionalFailureInfo(
+ stripTestNameAugmentations(
+ description.getMethodName()));
}
scope = LifecycleScope.SUITE;
testFailed = false;
}
+ /**
+ * The {@link Description} object in JUnit does not expose the actual test method,
+ * instead it has the concept of a unique "name" of a test. To run the same method (tests)
+ * repeatedly, randomizedtesting must make those "names" unique: it appends the current iteration
+ * and seeds to the test method's name. We strip this information here.
+ */
+ private String stripTestNameAugmentations(String methodName) {
+ if (methodName != null) {
+ methodName = methodName.replaceAll("\\s*\\{.+?\\}", "");
+ }
+ return methodName;
+ }
+
@Override
public void testRunFinished(Result result) throws Exception {
if (printDiagnosticsAfterClass || LuceneTestCase.VERBOSE) {
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java Sun Aug 11 12:19:13 2013
@@ -18,10 +18,12 @@ package org.apache.lucene.util;
*/
import java.io.PrintStream;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
@@ -84,6 +86,9 @@ final class TestRuleSetupAndRestoreClass
@Override
public void message(String component, String message) {
+ if ("TP".equals(component)) {
+ return; // ignore test points!
+ }
final String name;
if (Thread.currentThread().getName().startsWith("TEST-")) {
// The name of the main thread is way too
@@ -170,8 +175,9 @@ final class TestRuleSetupAndRestoreClass
final DocValuesFormat dvFormat;
if ("random".equals(TEST_DOCVALUESFORMAT)) {
// pick one from SPI
- String formats[] = DocValuesFormat.availableDocValuesFormats().toArray(new String[0]);
- dvFormat = DocValuesFormat.forName(formats[random.nextInt(formats.length)]);
+ List<String> formats = new ArrayList<String>(DocValuesFormat.availableDocValuesFormats());
+ formats.remove("Facet42");
+ dvFormat = DocValuesFormat.forName(formats.get(random.nextInt(formats.size())));
} else {
dvFormat = DocValuesFormat.forName(TEST_DOCVALUESFORMAT);
}
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java Sun Aug 11 12:19:13 2013
@@ -49,7 +49,12 @@ import org.apache.lucene.codecs.lucene42
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType.NumericType;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.AtomicReader;
@@ -78,14 +83,15 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.FilteredQuery.FilterStrategy;
import org.apache.lucene.search.FilteredQuery;
+import org.apache.lucene.search.FilteredQuery.FilterStrategy;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.junit.Assert;
+
import com.carrotsearch.randomizedtesting.RandomizedContext;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@@ -260,17 +266,6 @@ public class _TestUtil {
}
}
- // NOTE: only works for TMP and LMP!!
- public static void setUseCompoundFile(MergePolicy mp, boolean v) {
- if (mp instanceof TieredMergePolicy) {
- ((TieredMergePolicy) mp).setUseCompoundFile(v);
- } else if (mp instanceof LogMergePolicy) {
- ((LogMergePolicy) mp).setUseCompoundFile(v);
- } else {
- throw new IllegalArgumentException("cannot set compound file for MergePolicy " + mp);
- }
- }
-
/** start and end are BOTH inclusive */
public static int nextInt(Random r, int start, int end) {
return RandomInts.randomIntBetween(r, start, end);
@@ -293,7 +288,11 @@ public class _TestUtil {
}
public static String randomSimpleString(Random r, int maxLength) {
- final int end = nextInt(r, 0, maxLength);
+ return randomSimpleString(r, 0, maxLength);
+ }
+
+ public static String randomSimpleString(Random r, int minLength, int maxLength) {
+ final int end = nextInt(r, minLength, maxLength);
if (end == 0) {
// allow 0 length
return "";
@@ -319,7 +318,7 @@ public class _TestUtil {
}
public static String randomSimpleString(Random r) {
- return randomSimpleString(r, 10);
+ return randomSimpleString(r, 0, 10);
}
/** Returns random string, including full unicode range. */
@@ -762,17 +761,17 @@ public class _TestUtil {
if (mp instanceof LogMergePolicy) {
LogMergePolicy lmp = (LogMergePolicy) mp;
lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
- lmp.setUseCompoundFile(true);
+ lmp.setNoCFSRatio(1.0);
} else if (mp instanceof TieredMergePolicy) {
TieredMergePolicy tmp = (TieredMergePolicy) mp;
tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
- tmp.setUseCompoundFile(true);
+ tmp.setNoCFSRatio(1.0);
}
MergeScheduler ms = w.getConfig().getMergeScheduler();
if (ms instanceof ConcurrentMergeScheduler) {
- ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2);
- ((ConcurrentMergeScheduler) ms).setMaxMergeCount(3);
+ // wtf... shouldnt it be even lower since its 1 by default?!?!
+ ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);
}
}
@@ -879,6 +878,7 @@ public class _TestUtil {
final Field field1 = (Field) f;
final Field field2;
final DocValuesType dvType = field1.fieldType().docValueType();
+ final NumericType numType = field1.fieldType().numericType();
if (dvType != null) {
switch(dvType) {
case NUMERIC:
@@ -893,6 +893,23 @@ public class _TestUtil {
default:
throw new IllegalStateException("unknown Type: " + dvType);
}
+ } else if (numType != null) {
+ switch (numType) {
+ case INT:
+ field2 = new IntField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
+ break;
+ case FLOAT:
+ field2 = new FloatField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
+ break;
+ case LONG:
+ field2 = new LongField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
+ break;
+ case DOUBLE:
+ field2 = new DoubleField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
+ break;
+ default:
+ throw new IllegalStateException("unknown Type: " + numType);
+ }
} else {
field2 = new Field(field1.name(), field1.stringValue(), field1.fieldType());
}
@@ -911,7 +928,7 @@ public class _TestUtil {
return null;
}
final TermsEnum termsEnum = terms.iterator(null);
- if (!termsEnum.seekExact(term, random.nextBoolean())) {
+ if (!termsEnum.seekExact(term)) {
return null;
}
return docs(random, termsEnum, liveDocs, reuse, flags);
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Sun Aug 11 12:19:13 2013
@@ -20,6 +20,8 @@ org.apache.lucene.codecs.mocksep.MockSep
org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat
org.apache.lucene.codecs.ramonly.RAMOnlyPostingsFormat
org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds
+org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapFixedInterval
+org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapDocFreqInterval
org.apache.lucene.codecs.bloom.TestBloomFilteredLucene41Postings
org.apache.lucene.codecs.asserting.AssertingPostingsFormat
org.apache.lucene.codecs.lucene40.Lucene40RWPostingsFormat
Modified: lucene/dev/branches/lucene4956/lucene/tools/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/tools/build.xml?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/tools/build.xml (original)
+++ lucene/dev/branches/lucene4956/lucene/tools/build.xml Sun Aug 11 12:19:13 2013
@@ -21,6 +21,8 @@
<description>Lucene Tools</description>
<property name="build.dir" location="../build/tools"/>
+
+ <property name="rat.additional-includes" value="forbiddenApis/**,prettify/**"/>
<import file="../common-build.xml"/>
Modified: lucene/dev/branches/lucene4956/lucene/tools/custom-tasks.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/tools/custom-tasks.xml?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/tools/custom-tasks.xml (original)
+++ lucene/dev/branches/lucene4956/lucene/tools/custom-tasks.xml Sun Aug 11 12:19:13 2013
@@ -1,4 +1,21 @@
-
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
<project name="custom-tasks">
<description>
This file is designed for importing into a main build file, and not intended
@@ -26,7 +43,7 @@
<!-- Typical version patterns. -->
<replaceregex pattern="\.rc[0-9]+" replace="" flags="gi" />
- <replaceregex pattern="\-(r)?([0-9\-\_\.])+(b(eta)?([0-9\-\.])*)?$" replace="" flags="gi" />
+ <replaceregex pattern="\-(r)?([0-9\-\_\.])+((b(eta)?)|((a(lpha)?))([0-9\-\.])*)?(\-tests)?$" replace="" flags="gi" />
<!-- git hashcode pattern: its always 40 chars right? -->
<replaceregex pattern="\-[a-z0-9]{40,40}$" replace="" flags="gi" />
Modified: lucene/dev/branches/lucene4956/lucene/tools/junit4/tests.policy
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/tools/junit4/tests.policy?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/tools/junit4/tests.policy (original)
+++ lucene/dev/branches/lucene4956/lucene/tools/junit4/tests.policy Sun Aug 11 12:19:13 2013
@@ -59,6 +59,10 @@ grant {
permission javax.management.MBeanPermission "*", "*";
permission javax.management.MBeanServerPermission "*";
permission javax.management.MBeanTrustPermission "*";
+ permission javax.security.auth.AuthPermission "*";
+ permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
+ permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
+ permission java.security.SecurityPermission "insertProvider.SaslPlainServer";
// TIKA uses BouncyCastle and that registers new provider for PDF parsing + MSOffice parsing. Maybe report as bug!
permission java.security.SecurityPermission "putProviderProperty.BC";
Modified: lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-apollo.js
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-apollo.js?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-apollo.js (original)
+++ lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-apollo.js Sun Aug 11 12:19:13 2013
@@ -1,2 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
PR.registerLangHandler(PR.createSimpleLexer([["com",/^#[^\n\r]*/,null,"#"],["pln",/^[\t\n\r \xa0]+/,null,"\t\n\r Â\xa0"],["str",/^"(?:[^"\\]|\\[\S\s])*(?:"|$)/,null,'"']],[["kwd",/^(?:ADS|AD|AUG|BZF|BZMF|CAE|CAF|CA|CCS|COM|CS|DAS|DCA|DCOM|DCS|DDOUBL|DIM|DOUBLE|DTCB|DTCF|DV|DXCH|EDRUPT|EXTEND|INCR|INDEX|NDX|INHINT|LXCH|MASK|MSK|MP|MSU|NOOP|OVSK|QXCH|RAND|READ|RELINT|RESUME|RETURN|ROR|RXOR|SQUARE|SU|TCR|TCAA|OVSK|TCF|TC|TS|WAND|WOR|WRITE|XCH|XLQ|XXALQ|ZL|ZQ|ADD|ADZ|SUB|SUZ|MPY|MPR|MPZ|DVP|COM|ABS|CLA|CLZ|LDQ|STO|STQ|ALS|LLS|LRS|TRA|TSQ|TMI|TOV|AXT|TIX|DLY|INP|OUT)\s/,
null],["typ",/^(?:-?GENADR|=MINUS|2BCADR|VN|BOF|MM|-?2CADR|-?[1-6]DNADR|ADRES|BBCON|[ES]?BANK=?|BLOCK|BNKSUM|E?CADR|COUNT\*?|2?DEC\*?|-?DNCHAN|-?DNPTR|EQUALS|ERASE|MEMORY|2?OCT|REMADR|SETLOC|SUBRO|ORG|BSS|BES|SYN|EQU|DEFINE|END)\s/,null],["lit",/^'(?:-*(?:\w|\\[!-~])(?:[\w-]*|\\[!-~])[!=?]?)?/],["pln",/^-*(?:[!-z]|\\[!-~])(?:[\w-]*|\\[!-~])[!=?]?/],["pun",/^[^\w\t\n\r "'-);\\\xa0]+/]]),["apollo","agc","aea"]);
Modified: lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-css.js
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-css.js?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-css.js (original)
+++ lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-css.js Sun Aug 11 12:19:13 2013
@@ -1,2 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t\n\f\r ]+/,null," \t\r\n "]],[["str",/^"(?:[^\n\f\r"\\]|\\(?:\r\n?|\n|\f)|\\[\S\s])*"/,null],["str",/^'(?:[^\n\f\r'\\]|\\(?:\r\n?|\n|\f)|\\[\S\s])*'/,null],["lang-css-str",/^url\(([^"')]*)\)/i],["kwd",/^(?:url|rgb|!important|@import|@page|@media|@charset|inherit)(?=[^\w-]|$)/i,null],["lang-css-kw",/^(-?(?:[_a-z]|\\[\da-f]+ ?)(?:[\w-]|\\\\[\da-f]+ ?)*)\s*:/i],["com",/^\/\*[^*]*\*+(?:[^*/][^*]*\*+)*\//],["com",
/^(?:<\!--|--\>)/],["lit",/^(?:\d+|\d*\.\d+)(?:%|[a-z]+)?/i],["lit",/^#[\da-f]{3,6}/i],["pln",/^-?(?:[_a-z]|\\[\da-f]+ ?)(?:[\w-]|\\\\[\da-f]+ ?)*/i],["pun",/^[^\s\w"']+/]]),["css"]);PR.registerLangHandler(PR.createSimpleLexer([],[["kwd",/^-?(?:[_a-z]|\\[\da-f]+ ?)(?:[\w-]|\\\\[\da-f]+ ?)*/i]]),["css-kw"]);PR.registerLangHandler(PR.createSimpleLexer([],[["str",/^[^"')]+/]]),["css-str"]);
Modified: lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-hs.js
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-hs.js?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-hs.js (original)
+++ lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-hs.js Sun Aug 11 12:19:13 2013
@@ -1,2 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t-\r ]+/,null,"\t\n \r "],["str",/^"(?:[^\n\f\r"\\]|\\[\S\s])*(?:"|$)/,null,'"'],["str",/^'(?:[^\n\f\r'\\]|\\[^&])'?/,null,"'"],["lit",/^(?:0o[0-7]+|0x[\da-f]+|\d+(?:\.\d+)?(?:e[+-]?\d+)?)/i,null,"0123456789"]],[["com",/^(?:--+[^\n\f\r]*|{-(?:[^-]|-+[^}-])*-})/],["kwd",/^(?:case|class|data|default|deriving|do|else|if|import|in|infix|infixl|infixr|instance|let|module|newtype|of|then|type|where|_)(?=[^\d'A-Za-z]|$)/,
null],["pln",/^(?:[A-Z][\w']*\.)*[A-Za-z][\w']*/],["pun",/^[^\d\t-\r "'A-Za-z]+/]]),["hs"]);
Modified: lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-lisp.js
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-lisp.js?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-lisp.js (original)
+++ lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-lisp.js Sun Aug 11 12:19:13 2013
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
var a=null;
PR.registerLangHandler(PR.createSimpleLexer([["opn",/^\(+/,a,"("],["clo",/^\)+/,a,")"],["com",/^;[^\n\r]*/,a,";"],["pln",/^[\t\n\r \xa0]+/,a,"\t\n\r \xa0"],["str",/^"(?:[^"\\]|\\[\S\s])*(?:"|$)/,a,'"']],[["kwd",/^(?:block|c[ad]+r|catch|con[ds]|def(?:ine|un)|do|eq|eql|equal|equalp|eval-when|flet|format|go|if|labels|lambda|let|load-time-value|locally|macrolet|multiple-value-call|nil|progn|progv|quote|require|return-from|setq|symbol-macrolet|t|tagbody|the|throw|unwind)\b/,a],
["lit",/^[+-]?(?:[#0]x[\da-f]+|\d+\/\d+|(?:\.\d+|\d+(?:\.\d*)?)(?:[de][+-]?\d+)?)/i],["lit",/^'(?:-*(?:\w|\\[!-~])(?:[\w-]*|\\[!-~])[!=?]?)?/],["pln",/^-*(?:[_a-z]|\\[!-~])(?:[\w-]*|\\[!-~])[!=?]?/i],["pun",/^[^\w\t\n\r "'-);\\\xa0]+/]]),["cl","el","lisp","scm"]);
Modified: lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-lua.js
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-lua.js?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-lua.js (original)
+++ lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-lua.js Sun Aug 11 12:19:13 2013
@@ -1,2 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t\n\r \xa0]+/,null,"\t\n\r Â\xa0"],["str",/^(?:"(?:[^"\\]|\\[\S\s])*(?:"|$)|'(?:[^'\\]|\\[\S\s])*(?:'|$))/,null,"\"'"]],[["com",/^--(?:\[(=*)\[[\S\s]*?(?:]\1]|$)|[^\n\r]*)/],["str",/^\[(=*)\[[\S\s]*?(?:]\1]|$)/],["kwd",/^(?:and|break|do|else|elseif|end|false|for|function|if|in|local|nil|not|or|repeat|return|then|true|until|while)\b/,null],["lit",/^[+-]?(?:0x[\da-f]+|(?:\.\d+|\d+(?:\.\d*)?)(?:e[+-]?\d+)?)/i],
["pln",/^[_a-z]\w*/i],["pun",/^[^\w\t\n\r \xa0][^\w\t\n\r "'+=\xa0-]*/]]),["lua"]);
Modified: lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-ml.js
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-ml.js?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-ml.js (original)
+++ lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-ml.js Sun Aug 11 12:19:13 2013
@@ -1,2 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t\n\r \xa0]+/,null,"\t\n\r Â\xa0"],["com",/^#(?:if[\t\n\r \xa0]+(?:[$_a-z][\w']*|``[^\t\n\r`]*(?:``|$))|else|endif|light)/i,null,"#"],["str",/^(?:"(?:[^"\\]|\\[\S\s])*(?:"|$)|'(?:[^'\\]|\\[\S\s])(?:'|$))/,null,"\"'"]],[["com",/^(?:\/\/[^\n\r]*|\(\*[\S\s]*?\*\))/],["kwd",/^(?:abstract|and|as|assert|begin|class|default|delegate|do|done|downcast|downto|elif|else|end|exception|extern|false|finally|for|fun|function|if|in|inherit|inline|interface|internal|lazy|let|match|member|module|mutable|namespace|new|null|of|open|or|override|private|public|rec|return|static|struct|then|to|true|try|type|upcast|use|val|void|when|while|with|yield|asr|land|lor|lsl|lsr|lxor|mod|sig|atomic|break|checked|component|const|constraint|constructor|continue|eager|event|external|fixed|functor|global|include|method|mixin|object|parallel|process|protected|pure|sealed|trait|virtual|volatile)\b/],
["lit",/^[+-]?(?:0x[\da-f]+|(?:\.\d+|\d+(?:\.\d*)?)(?:e[+-]?\d+)?)/i],["pln",/^(?:[_a-z][\w']*[!#?]?|``[^\t\n\r`]*(?:``|$))/i],["pun",/^[^\w\t\n\r "'\xa0]+/]]),["fs","ml"]);
Modified: lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-proto.js
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-proto.js?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-proto.js (original)
+++ lucene/dev/branches/lucene4956/lucene/tools/prettify/lang-proto.js Sun Aug 11 12:19:13 2013
@@ -1 +1,17 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
PR.registerLangHandler(PR.sourceDecorator({keywords:"bytes,default,double,enum,extend,extensions,false,group,import,max,message,option,optional,package,repeated,required,returns,rpc,service,syntax,to,true",types:/^(bool|(double|s?fixed|[su]?int)(32|64)|float|string)\b/,cStyleComments:!0}),["proto"]);