You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/02/09 00:53:25 UTC
svn commit: r1658277 [13/38] - in /lucene/dev/branches/lucene6005: ./
dev-tools/ dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/contrib/velocity/ dev-tools/maven/lucene/replicator/
dev-tools/maven/solr/ dev-tools/maven/solr/contrib/...
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java Sun Feb 8 23:53:14 2015
@@ -280,20 +280,8 @@ public class AssertingLeafReader extends
private int doc;
public AssertingDocsEnum(DocsEnum in) {
- this(in, true);
- }
-
- public AssertingDocsEnum(DocsEnum in, boolean failOnUnsupportedDocID) {
super(in);
- try {
- int docid = in.docID();
- assert docid == -1 : in.getClass() + ": invalid initial doc id: " + docid;
- } catch (UnsupportedOperationException e) {
- if (failOnUnsupportedDocID) {
- throw e;
- }
- }
- doc = -1;
+ this.doc = in.docID();
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java Sun Feb 8 23:53:14 2015
@@ -53,7 +53,8 @@ public abstract class BaseCompoundFormat
Directory dir = newDirectory();
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Collections.<String>emptyList(), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Collections.emptySet());
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
assertEquals(0, cfs.listAll().length);
cfs.close();
@@ -72,7 +73,8 @@ public abstract class BaseCompoundFormat
createSequenceFile(dir, testfile, (byte) 0, data[i]);
SegmentInfo si = newSegmentInfo(dir, "_" + i);
- si.getCodec().compoundFormat().write(dir, si, Collections.singleton(testfile), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Collections.singleton(testfile));
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
IndexInput expected = dir.openInput(testfile, newIOContext(random()));
@@ -96,7 +98,8 @@ public abstract class BaseCompoundFormat
createSequenceFile(dir, files[1], (byte) 0, 114);
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Arrays.asList(files), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Arrays.asList(files));
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
for (String file : files) {
@@ -122,7 +125,8 @@ public abstract class BaseCompoundFormat
out.close();
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Collections.singleton(testfile), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Collections.singleton(testfile));
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
assertEquals(1, cfs.listAll().length);
cfs.close();
@@ -147,7 +151,8 @@ public abstract class BaseCompoundFormat
out.close();
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Collections.singleton(testfile), MergeState.CheckAbort.NONE, myContext);
+ si.setFiles(Collections.singleton(testfile));
+ si.getCodec().compoundFormat().write(dir, si, myContext);
dir.close();
}
@@ -166,7 +171,8 @@ public abstract class BaseCompoundFormat
out.close();
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Collections.singleton(testfile), MergeState.CheckAbort.NONE, context);
+ si.setFiles(Collections.singleton(testfile));
+ si.getCodec().compoundFormat().write(dir, si, context);
dir.close();
}
@@ -211,7 +217,8 @@ public abstract class BaseCompoundFormat
Directory dir = newDirectory();
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Collections.<String>emptyList(), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Collections.emptyList());
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
try {
cfs.createOutput("bogus", IOContext.DEFAULT);
@@ -233,7 +240,8 @@ public abstract class BaseCompoundFormat
out.close();
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Collections.<String>emptyList(), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Collections.emptyList());
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
try {
cfs.deleteFile(testfile);
@@ -255,7 +263,8 @@ public abstract class BaseCompoundFormat
out.close();
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Collections.<String>emptyList(), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Collections.emptyList());
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
try {
cfs.renameFile(testfile, "bogus");
@@ -277,7 +286,8 @@ public abstract class BaseCompoundFormat
out.close();
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Collections.<String>emptyList(), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Collections.emptyList());
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
try {
cfs.sync(Collections.singleton(testfile));
@@ -299,7 +309,8 @@ public abstract class BaseCompoundFormat
out.close();
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Collections.<String>emptyList(), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Collections.emptyList());
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
try {
cfs.makeLock("foobar");
@@ -338,7 +349,8 @@ public abstract class BaseCompoundFormat
String files[] = dir.listAll();
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Arrays.asList(files), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Arrays.asList(files));
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
for (int i = 0; i < files.length; i++) {
@@ -369,7 +381,8 @@ public abstract class BaseCompoundFormat
assertEquals(0, dir.getFileHandleCount());
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, Arrays.asList(dir.listAll()), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(Arrays.asList(dir.listAll()));
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
final IndexInput[] ins = new IndexInput[FILE_COUNT];
@@ -722,7 +735,8 @@ public abstract class BaseCompoundFormat
}
SegmentInfo si = newSegmentInfo(dir, "_123");
- si.getCodec().compoundFormat().write(dir, si, files, MergeState.CheckAbort.NONE, IOContext.DEFAULT);
+ si.setFiles(files);
+ si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
return cfs;
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Sun Feb 8 23:53:14 2015
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map.Entry;
import java.util.Map;
import java.util.Set;
@@ -46,6 +47,7 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
@@ -3004,6 +3006,151 @@ public abstract class BaseDocValuesForma
directory.close();
}
+ public void testSortedEnumAdvanceIndependently() throws IOException {
+ Directory directory = newDirectory();
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
+ iwconfig.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);
+
+ Document doc = iwriter.newDocument();
+ doc.addAtom("field", new BytesRef("2"));
+ iwriter.addDocument(doc);
+
+ doc = iwriter.newDocument();
+ doc.addAtom("field", new BytesRef("1"));
+ iwriter.addDocument(doc);
+
+ doc = iwriter.newDocument();
+ doc.addAtom("field", new BytesRef("3"));
+ iwriter.addDocument(doc);
+
+ iwriter.commit();
+ iwriter.forceMerge(1);
+
+ DirectoryReader ireader = iwriter.getReader();
+ iwriter.close();
+
+ SortedDocValues dv = getOnlySegmentReader(ireader).getSortedDocValues("field");
+ doTestSortedSetEnumAdvanceIndependently(DocValues.singleton(dv));
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testSortedSetEnumAdvanceIndependently() throws IOException {
+ Directory directory = newDirectory();
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
+ iwconfig.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);
+ FieldTypes fieldTypes = iwriter.getFieldTypes();
+ fieldTypes.setMultiValued("field");
+
+ Document doc = iwriter.newDocument();
+ doc.addAtom("field", new BytesRef("2"));
+ doc.addAtom("field", new BytesRef("3"));
+ iwriter.addDocument(doc);
+
+ doc = iwriter.newDocument();
+ doc.addAtom("field", new BytesRef("1"));
+ doc.addAtom("field", new BytesRef("3"));
+ iwriter.addDocument(doc);
+
+ doc = iwriter.newDocument();
+ doc.addAtom("field", new BytesRef("1"));
+ doc.addAtom("field", new BytesRef("2"));
+ iwriter.addDocument(doc);
+
+ iwriter.commit();
+ iwriter.forceMerge(1);
+
+ DirectoryReader ireader = iwriter.getReader();
+ iwriter.close();
+
+ SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field");
+ doTestSortedSetEnumAdvanceIndependently(dv);
+
+ ireader.close();
+ directory.close();
+ }
+
+ protected void doTestSortedSetEnumAdvanceIndependently(SortedSetDocValues dv) throws IOException {
+ if (dv.getValueCount() < 2) {
+ return;
+ }
+ List<BytesRef> terms = new ArrayList<>();
+ TermsEnum te = dv.termsEnum();
+ terms.add(BytesRef.deepCopyOf(te.next()));
+ terms.add(BytesRef.deepCopyOf(te.next()));
+
+ // Make sure that calls to next() does not modify the term of the other enum
+ TermsEnum enum1 = dv.termsEnum();
+ TermsEnum enum2 = dv.termsEnum();
+ BytesRefBuilder term1 = new BytesRefBuilder();
+ BytesRefBuilder term2 = new BytesRefBuilder();
+
+ term1.copyBytes(enum1.next());
+ term2.copyBytes(enum2.next());
+ term1.copyBytes(enum1.next());
+
+ assertEquals(term1.get(), enum1.term());
+ assertEquals(term2.get(), enum2.term());
+
+ // Same for seekCeil
+ enum1 = dv.termsEnum();
+ enum2 = dv.termsEnum();
+ term1 = new BytesRefBuilder();
+ term2 = new BytesRefBuilder();
+
+ term2.copyBytes(enum2.next());
+ BytesRefBuilder seekTerm = new BytesRefBuilder();
+ seekTerm.append(terms.get(0));
+ seekTerm.append((byte) 0);
+ enum1.seekCeil(seekTerm.get());
+ term1.copyBytes(enum1.term());
+
+ assertEquals(term1.get(), enum1.term());
+ assertEquals(term2.get(), enum2.term());
+
+ // Same for seekCeil on an exact value
+ enum1 = dv.termsEnum();
+ enum2 = dv.termsEnum();
+ term1 = new BytesRefBuilder();
+ term2 = new BytesRefBuilder();
+
+ term2.copyBytes(enum2.next());
+ enum1.seekCeil(terms.get(1));
+ term1.copyBytes(enum1.term());
+
+ assertEquals(term1.get(), enum1.term());
+ assertEquals(term2.get(), enum2.term());
+
+ // Same for seekExact
+ enum1 = dv.termsEnum();
+ enum2 = dv.termsEnum();
+ term1 = new BytesRefBuilder();
+ term2 = new BytesRefBuilder();
+
+ term2.copyBytes(enum2.next());
+ final boolean found = enum1.seekExact(terms.get(1));
+ assertTrue(found);
+ term1.copyBytes(enum1.term());
+
+ // Same for seek by ord
+ enum1 = dv.termsEnum();
+ enum2 = dv.termsEnum();
+ term1 = new BytesRefBuilder();
+ term2 = new BytesRefBuilder();
+
+ term2.copyBytes(enum2.next());
+ enum1.seekExact(1);
+ term1.copyBytes(enum1.term());
+
+ assertEquals(term1.get(), enum1.term());
+ assertEquals(term2.get(), enum2.term());
+ }
+
protected boolean codecAcceptsHugeBinaryValues(String field) {
return true;
}
@@ -3023,4 +3170,5 @@ public abstract class BaseDocValuesForma
protected boolean codecSupportsSortedNumeric() {
return true;
}
+
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java Sun Feb 8 23:53:14 2015
@@ -207,7 +207,7 @@ abstract class BaseIndexFileFormatTestCa
w.forceMerge(1);
w.commit();
w.close();
- IndexReader reader = DirectoryReader.open(dir);
+ DirectoryReader reader = DirectoryReader.open(dir);
Directory dir2 = newDirectory();
if (dir2 instanceof MockDirectoryWrapper) {
@@ -221,7 +221,8 @@ abstract class BaseIndexFileFormatTestCa
w = new IndexWriter(dir2, cfg);
fieldTypes = w.getFieldTypes();
fieldTypes.disableExistsFilters();
- w.addIndexes(reader);
+ TestUtil.addIndexesSlowly(w, reader);
+
w.commit();
w.close();
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java Sun Feb 8 23:53:14 2015
@@ -551,7 +551,7 @@ public abstract class BaseStoredFieldsFo
Directory dir2 = newDirectory();
w = new RandomIndexWriter(random(), dir2);
- w.addIndexes(reader);
+ TestUtil.addIndexesSlowly(w.w, reader);
reader.close();
dir.close();
@@ -679,7 +679,7 @@ public abstract class BaseStoredFieldsFo
}
dirs[i] = newDirectory();
IndexWriter adder = new IndexWriter(dirs[i], new IndexWriterConfig(null));
- adder.addIndexes(reader);
+ TestUtil.addIndexesSlowly(adder, reader);
adder.commit();
adder.close();
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java Sun Feb 8 23:53:14 2015
@@ -503,9 +503,7 @@ public abstract class BaseTermVectorsFor
try {
docsAndPositionsEnum.nextPosition();
fail();
- } catch (Exception e) {
- // ok
- } catch (AssertionError e) {
+ } catch (Exception | AssertionError e) {
// ok
}
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java Sun Feb 8 23:53:14 2015
@@ -138,7 +138,7 @@ public class MockRandomMergePolicy exten
static class MockRandomOneMerge extends OneMerge {
final Random r;
- ArrayList<LeafReader> readers;
+ ArrayList<CodecReader> readers;
MockRandomOneMerge(List<SegmentCommitInfo> segments, long seed) {
super(segments);
@@ -146,21 +146,23 @@ public class MockRandomMergePolicy exten
}
@Override
- public List<LeafReader> getMergeReaders() throws IOException {
+ public List<CodecReader> getMergeReaders() throws IOException {
if (readers == null) {
- readers = new ArrayList<LeafReader>(super.getMergeReaders());
+ readers = new ArrayList<CodecReader>(super.getMergeReaders());
for (int i = 0; i < readers.size(); i++) {
// wrap it (e.g. prevent bulk merge etc)
+ // TODO: cut this over to FilterCodecReader api, we can explicitly
+ // enable/disable bulk merge for portions of the index we want.
int thingToDo = r.nextInt(7);
if (thingToDo == 0) {
// simple no-op FilterReader
- readers.set(i, new FilterLeafReader(readers.get(i)));
+ readers.set(i, SlowCodecReaderWrapper.wrap(new FilterLeafReader(readers.get(i))));
} else if (thingToDo == 1) {
// renumber fields
// NOTE: currently this only "blocks" bulk merges just by
// being a FilterReader. But it might find bugs elsewhere,
// and maybe the situation can be improved in the future.
- readers.set(i, new MismatchedLeafReader(readers.get(i), r));
+ readers.set(i, SlowCodecReaderWrapper.wrap(new MismatchedLeafReader(readers.get(i), r)));
}
// otherwise, reader is unchanged
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Sun Feb 8 23:53:14 2015
@@ -227,7 +227,7 @@ public class RandomIndexWriter implement
w.addIndexes(dirs);
}
- public void addIndexes(IndexReader... readers) throws IOException {
+ public void addIndexes(CodecReader... readers) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.addIndexes(readers);
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/SuppressingConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/SuppressingConcurrentMergeScheduler.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/SuppressingConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/SuppressingConcurrentMergeScheduler.java Sun Feb 8 23:53:14 2015
@@ -1,5 +1,7 @@
package org.apache.lucene.index;
+import org.apache.lucene.store.Directory;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -17,22 +19,17 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.io.IOException;
-
-import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.util.IOUtils;
-
/** A {@link ConcurrentMergeScheduler} that ignores AlreadyClosedException. */
public abstract class SuppressingConcurrentMergeScheduler extends ConcurrentMergeScheduler {
@Override
- protected void handleMergeException(Throwable exc) {
+ protected void handleMergeException(Directory dir, Throwable exc) {
while (true) {
if (isOK(exc)) {
return;
}
exc = exc.getCause();
if (exc == null) {
- super.handleMergeException(exc);
+ super.handleMergeException(dir, exc);
}
}
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java Sun Feb 8 23:53:14 2015
@@ -442,6 +442,11 @@ public abstract class ThreadedIndexingAn
} else if (mp instanceof LogMergePolicy) {
((LogMergePolicy) mp).setMaxMergeDocs(100000);
}
+ // when running nightly, merging can still have crazy parameters,
+ // and might use many per-field codecs. turn on CFS for IW flushes
+ // and ensure CFS ratio is reasonable to keep it contained.
+ conf.setUseCompoundFile(true);
+ mp.setNoCFSRatio(Math.max(0.25d, mp.getNoCFSRatio()));
}
conf.setMergedSegmentWarmer(new IndexWriter.IndexReaderWarmer() {
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingBulkScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingBulkScorer.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingBulkScorer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingBulkScorer.java Sun Feb 8 23:53:14 2015
@@ -18,39 +18,31 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
-import java.lang.ref.WeakReference;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
import java.util.Random;
-import java.util.WeakHashMap;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.util.VirtualMethod;
-/** Wraps a Scorer with additional checks */
-public class AssertingBulkScorer extends BulkScorer {
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
- private static final VirtualMethod<BulkScorer> SCORE_COLLECTOR = new VirtualMethod<>(BulkScorer.class, "score", LeafCollector.class);
- private static final VirtualMethod<BulkScorer> SCORE_COLLECTOR_RANGE = new VirtualMethod<>(BulkScorer.class, "score", LeafCollector.class, int.class);
+/** Wraps a Scorer with additional checks */
+final class AssertingBulkScorer extends BulkScorer {
- public static BulkScorer wrap(Random random, BulkScorer other) {
+ public static BulkScorer wrap(Random random, BulkScorer other, int maxDoc) {
if (other == null || other instanceof AssertingBulkScorer) {
return other;
}
- return new AssertingBulkScorer(random, other);
- }
-
- public static boolean shouldWrap(BulkScorer inScorer) {
- return SCORE_COLLECTOR.isOverriddenAsOf(inScorer.getClass()) || SCORE_COLLECTOR_RANGE.isOverriddenAsOf(inScorer.getClass());
+ return new AssertingBulkScorer(random, other, maxDoc);
}
final Random random;
final BulkScorer in;
+ final int maxDoc;
+ int max = 0;
- private AssertingBulkScorer(Random random, BulkScorer in) {
+ private AssertingBulkScorer(Random random, BulkScorer in, int maxDoc) {
this.random = random;
this.in = in;
+ this.maxDoc = maxDoc;
}
public BulkScorer getIn() {
@@ -59,10 +51,12 @@ public class AssertingBulkScorer extends
@Override
public void score(LeafCollector collector) throws IOException {
+ assert max == 0;
+ collector = new AssertingLeafCollector(random, collector, 0, DocsEnum.NO_MORE_DOCS);
if (random.nextBoolean()) {
try {
- final boolean remaining = in.score(collector, DocsEnum.NO_MORE_DOCS);
- assert !remaining;
+ final int next = score(collector, 0, DocsEnum.NO_MORE_DOCS);
+ assert next == DocIdSetIterator.NO_MORE_DOCS;
} catch (UnsupportedOperationException e) {
in.score(collector);
}
@@ -72,8 +66,19 @@ public class AssertingBulkScorer extends
}
@Override
- public boolean score(LeafCollector collector, int max) throws IOException {
- return in.score(collector, max);
+ public int score(LeafCollector collector, int min, final int max) throws IOException {
+ assert min >= this.max: "Scoring backward: min=" + min + " while previous max was max=" + this.max;
+ assert min < max : "max must be greater than min, got min=" + min + ", and max=" + max;
+ this.max = max;
+ collector = new AssertingLeafCollector(random, collector, min, max);
+ final int next = in.score(collector, min, max);
+ assert next >= max;
+ if (max >= maxDoc || next >= maxDoc) {
+ assert next == DocIdSetIterator.NO_MORE_DOCS;
+ return DocIdSetIterator.NO_MORE_DOCS;
+ } else {
+ return RandomInts.randomIntBetween(random, max, next);
+ }
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java Sun Feb 8 23:53:14 2015
@@ -91,7 +91,7 @@ public class AssertingIndexSearcher exte
@Override
protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
// TODO: shouldn't we AssertingCollector.wrap(collector) here?
- super.search(leaves, AssertingWeight.wrap(random, weight), collector);
+ super.search(leaves, AssertingWeight.wrap(random, weight), AssertingCollector.wrap(random, collector));
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java Sun Feb 8 23:53:14 2015
@@ -29,14 +29,12 @@ class AssertingWeight extends Weight {
return other instanceof AssertingWeight ? other : new AssertingWeight(random, other);
}
- final boolean scoresDocsOutOfOrder;
final Random random;
final Weight in;
AssertingWeight(Random random, Weight in) {
this.random = random;
this.in = in;
- scoresDocsOutOfOrder = in.scoresDocsOutOfOrder() || random.nextBoolean();
}
@Override
@@ -61,43 +59,18 @@ class AssertingWeight extends Weight {
@Override
public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException {
- // if the caller asks for in-order scoring or if the weight does not support
- // out-of order scoring then collection will have to happen in-order.
final Scorer inScorer = in.scorer(context, acceptDocs);
+ assert inScorer == null || inScorer.docID() == -1;
return AssertingScorer.wrap(new Random(random.nextLong()), inScorer);
}
@Override
- public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, Bits acceptDocs) throws IOException {
- // if the caller asks for in-order scoring or if the weight does not support
- // out-of order scoring then collection will have to happen in-order.
- BulkScorer inScorer = in.bulkScorer(context, scoreDocsInOrder, acceptDocs);
+ public BulkScorer bulkScorer(LeafReaderContext context, Bits acceptDocs) throws IOException {
+ BulkScorer inScorer = in.bulkScorer(context, acceptDocs);
if (inScorer == null) {
return null;
}
- if (AssertingBulkScorer.shouldWrap(inScorer)) {
- // The incoming scorer already has a specialized
- // implementation for BulkScorer, so we should use it:
- inScorer = AssertingBulkScorer.wrap(new Random(random.nextLong()), inScorer);
- } else if (random.nextBoolean()) {
- // Let super wrap this.scorer instead, so we use
- // AssertingScorer:
- inScorer = super.bulkScorer(context, scoreDocsInOrder, acceptDocs);
- }
-
- if (scoreDocsInOrder == false && random.nextBoolean()) {
- // The caller claims it can handle out-of-order
- // docs; let's confirm that by pulling docs and
- // randomly shuffling them before collection:
- inScorer = new AssertingBulkOutOfOrderScorer(new Random(random.nextLong()), inScorer);
- }
- return inScorer;
- }
-
- @Override
- public boolean scoresDocsOutOfOrder() {
- return scoresDocsOutOfOrder;
+ return AssertingBulkScorer.wrap(new Random(random.nextLong()), inScorer, context.reader().maxDoc());
}
}
-
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java Sun Feb 8 23:53:14 2015
@@ -139,10 +139,6 @@ public class CheckHits {
protected void doSetNextReader(LeafReaderContext context) throws IOException {
base = context.docBase;
}
- @Override
- public boolean acceptsDocsOutOfOrder() {
- return true;
- }
}
/**
@@ -511,10 +507,6 @@ public class CheckHits {
protected void doSetNextReader(LeafReaderContext context) throws IOException {
base = context.docBase;
}
- @Override
- public boolean acceptsDocsOutOfOrder() {
- return true;
- }
}
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java Sun Feb 8 23:53:14 2015
@@ -110,6 +110,7 @@ public class QueryUtils {
if (s!=null) {
checkFirstSkipTo(q1,s);
checkSkipTo(q1,s);
+ checkBulkScorerSkipTo(random, q1, s);
if (wrap) {
check(random, q1, wrapUnderlyingReader(random, s, -1), false);
check(random, q1, wrapUnderlyingReader(random, s, 0), false);
@@ -218,7 +219,6 @@ public class QueryUtils {
public static void checkSkipTo(final Query q, final IndexSearcher s) throws IOException {
//System.out.println("Checking "+q);
final List<LeafReaderContext> readerContextArray = s.getTopReaderContext().leaves();
- if (s.createNormalizedWeight(q).scoresDocsOutOfOrder()) return; // in this case order of skipTo() might differ from that of next().
final int skip_op = 0;
final int next_op = 1;
@@ -322,11 +322,6 @@ public class QueryUtils {
this.scorer = null;
lastDoc[0] = -1;
}
-
- @Override
- public boolean acceptsDocsOutOfOrder() {
- return false;
- }
});
if (lastReader[0] != null) {
@@ -408,10 +403,6 @@ public class QueryUtils {
lastDoc[0] = -1;
liveDocs = context.reader().getLiveDocs();
}
- @Override
- public boolean acceptsDocsOutOfOrder() {
- return false;
- }
});
if (lastReader[0] != null) {
@@ -428,4 +419,56 @@ public class QueryUtils {
}
}
}
+
+ /** Check that the scorer and bulk scorer advance consistently. */
+ public static void checkBulkScorerSkipTo(Random r, Query query, IndexSearcher searcher) throws IOException {
+ Weight weight = searcher.createNormalizedWeight(query);
+ for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
+ final Scorer scorer = weight.scorer(context, context.reader().getLiveDocs());
+ final BulkScorer bulkScorer = weight.bulkScorer(context, context.reader().getLiveDocs());
+ if (scorer == null && bulkScorer == null) {
+ continue;
+ }
+ int upTo = 0;
+ while (true) {
+ final int min = upTo + r.nextInt(5);
+ final int max = min + 1 + r.nextInt(r.nextBoolean() ? 10 : 5000);
+ if (scorer.docID() < min) {
+ scorer.advance(min);
+ }
+ final int next = bulkScorer.score(new LeafCollector() {
+ Scorer scorer2;
+ @Override
+ public void setScorer(Scorer scorer) throws IOException {
+ this.scorer2 = scorer;
+ }
+ @Override
+ public void collect(int doc) throws IOException {
+ assert doc >= min;
+ assert doc < max;
+ Assert.assertEquals(scorer.docID(), doc);
+ Assert.assertEquals(scorer.score(), scorer2.score(), 0.01f);
+ scorer.nextDoc();
+ }
+ }, min, max);
+ assert max <= next;
+ assert next <= scorer.docID();
+ upTo = max;
+
+ if (scorer.docID() == DocIdSetIterator.NO_MORE_DOCS) {
+ bulkScorer.score(new LeafCollector() {
+ @Override
+ public void setScorer(Scorer scorer) throws IOException {}
+
+ @Override
+ public void collect(int doc) throws IOException {
+ // no more matches
+ assert false;
+ }
+ }, upTo, DocIdSetIterator.NO_MORE_DOCS);
+ break;
+ }
+ }
+ }
+ }
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Sun Feb 8 23:53:14 2015
@@ -611,7 +611,7 @@ public class MockDirectoryWrapper extend
// throttling REALLY slows down tests, so don't do it very often for SOMETIMES.
if (throttling == Throttling.ALWAYS ||
- (throttling == Throttling.SOMETIMES && randomState.nextInt(200) == 0) && !(in instanceof RateLimitedDirectoryWrapper)) {
+ (throttling == Throttling.SOMETIMES && randomState.nextInt(200) == 0)) {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")");
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Sun Feb 8 23:53:14 2015
@@ -34,6 +34,7 @@ import java.lang.reflect.Method;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.text.Collator;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -72,8 +73,8 @@ import org.apache.lucene.index.FieldFilt
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.ReaderClosedListener;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
@@ -85,6 +86,7 @@ import org.apache.lucene.index.LogDocMer
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.index.MergeTrigger;
import org.apache.lucene.index.MismatchedDirectoryReader;
import org.apache.lucene.index.MismatchedLeafReader;
import org.apache.lucene.index.MockRandomMergePolicy;
@@ -102,8 +104,8 @@ import org.apache.lucene.index.SortedDoc
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.AssertingIndexSearcher;
import org.apache.lucene.search.DocIdSet;
@@ -120,13 +122,11 @@ import org.apache.lucene.store.FSDirecto
import org.apache.lucene.store.FSLockFactory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IOContext.Context;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.MergeInfo;
-import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper.Throttling;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NRTCachingDirectory;
-import org.apache.lucene.store.RateLimitedDirectoryWrapper;
import org.apache.lucene.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.RegExp;
@@ -151,16 +151,16 @@ import com.carrotsearch.randomizedtestin
import com.carrotsearch.randomizedtesting.annotations.SeedDecorators;
import com.carrotsearch.randomizedtesting.annotations.TestGroup;
import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction.Action;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup.Group;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies.Consequence;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import com.carrotsearch.randomizedtesting.rules.NoClassHooksShadowingRule;
@@ -691,10 +691,8 @@ public abstract class LuceneTestCase ext
m.setAccessible(true);
try {
m.invoke(IndexWriter.class, limit);
- } catch (IllegalAccessException iae) {
+ } catch (IllegalAccessException | InvocationTargetException iae) {
throw new RuntimeException(iae);
- } catch (InvocationTargetException ite) {
- throw new RuntimeException(ite);
}
}
@@ -944,13 +942,18 @@ public abstract class LuceneTestCase ext
} else {
cms = new ConcurrentMergeScheduler() {
@Override
- protected synchronized void maybeStall() {
+ protected synchronized boolean maybeStall(IndexWriter writer) {
+ return true;
}
};
}
int maxThreadCount = TestUtil.nextInt(r, 1, 4);
int maxMergeCount = TestUtil.nextInt(r, maxThreadCount, maxThreadCount + 4);
cms.setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
+ if (random().nextBoolean()) {
+ cms.disableAutoIOThrottle();
+ }
+ cms.setForceMergeMBPerSec(10 + 10*random().nextDouble());
c.setMergeScheduler(cms);
} else {
// Always use consistent settings, else CMS's dynamic (SSD or not)
@@ -1377,27 +1380,6 @@ public abstract class LuceneTestCase ext
directory = new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble());
}
- if (TEST_NIGHTLY && rarely(random) && !bare) {
- final double maxMBPerSec = TestUtil.nextInt(random, 20, 40);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("LuceneTestCase: will rate limit output IndexOutput to " + maxMBPerSec + " MB/sec");
- }
- final RateLimitedDirectoryWrapper rateLimitedDirectoryWrapper = new RateLimitedDirectoryWrapper(directory);
- switch (random.nextInt(10)) {
- case 3: // sometimes rate limit on flush
- rateLimitedDirectoryWrapper.setMaxWriteMBPerSec(maxMBPerSec, Context.FLUSH);
- break;
- case 2: // sometimes rate limit flush & merge
- rateLimitedDirectoryWrapper.setMaxWriteMBPerSec(maxMBPerSec, Context.FLUSH);
- rateLimitedDirectoryWrapper.setMaxWriteMBPerSec(maxMBPerSec, Context.MERGE);
- break;
- default:
- rateLimitedDirectoryWrapper.setMaxWriteMBPerSec(maxMBPerSec, Context.MERGE);
- }
- directory = rateLimitedDirectoryWrapper;
-
- }
-
if (bare) {
BaseDirectoryWrapper base = new BaseDirectoryWrapper(directory);
closeAfterSuite(new CloseableDirectory(base, suiteFailureMarker));
@@ -1505,78 +1487,85 @@ public abstract class LuceneTestCase ext
throw null; // dummy to prevent compiler failure
}
}
+
+ public static IndexReader wrapReader(IndexReader r) throws IOException {
+ Random random = random();
+
+ // TODO: remove this, and fix those tests to wrap before putting slow around:
+ final boolean wasOriginallyAtomic = r instanceof LeafReader;
+ for (int i = 0, c = random.nextInt(6)+1; i < c; i++) {
+ switch(random.nextInt(6)) {
+ case 0:
+ r = SlowCompositeReaderWrapper.wrap(r);
+ break;
+ case 1:
+ // will create no FC insanity in atomic case, as ParallelLeafReader has own cache key:
+ r = (r instanceof LeafReader) ?
+ new ParallelLeafReader((LeafReader) r) :
+ new ParallelCompositeReader((CompositeReader) r);
+ break;
+ case 2:
+ // Häckidy-Hick-Hack: a standard MultiReader will cause FC insanity, so we use
+ // QueryUtils' reader with a fake cache key, so insanity checker cannot walk
+ // along our reader:
+ r = new FCInvisibleMultiReader(r);
+ break;
+ case 3:
+ final LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
+ final List<String> allFields = new ArrayList<>();
+ for (FieldInfo fi : ar.getFieldInfos()) {
+ allFields.add(fi.name);
+ }
+ Collections.shuffle(allFields, random);
+ final int end = allFields.isEmpty() ? 0 : random.nextInt(allFields.size());
+ final Set<String> fields = new HashSet<>(allFields.subList(0, end));
+ // will create no FC insanity as ParallelLeafReader has own cache key:
+ r = new ParallelLeafReader(
+ new FieldFilterLeafReader(ar, fields, false),
+ new FieldFilterLeafReader(ar, fields, true)
+ );
+ break;
+ case 4:
+ // Häckidy-Hick-Hack: a standard Reader will cause FC insanity, so we use
+ // QueryUtils' reader with a fake cache key, so insanity checker cannot walk
+ // along our reader:
+ if (r instanceof LeafReader) {
+ r = new AssertingLeafReader((LeafReader)r);
+ } else if (r instanceof DirectoryReader) {
+ r = new AssertingDirectoryReader((DirectoryReader)r);
+ }
+ break;
+ case 5:
+ if (r instanceof LeafReader) {
+ r = new MismatchedLeafReader((LeafReader)r, random);
+ } else if (r instanceof DirectoryReader) {
+ r = new MismatchedDirectoryReader((DirectoryReader)r, random);
+ }
+ break;
+ default:
+ fail("should not get here");
+ }
+ }
+ if (wasOriginallyAtomic) {
+ r = SlowCompositeReaderWrapper.wrap(r);
+ } else if ((r instanceof CompositeReader) && !(r instanceof FCInvisibleMultiReader)) {
+ // prevent cache insanity caused by e.g. ParallelCompositeReader, to fix we wrap one more time:
+ r = new FCInvisibleMultiReader(r);
+ }
+ if (VERBOSE) {
+ System.out.println("wrapReader wrapped: " +r);
+ }
+
+ return r;
+ }
/**
* Sometimes wrap the IndexReader as slow, parallel or filter reader (or
* combinations of that)
*/
public static IndexReader maybeWrapReader(IndexReader r) throws IOException {
- Random random = random();
if (rarely()) {
- // TODO: remove this, and fix those tests to wrap before putting slow around:
- final boolean wasOriginallyAtomic = r instanceof LeafReader;
- for (int i = 0, c = random.nextInt(6)+1; i < c; i++) {
- switch(random.nextInt(6)) {
- case 0:
- r = SlowCompositeReaderWrapper.wrap(r);
- break;
- case 1:
- // will create no FC insanity in atomic case, as ParallelLeafReader has own cache key:
- r = (r instanceof LeafReader) ?
- new ParallelLeafReader((LeafReader) r) :
- new ParallelCompositeReader((CompositeReader) r);
- break;
- case 2:
- // Häckidy-Hick-Hack: a standard MultiReader will cause FC insanity, so we use
- // QueryUtils' reader with a fake cache key, so insanity checker cannot walk
- // along our reader:
- r = new FCInvisibleMultiReader(r);
- break;
- case 3:
- final LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
- final List<String> allFields = new ArrayList<>();
- for (FieldInfo fi : ar.getFieldInfos()) {
- allFields.add(fi.name);
- }
- Collections.shuffle(allFields, random);
- final int end = allFields.isEmpty() ? 0 : random.nextInt(allFields.size());
- final Set<String> fields = new HashSet<>(allFields.subList(0, end));
- // will create no FC insanity as ParallelLeafReader has own cache key:
- r = new ParallelLeafReader(
- new FieldFilterLeafReader(ar, fields, false),
- new FieldFilterLeafReader(ar, fields, true)
- );
- break;
- case 4:
- // Häckidy-Hick-Hack: a standard Reader will cause FC insanity, so we use
- // QueryUtils' reader with a fake cache key, so insanity checker cannot walk
- // along our reader:
- if (r instanceof LeafReader) {
- r = new AssertingLeafReader((LeafReader)r);
- } else if (r instanceof DirectoryReader) {
- r = new AssertingDirectoryReader((DirectoryReader)r);
- }
- break;
- case 5:
- if (r instanceof LeafReader) {
- r = new MismatchedLeafReader((LeafReader)r, random);
- } else if (r instanceof DirectoryReader) {
- r = new MismatchedDirectoryReader((DirectoryReader)r, random);
- }
- break;
- default:
- fail("should not get here");
- }
- }
- if (wasOriginallyAtomic) {
- r = SlowCompositeReaderWrapper.wrap(r);
- } else if ((r instanceof CompositeReader) && !(r instanceof FCInvisibleMultiReader)) {
- // prevent cache insanity caused by e.g. ParallelCompositeReader, to fix we wrap one more time:
- r = new FCInvisibleMultiReader(r);
- }
- if (VERBOSE) {
- System.out.println("maybeWrapReader wrapped: " +r);
- }
+ r = wrapReader(r);
}
return r;
}
@@ -2510,4 +2499,17 @@ public abstract class LuceneTestCase ext
return c.getTotalHits();
}
}
+
+ /**
+ * Compares two strings with a collator, also looking to see if the the strings
+ * are impacted by jdk bugs. may not avoid all jdk bugs in tests.
+ * see https://bugs.openjdk.java.net/browse/JDK-8071862
+ */
+ public static int collate(Collator collator, String s1, String s2) {
+ int v1 = collator.compare(s1, s2);
+ int v2 = collator.getCollationKey(s1).compareTo(collator.getCollationKey(s2));
+ // if collation keys don't really respect collation order, things are screwed.
+ assumeTrue("hit JDK collator bug", Integer.signum(v1) == Integer.signum(v2));
+ return v1;
+ }
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java Sun Feb 8 23:53:14 2015
@@ -205,7 +205,7 @@ final class TestRuleTemporaryFilesCleanu
"Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
+ javaTempDir.toAbsolutePath());
}
- f = javaTempDir.resolve(prefix + "-" + ctx.getRunnerSeedAsString()
+ f = javaTempDir.resolve(prefix + " " + ctx.getRunnerSeedAsString()
+ "-" + String.format(Locale.ENGLISH, "%03d", attempt));
try {
Files.createDirectory(f);
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java Sun Feb 8 23:53:14 2015
@@ -29,6 +29,7 @@ import java.nio.CharBuffer;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -58,6 +59,7 @@ import org.apache.lucene.codecs.perfield
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
@@ -70,8 +72,10 @@ import org.apache.lucene.index.LeafReade
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.SlowCodecReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TieredMergePolicy;
@@ -870,6 +874,16 @@ public final class TestUtil {
return false;
}
}
+
+ public static void addIndexesSlowly(IndexWriter writer, DirectoryReader... readers) throws IOException {
+ List<CodecReader> leaves = new ArrayList<>();
+ for (DirectoryReader reader : readers) {
+ for (LeafReaderContext context : reader.leaves()) {
+ leaves.add(SlowCodecReaderWrapper.wrap(context.reader()));
+ }
+ }
+ writer.addIndexes(leaves.toArray(new CodecReader[leaves.size()]));
+ }
/** just tries to configure things to keep the open file
* count lowish */
Modified: lucene/dev/branches/lucene6005/lucene/tools/junit4/tests.policy
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/tools/junit4/tests.policy?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/tools/junit4/tests.policy (original)
+++ lucene/dev/branches/lucene6005/lucene/tools/junit4/tests.policy Sun Feb 8 23:53:14 2015
@@ -77,4 +77,14 @@ grant {
// SSL related properties for Solr tests
permission java.security.SecurityPermission "getProperty.ssl.*";
+ // SASL/Kerberos related properties for Solr tests
+ permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KerberosTicket * \"*\"", "read";
+
+ // may only be necessary with Java 7?
+ permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * \"*\"", "read";
+ permission javax.security.auth.PrivateCredentialPermission "sun.security.jgss.krb5.Krb5Util$KeysFromKeyTab * \"*\"", "read";
+
+ permission javax.security.auth.kerberos.ServicePermission "krbtgt/EXAMPLE.COM@EXAMPLE.COM", "initiate";
+ permission javax.security.auth.kerberos.ServicePermission "zookeeper/127.0.0.1@EXAMPLE.COM", "initiate";
+ permission javax.security.auth.kerberos.ServicePermission "zookeeper/127.0.0.1@EXAMPLE.COM", "accept";
};
Modified: lucene/dev/branches/lucene6005/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java Sun Feb 8 23:53:14 2015
@@ -75,7 +75,7 @@ import javax.xml.xpath.XPathFactory;
*/
public class GetMavenDependenciesTask extends Task {
private static final Pattern PROPERTY_PREFIX_FROM_IVY_XML_FILE_PATTERN = Pattern.compile
- ("[/\\\\](lucene|solr)[/\\\\](?:(?:contrib|(analysis)|(example))[/\\\\])?([^/\\\\]+)[/\\\\]ivy\\.xml");
+ ("[/\\\\](lucene|solr)[/\\\\](?:(?:contrib|(analysis)|(example)|(server))[/\\\\])?([^/\\\\]+)[/\\\\]ivy\\.xml");
private static final Pattern COORDINATE_KEY_PATTERN = Pattern.compile("/([^/]+)/([^/]+)");
private static final Pattern MODULE_DEPENDENCIES_COORDINATE_KEY_PATTERN
= Pattern.compile("(.*?)(\\.test)?\\.dependencies");
@@ -251,7 +251,7 @@ public class GetMavenDependenciesTask ex
}
/**
- * For each module that include other modules' external dependencies via
+ * For each module that includes other modules' external dependencies via
* including all files under their ".../lib/" dirs in their (test.)classpath,
* add the other modules' dependencies to its set of external dependencies.
*/
@@ -320,6 +320,7 @@ public class GetMavenDependenciesTask ex
if (globalOptionalExternalDependencies.contains(dependencyCoordinate)
|| (perModuleOptionalExternalDependencies.containsKey(module)
&& perModuleOptionalExternalDependencies.get(module).contains(dependencyCoordinate))) {
+ // make a copy of the dep and set optional=true
dep = new ExternalDependency(dep.groupId, dep.artifactId, dep.classifier, dep.isTestDependency, true);
}
deps.add(dep);
@@ -681,7 +682,7 @@ public class GetMavenDependenciesTask ex
String module = getModuleName(ivyXmlFile);
log("Collecting external dependencies from: " + ivyXmlFile.getPath(), verboseLevel);
Document document = documentBuilder.parse(ivyXmlFile);
- // Exclude the 'start' configuration in solr/example/ivy.xml
+ // Exclude the 'start' configuration in solr/server/ivy.xml
String dependencyPath = "/ivy-module/dependencies/dependency[not(starts-with(@conf,'start'))]";
NodeList dependencies = (NodeList)xpath.evaluate(dependencyPath, document, XPathConstants.NODESET);
for (int depNum = 0 ; depNum < dependencies.getLength() ; ++depNum) {
@@ -813,12 +814,13 @@ public class GetMavenDependenciesTask ex
builder.append(matcher.group(1));
if (null != matcher.group(2)) { // "lucene/analysis/..."
builder.append("-analyzers");
- }
- if (null != matcher.group(3)) { // "solr/example/..."
+ } else if (null != matcher.group(3)) { // "solr/example/..."
builder.append("-example");
+ } else if (null != matcher.group(4)) { // "solr/server/..."
+ builder.append("-server");
}
builder.append('-');
- builder.append(matcher.group(4));
+ builder.append(matcher.group(5));
return builder.toString().replace("solr-solr-", "solr-");
}