You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/11/27 21:59:48 UTC
svn commit: r1642230 - in /lucene/dev/branches/lucene6005: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/java/org/apache/lucene/util/
lucene/core/src/test/org/apache/lucene/analysis/
lucene/core/src/test/org/apa...
Author: mikemccand
Date: Thu Nov 27 20:59:47 2014
New Revision: 1642230
URL: http://svn.apache.org/r1642230
Log:
LUCENE-6005: merge trunk
Added:
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FilterCache.java
- copied unchanged from r1642229, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilterCache.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java
- copied unchanged from r1642229, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java
- copied unchanged from r1642229, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java
- copied unchanged from r1642229, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/FrequencyTrackingRingBuffer.java
- copied unchanged from r1642229, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/FrequencyTrackingRingBuffer.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
- copied unchanged from r1642229, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestFrequencyTrackingRingBuffer.java
- copied unchanged from r1642229, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestFrequencyTrackingRingBuffer.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java
- copied unchanged from r1642229, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java
Modified:
lucene/dev/branches/lucene6005/ (props changed)
lucene/dev/branches/lucene6005/lucene/ (props changed)
lucene/dev/branches/lucene6005/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene6005/lucene/build.xml (contents, props changed)
lucene/dev/branches/lucene6005/lucene/core/ (props changed)
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestLookaheadTokenFilter.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/fst/TestBytesStore.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
lucene/dev/branches/lucene6005/lucene/facet/ (props changed)
lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
lucene/dev/branches/lucene6005/lucene/join/ (props changed)
lucene/dev/branches/lucene6005/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java
lucene/dev/branches/lucene6005/lucene/misc/ (props changed)
lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java
lucene/dev/branches/lucene6005/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/BaseBitSetTestCase.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/lucene6005/solr/ (props changed)
lucene/dev/branches/lucene6005/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene6005/solr/core/ (props changed)
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/util/CommandOperation.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
Modified: lucene/dev/branches/lucene6005/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/CHANGES.txt?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6005/lucene/CHANGES.txt Thu Nov 27 20:59:47 2014
@@ -115,6 +115,8 @@ New Features
suggesters from multi-valued fields. (Varun Thacker via Mike
McCandless)
+* LUCENE-6077: Added a filter cache. (Adrien Grand, Robert Muir)
+
API Changes
* LUCENE-5900: Deprecated more constructors taking Version in *InfixSuggester and
Modified: lucene/dev/branches/lucene6005/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/build.xml?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/build.xml (original)
+++ lucene/dev/branches/lucene6005/lucene/build.xml Thu Nov 27 20:59:47 2014
@@ -448,7 +448,7 @@
</sequential>
</target>
- <target name="test-modules" depends="compile-test">
+ <target name="test-modules">
<modules-crawl target="test" failonerror="true"/>
</target>
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java Thu Nov 27 20:59:47 2014
@@ -22,7 +22,6 @@ import static org.apache.lucene.search.D
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.WeakHashMap;
@@ -41,13 +40,23 @@ import org.apache.lucene.util.RoaringDoc
*/
public class CachingWrapperFilter extends Filter implements Accountable {
private final Filter filter;
+ private final FilterCachingPolicy policy;
private final Map<Object,DocIdSet> cache = Collections.synchronizedMap(new WeakHashMap<Object,DocIdSet>());
- /** Wraps another filter's result and caches it.
+ /** Wraps another filter's result and caches it according to the provided policy.
* @param filter Filter to cache results of
+ * @param policy policy defining which filters should be cached on which segments
*/
- public CachingWrapperFilter(Filter filter) {
+ public CachingWrapperFilter(Filter filter, FilterCachingPolicy policy) {
this.filter = filter;
+ this.policy = policy;
+ }
+
+ /** Same as {@link CachingWrapperFilter#CachingWrapperFilter(Filter, FilterCachingPolicy)}
+ * but enforces the use of the
+ * {@link FilterCachingPolicy.CacheOnLargeSegments#DEFAULT} policy. */
+ public CachingWrapperFilter(Filter filter) {
+ this(filter, FilterCachingPolicy.CacheOnLargeSegments.DEFAULT);
}
/**
@@ -99,14 +108,17 @@ public class CachingWrapperFilter extend
if (docIdSet != null) {
hitCount++;
} else {
- missCount++;
- docIdSet = docIdSetToCache(filter.getDocIdSet(context, null), reader);
- if (docIdSet == null) {
- // We use EMPTY as a sentinel for the empty set, which is cacheable
- docIdSet = EMPTY;
+ docIdSet = filter.getDocIdSet(context, null);
+ if (policy.shouldCache(filter, context, docIdSet)) {
+ missCount++;
+ docIdSet = docIdSetToCache(docIdSet, reader);
+ if (docIdSet == null) {
+ // We use EMPTY as a sentinel for the empty set, which is cacheable
+ docIdSet = EMPTY;
+ }
+ assert docIdSet.isCacheable();
+ cache.put(key, docIdSet);
}
- assert docIdSet.isCacheable();
- cache.put(key, docIdSet);
}
return docIdSet == EMPTY ? null : BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestLookaheadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestLookaheadTokenFilter.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestLookaheadTokenFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestLookaheadTokenFilter.java Thu Nov 27 20:59:47 2014
@@ -32,8 +32,9 @@ public class TestLookaheadTokenFilter ex
TokenStream output = new MockRandomLookaheadTokenFilter(random, tokenizer);
return new TokenStreamComponents(tokenizer, output);
}
- };
- checkRandomData(random(), a, 200*RANDOM_MULTIPLIER, 8192);
+ };
+ int maxLength = TEST_NIGHTLY ? 8192 : 1024;
+ checkRandomData(random(), a, 50*RANDOM_MULTIPLIER, maxLength);
}
private static class NeverPeeksLookaheadTokenFilter extends LookaheadTokenFilter<LookaheadTokenFilter.Position> {
@@ -60,8 +61,9 @@ public class TestLookaheadTokenFilter ex
TokenStream output = new NeverPeeksLookaheadTokenFilter(tokenizer);
return new TokenStreamComponents(tokenizer, output);
}
- };
- checkRandomData(random(), a, 200*RANDOM_MULTIPLIER, 8192);
+ };
+ int maxLength = TEST_NIGHTLY ? 8192 : 1024;
+ checkRandomData(random(), a, 50*RANDOM_MULTIPLIER, maxLength);
}
public void testMissedFirstToken() throws Exception {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Thu Nov 27 20:59:47 2014
@@ -224,7 +224,8 @@ public class TestConcurrentMergeSchedule
);
FieldTypes fieldTypes = writer.getFieldTypes();
- for(int iter=0;iter<10;iter++) {
+ int numIters = TEST_NIGHTLY ? 10 : 3;
+ for(int iter=0;iter<numIters;iter++) {
for(int j=0;j<201;j++) {
Document2 doc = writer.newDocument();
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java Thu Nov 27 20:59:47 2014
@@ -71,8 +71,8 @@ public class TestExitableDirectoryReader
@Override
public BytesRef next() throws IOException {
try {
- // Sleep for 1s before each .next() call.
- Thread.sleep(1000);
+ // Sleep for 100ms before each .next() call.
+ Thread.sleep(100);
} catch (InterruptedException e) {
}
return in.next();
@@ -119,10 +119,10 @@ public class TestExitableDirectoryReader
Query query = new PrefixQuery(new Term("default", "o"));
- // Set a fairly high timeout value (10 seconds) and expect the query to complete in that time frame.
+ // Set a fairly high timeout value (1 second) and expect the query to complete in that time frame.
// Not checking the validity of the result, all we are bothered about in this test is the timing out.
directoryReader = DirectoryReader.open(directory);
- exitableDirectoryReader = new ExitableDirectoryReader(directoryReader, new QueryTimeoutImpl(10000));
+ exitableDirectoryReader = new ExitableDirectoryReader(directoryReader, new QueryTimeoutImpl(1000));
reader = new TestReader(exitableDirectoryReader);
searcher = new IndexSearcher(reader);
searcher.search(query, 10);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Thu Nov 27 20:59:47 2014
@@ -853,6 +853,12 @@ public class TestIndexWriter extends Luc
// When interrupt arrives in w.close(), this can
// lead to double-write of files:
dir.setPreventDoubleWrite(false);
+
+ // open/close slowly sometimes
+ dir.setUseSlowOpenClosers(true);
+
+ // throttle a little
+ dir.setThrottling(MockDirectoryWrapper.Throttling.SOMETIMES);
IndexWriter w = null;
while(!finish) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java Thu Nov 27 20:59:47 2014
@@ -40,29 +40,12 @@ import org.apache.lucene.store.RAMDirect
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.ThreadInterruptedException;
import org.junit.Test;
-// nocommit sometimes fails? ant test -Dtestcase=TestIndexWriterReader -Dtests.method=testAddIndexesAndDoDeletesThreads -Dtests.seed=964B420DA6617A73 -Dtests.locale=fr -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
-/*
- [junit4] > Throwable #1: java.io.EOFException: read past EOF: RAMInputStream(name=FieldTypes)
- [junit4] > at __randomizedtesting.SeedInfo.seed([57595240E47BF490:8D6A12648FE2F4AC]:0)
- [junit4] > at org.apache.lucene.store.RAMInputStream.switchCurrentBuffer(RAMInputStream.java:98)
- [junit4] > at org.apache.lucene.store.RAMInputStream.readBytes(RAMInputStream.java:81)
- [junit4] > at org.apache.lucene.store.BufferedChecksumIndexInput.readBytes(BufferedChecksumIndexInput.java:49)
- [junit4] > at org.apache.lucene.store.DataInput.readString(DataInput.java:234)
- [junit4] > at org.apache.lucene.document.FieldTypes$FieldType.<init>(FieldTypes.java:955)
- [junit4] > at org.apache.lucene.document.FieldTypes.readFromString(FieldTypes.java:3191)
- [junit4] > at org.apache.lucene.document.FieldTypes.loadFields(FieldTypes.java:1126)
- [junit4] > at org.apache.lucene.document.FieldTypes.<init>(FieldTypes.java:1112)
- [junit4] > at org.apache.lucene.document.FieldTypes.getFieldTypes(FieldTypes.java:3222)
- [junit4] > at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:481)
- [junit4] > at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:224)
- [junit4] > at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:211)
- [junit4] > at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:207)
- [junit4] > at org.apache.lucene.index.TestIndexWriterReader.testAddIndexesAndDoDeletesThreads(TestIndexWriterReader.java:383)
-*/
+@SuppressCodecs("SimpleText") // too slow here
public class TestIndexWriterReader extends LuceneTestCase {
private final int numThreads = TEST_NIGHTLY ? 5 : 3;
@@ -373,6 +356,7 @@ public class TestIndexWriterReader exten
dir1.close();
}
+ @Slow
public void testAddIndexesAndDoDeletesThreads() throws Throwable {
final int numIter = 2;
int numDirs = 3;
@@ -646,7 +630,7 @@ public class TestIndexWriterReader exten
((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);
- int num = atLeast(100);
+ int num = TEST_NIGHTLY ? atLeast(100) : atLeast(10);
for (int i = 0; i < num; i++) {
writer.addDocument(DocHelper.createDocument(writer, i, "test", 4));
}
@@ -725,6 +709,7 @@ public class TestIndexWriterReader exten
}
// Stress test reopen during addIndexes
+ @Nightly
public void testDuringAddIndexes() throws Exception {
Directory dir1 = getAssertNoDeletesDirectory(newDirectory());
final IndexWriter writer = new IndexWriter(
@@ -744,19 +729,20 @@ public class TestIndexWriterReader exten
DirectoryReader r = writer.getReader();
- final float SECONDS = 0.5f;
-
- final long endTime = (long) (System.currentTimeMillis() + 1000.*SECONDS);
+ final int numIterations = 10;
final List<Throwable> excs = Collections.synchronizedList(new ArrayList<Throwable>());
// Only one thread can addIndexes at a time, because
// IndexWriter acquires a write lock in each directory:
final Thread[] threads = new Thread[1];
+ final AtomicBoolean threadDone = new AtomicBoolean(false);
for(int i=0;i<threads.length;i++) {
threads[i] = new Thread() {
@Override
public void run() {
+ int count = 0;
do {
+ count++;
try {
writer.addIndexes(dirs);
writer.maybeMerge();
@@ -764,7 +750,8 @@ public class TestIndexWriterReader exten
excs.add(t);
throw new RuntimeException(t);
}
- } while(System.currentTimeMillis() < endTime);
+ } while(count < numIterations);
+ threadDone.set(true);
}
};
threads[i].setDaemon(true);
@@ -772,17 +759,17 @@ public class TestIndexWriterReader exten
}
int lastCount = 0;
- while(System.currentTimeMillis() < endTime) {
+ while(threadDone.get() == false) {
DirectoryReader r2 = DirectoryReader.openIfChanged(r);
if (r2 != null) {
r.close();
r = r2;
+ Query q = new TermQuery(new Term("indexname", "test"));
+ IndexSearcher searcher = newSearcher(r);
+ final int count = searcher.search(q, 10).totalHits;
+ assertTrue(count >= lastCount);
+ lastCount = count;
}
- Query q = new TermQuery(new Term("indexname", "test"));
- IndexSearcher searcher = newSearcher(r);
- final int count = searcher.search(q, 10).totalHits;
- assertTrue(count >= lastCount);
- lastCount = count;
}
for(int i=0;i<threads.length;i++) {
@@ -833,12 +820,11 @@ public class TestIndexWriterReader exten
DirectoryReader r = writer.getReader();
- final float SECONDS = 0.5f;
-
- final long endTime = (long) (System.currentTimeMillis() + 1000.*SECONDS);
+ final int iters = TEST_NIGHTLY ? 1000 : 10;
final List<Throwable> excs = Collections.synchronizedList(new ArrayList<Throwable>());
final Thread[] threads = new Thread[numThreads];
+ final AtomicInteger remainingThreads = new AtomicInteger(numThreads);
for(int i=0;i<numThreads;i++) {
threads[i] = new Thread() {
final Random r = new Random(random().nextLong());
@@ -861,7 +847,8 @@ public class TestIndexWriterReader exten
excs.add(t);
throw new RuntimeException(t);
}
- } while(System.currentTimeMillis() < endTime);
+ } while(count < iters);
+ remainingThreads.decrementAndGet();
}
};
threads[i].setDaemon(true);
@@ -869,15 +856,15 @@ public class TestIndexWriterReader exten
}
int sum = 0;
- while(System.currentTimeMillis() < endTime) {
+ while(remainingThreads.get() > 0) {
DirectoryReader r2 = DirectoryReader.openIfChanged(r);
if (r2 != null) {
r.close();
r = r2;
+ Query q = new TermQuery(new Term("indexname", "test"));
+ IndexSearcher searcher = newSearcher(r);
+ sum += searcher.search(q, 10).totalHits;
}
- Query q = new TermQuery(new Term("indexname", "test"));
- IndexSearcher searcher = newSearcher(r);
- sum += searcher.search(q, 10).totalHits;
}
for(int i=0;i<numThreads;i++) {
@@ -1134,7 +1121,7 @@ public class TestIndexWriterReader exten
/** Make sure if all we do is open NRT reader against
* writer, we don't see merge starvation. */
public void testTooManySegments() throws Exception {
- Directory dir = getAssertNoDeletesDirectory(newDirectory());
+ Directory dir = getAssertNoDeletesDirectory(new RAMDirectory());
// Don't use newIndexWriterConfig, because we need a
// "sane" mergePolicy:
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Thu Nov 27 20:59:47 2014
@@ -91,7 +91,7 @@ public class TestCachingWrapperFilter ex
public void testEmpty() throws Exception {
Query query = new BooleanQuery();
Filter expected = new QueryWrapperFilter(query);
- Filter actual = new CachingWrapperFilter(expected);
+ Filter actual = new CachingWrapperFilter(expected, MAYBE_CACHE_POLICY);
assertFilterEquals(expected, actual);
}
@@ -101,14 +101,14 @@ public class TestCachingWrapperFilter ex
query.add(new TermQuery(new Term("id", "0")), BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term("id", "0")), BooleanClause.Occur.MUST_NOT);
Filter expected = new QueryWrapperFilter(query);
- Filter actual = new CachingWrapperFilter(expected);
+ Filter actual = new CachingWrapperFilter(expected, MAYBE_CACHE_POLICY);
assertFilterEquals(expected, actual);
}
/** test null docidset */
public void testEmpty3() throws Exception {
Filter expected = new PrefixFilter(new Term("bogusField", "bogusVal"));
- Filter actual = new CachingWrapperFilter(expected);
+ Filter actual = new CachingWrapperFilter(expected, MAYBE_CACHE_POLICY);
assertFilterEquals(expected, actual);
}
@@ -118,7 +118,7 @@ public class TestCachingWrapperFilter ex
int id = random().nextInt(ir.maxDoc());
Query query = new TermQuery(new Term("id", Integer.toString(id)));
Filter expected = new QueryWrapperFilter(query);
- Filter actual = new CachingWrapperFilter(expected);
+ Filter actual = new CachingWrapperFilter(expected, MAYBE_CACHE_POLICY);
assertFilterEquals(expected, actual);
}
}
@@ -131,7 +131,7 @@ public class TestCachingWrapperFilter ex
Query query = TermRangeQuery.newStringRange("id",
Integer.toString(id_start), Integer.toString(id_end), true, true);
Filter expected = new QueryWrapperFilter(query);
- Filter actual = new CachingWrapperFilter(expected);
+ Filter actual = new CachingWrapperFilter(expected, MAYBE_CACHE_POLICY);
assertFilterEquals(expected, actual);
}
}
@@ -140,7 +140,7 @@ public class TestCachingWrapperFilter ex
public void testDense() throws Exception {
Query query = new MatchAllDocsQuery();
Filter expected = new QueryWrapperFilter(query);
- Filter actual = new CachingWrapperFilter(expected);
+ Filter actual = new CachingWrapperFilter(expected, FilterCachingPolicy.ALWAYS_CACHE);
assertFilterEquals(expected, actual);
}
@@ -152,7 +152,7 @@ public class TestCachingWrapperFilter ex
IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
LeafReaderContext context = (LeafReaderContext) reader.getContext();
MockFilter filter = new MockFilter();
- CachingWrapperFilter cacher = new CachingWrapperFilter(filter);
+ CachingWrapperFilter cacher = new CachingWrapperFilter(filter, FilterCachingPolicy.ALWAYS_CACHE);
// first time, nested filter is called
DocIdSet strongRef = cacher.getDocIdSet(context, context.reader().getLiveDocs());
@@ -184,7 +184,7 @@ public class TestCachingWrapperFilter ex
return null;
}
};
- CachingWrapperFilter cacher = new CachingWrapperFilter(filter);
+ CachingWrapperFilter cacher = new CachingWrapperFilter(filter, MAYBE_CACHE_POLICY);
// the caching filter should return the empty set constant
assertNull(cacher.getDocIdSet(context, context.reader().getLiveDocs()));
@@ -217,7 +217,7 @@ public class TestCachingWrapperFilter ex
};
}
};
- CachingWrapperFilter cacher = new CachingWrapperFilter(filter);
+ CachingWrapperFilter cacher = new CachingWrapperFilter(filter, FilterCachingPolicy.ALWAYS_CACHE);
// the caching filter should return the empty set constant
assertNull(cacher.getDocIdSet(context, context.reader().getLiveDocs()));
@@ -229,7 +229,7 @@ public class TestCachingWrapperFilter ex
private static void assertDocIdSetCacheable(IndexReader reader, Filter filter, boolean shouldCacheable) throws IOException {
assertTrue(reader.getContext() instanceof LeafReaderContext);
LeafReaderContext context = (LeafReaderContext) reader.getContext();
- final CachingWrapperFilter cacher = new CachingWrapperFilter(filter);
+ final CachingWrapperFilter cacher = new CachingWrapperFilter(filter, FilterCachingPolicy.ALWAYS_CACHE);
final DocIdSet originalSet = filter.getDocIdSet(context, context.reader().getLiveDocs());
final DocIdSet cachedSet = cacher.getDocIdSet(context, context.reader().getLiveDocs());
if (originalSet == null) {
@@ -307,7 +307,7 @@ public class TestCachingWrapperFilter ex
final Filter startFilter = new QueryWrapperFilter(new TermQuery(new Term("id", "1")));
- CachingWrapperFilter filter = new CachingWrapperFilter(startFilter);
+ CachingWrapperFilter filter = new CachingWrapperFilter(startFilter, FilterCachingPolicy.ALWAYS_CACHE);
docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
assertTrue(filter.ramBytesUsed() > 0);
@@ -355,7 +355,7 @@ public class TestCachingWrapperFilter ex
assertEquals("[just filter] Should *not* find a hit...", 0, docs.totalHits);
// apply deletes dynamically:
- filter = new CachingWrapperFilter(startFilter);
+ filter = new CachingWrapperFilter(startFilter, FilterCachingPolicy.ALWAYS_CACHE);
writer.addDocument(doc);
reader = refreshReader(reader);
searcher = newSearcher(reader, false);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java Thu Nov 27 20:59:47 2014
@@ -33,6 +33,8 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.RAMDirectory;
public class TestSloppyPhraseQuery extends LuceneTestCase {
@@ -138,7 +140,7 @@ public class TestSloppyPhraseQuery exten
private float checkPhraseQuery(String doc, PhraseQuery query, int slop, int expectedNumResults) throws Exception {
query.setSlop(slop);
- Directory ramDir = newDirectory();
+ MockDirectoryWrapper ramDir = new MockDirectoryWrapper(random(), new RAMDirectory());
RandomIndexWriter writer = new RandomIndexWriter(random(), ramDir, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));
addDocument(writer, doc);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java Thu Nov 27 20:59:47 2014
@@ -74,8 +74,7 @@ public class TestTopDocsMerge extends Lu
IndexReader reader = null;
Directory dir = null;
- final int numDocs = atLeast(1000);
- //final int numDocs = atLeast(50);
+ final int numDocs = TEST_NIGHTLY ? atLeast(1000) : atLeast(100);
final String[] tokens = new String[] {"a", "b", "c", "d", "e"};
@@ -164,7 +163,8 @@ public class TestTopDocsMerge extends Lu
sortFields.add(new SortField(null, SortField.Type.DOC, true));
sortFields.add(new SortField(null, SortField.Type.DOC, false));
- for(int iter=0;iter<1000*RANDOM_MULTIPLIER;iter++) {
+ int numIters = atLeast(300);
+ for(int iter=0;iter<numIters;iter++) {
// TODO: custom FieldComp...
final Query query = new TermQuery(new Term("text", tokens[random().nextInt(tokens.length)]));
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java Thu Nov 27 20:59:47 2014
@@ -208,8 +208,9 @@ public class TestFixedBitSet extends Bas
// large enough to flush obvious bugs, small enough to run in <.5 sec as part of a
// larger testsuite.
public void testSmall() throws IOException {
- doRandomSets(atLeast(1200), atLeast(1000), 1);
- doRandomSets(atLeast(1200), atLeast(1000), 2);
+ final int iters = TEST_NIGHTLY ? atLeast(1000) : 100;
+ doRandomSets(atLeast(1200), iters, 1);
+ doRandomSets(atLeast(1200), iters, 2);
}
// uncomment to run a bigger test (~2 minutes).
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/fst/TestBytesStore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/fst/TestBytesStore.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/fst/TestBytesStore.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/fst/TestBytesStore.java Thu Nov 27 20:59:47 2014
@@ -31,8 +31,9 @@ public class TestBytesStore extends Luce
public void testRandom() throws Exception {
final int iters = atLeast(10);
+ final int maxBytes = TEST_NIGHTLY ? 200000 : 20000;
for(int iter=0;iter<iters;iter++) {
- final int numBytes = TestUtil.nextInt(random(), 1, 200000);
+ final int numBytes = TestUtil.nextInt(random(), 1, maxBytes);
final byte[] expected = new byte[numBytes];
final int blockBits = TestUtil.nextInt(random(), 8, 15);
final BytesStore bytes = new BytesStore(blockBits);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java Thu Nov 27 20:59:47 2014
@@ -85,7 +85,8 @@ public class TestDirectPacked extends Lu
private void doTestBpv(Directory directory, int bpv) throws Exception {
MyRandom random = new MyRandom(random().nextLong());
- for (int i = 0; i < 100; i++) {
+ int numIters = TEST_NIGHTLY ? 100 : 10;
+ for (int i = 0; i < numIters; i++) {
long original[] = randomLongs(random, bpv);
int bitsRequired = bpv == 64 ? 64 : DirectWriter.bitsRequired(1L<<(bpv-1));
String name = "bpv" + bpv + "_" + i;
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java Thu Nov 27 20:59:47 2014
@@ -59,6 +59,7 @@ import org.apache.lucene.search.Constant
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.FilterCachingPolicy;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.NumericRangeFilter;
@@ -914,7 +915,7 @@ public class TestRangeFacetCounts extend
final AtomicBoolean filterWasUsed = new AtomicBoolean();
if (random().nextBoolean()) {
// Sort of silly:
- fastMatchFilter = new CachingWrapperFilter(new QueryWrapperFilter(new MatchAllDocsQuery())) {
+ fastMatchFilter = new CachingWrapperFilter(new QueryWrapperFilter(new MatchAllDocsQuery()), FilterCachingPolicy.ALWAYS_CACHE) {
@Override
protected DocIdSet cacheImpl(DocIdSetIterator iterator, LeafReader reader)
throws IOException {
Modified: lucene/dev/branches/lucene6005/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java Thu Nov 27 20:59:47 2014
@@ -25,6 +25,7 @@ import org.apache.lucene.search.CachingW
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.FilterCachingPolicy;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BitDocIdSet;
@@ -39,7 +40,7 @@ public class BitDocIdSetCachingWrapperFi
/** Sole constructor. */
public BitDocIdSetCachingWrapperFilter(Filter filter) {
super();
- this.filter = new CachingWrapperFilter(filter) {
+ this.filter = new CachingWrapperFilter(filter, FilterCachingPolicy.ALWAYS_CACHE) {
@Override
protected BitDocIdSet docIdSetToCache(DocIdSet docIdSet, LeafReader reader) throws IOException {
if (docIdSet == null || docIdSet instanceof BitDocIdSet) {
Modified: lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java Thu Nov 27 20:59:47 2014
@@ -36,6 +36,7 @@ import org.apache.lucene.search.CachingW
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.FilterCachingPolicy;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
@@ -50,7 +51,7 @@ public class TestBlockJoinSorter extends
private static class FixedBitSetCachingWrapperFilter extends CachingWrapperFilter {
public FixedBitSetCachingWrapperFilter(Filter filter) {
- super(filter);
+ super(filter, FilterCachingPolicy.ALWAYS_CACHE);
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Thu Nov 27 20:59:47 2014
@@ -37,6 +37,7 @@ import org.apache.lucene.codecs.blockter
import org.apache.lucene.codecs.blockterms.LuceneVarGapFixedInterval;
import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
import org.apache.lucene.codecs.bloom.TestBloomFilteredLucenePostings;
+import org.apache.lucene.codecs.memory.DirectDocValuesFormat;
import org.apache.lucene.codecs.memory.DirectPostingsFormat;
import org.apache.lucene.codecs.memory.FSTOrdPostingsFormat;
import org.apache.lucene.codecs.memory.FSTPostingsFormat;
@@ -136,15 +137,16 @@ public class RandomCodec extends Asserti
new LuceneFixedGap(TestUtil.nextInt(random, 1, 1000)),
new LuceneVarGapFixedInterval(TestUtil.nextInt(random, 1, 1000)),
new LuceneVarGapDocFreqInterval(TestUtil.nextInt(random, 1, 100), TestUtil.nextInt(random, 1, 1000)),
- new SimpleTextPostingsFormat(),
+ random.nextInt(10) == 0 ? new SimpleTextPostingsFormat() : TestUtil.getDefaultPostingsFormat(),
new AssertingPostingsFormat(),
new MemoryPostingsFormat(true, random.nextFloat()),
new MemoryPostingsFormat(false, random.nextFloat()));
addDocValues(avoidCodecs,
TestUtil.getDefaultDocValuesFormat(),
+ new DirectDocValuesFormat(), // maybe not a great idea...
new MemoryDocValuesFormat(),
- new SimpleTextDocValuesFormat(),
+ random.nextInt(10) == 0 ? new SimpleTextDocValuesFormat() : TestUtil.getDefaultDocValuesFormat(),
new AssertingDocValuesFormat());
Collections.shuffle(formats, random);
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Thu Nov 27 20:59:47 2014
@@ -75,7 +75,7 @@ public class MockDirectoryWrapper extend
boolean preventDoubleWrite = true;
boolean trackDiskUsage = false;
boolean wrapLocking = true;
- boolean useSlowOpenClosers = true;
+ boolean useSlowOpenClosers = LuceneTestCase.TEST_NIGHTLY;
boolean enableVirusScanner = true;
boolean allowRandomFileNotFoundException = true;
boolean allowReadingFilesStillOpenForWrite = false;
@@ -85,7 +85,7 @@ public class MockDirectoryWrapper extend
Map<String,Exception> openLocks = Collections.synchronizedMap(new HashMap<String,Exception>());
volatile boolean crashed;
private ThrottledIndexOutput throttledOutput;
- private Throttling throttling = Throttling.SOMETIMES;
+ private Throttling throttling = LuceneTestCase.TEST_NIGHTLY ? Throttling.SOMETIMES : Throttling.NEVER;
final AtomicInteger inputCloneCount = new AtomicInteger();
@@ -191,9 +191,9 @@ public class MockDirectoryWrapper extend
}
/**
- * By default, opening and closing has a rare small sleep to catch race conditions
+ * Add a rare small sleep to catch race conditions in open/close
* <p>
- * You can disable this if you dont need it
+ * You can enable this if you need it.
*/
public void setUseSlowOpenClosers(boolean v) {
useSlowOpenClosers = v;
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/BaseBitSetTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/BaseBitSetTestCase.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/BaseBitSetTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/BaseBitSetTestCase.java Thu Nov 27 20:59:47 2014
@@ -176,7 +176,7 @@ public abstract class BaseBitSetTestCase
final int numBits = 1 + random().nextInt(100000);
BitSet set1 = new JavaUtilBitSet(randomSet(numBits, 0), numBits);
T set2 = copyOf(set1, numBits);
- final int iters = 50 + random().nextInt(50);
+ final int iters = TEST_NIGHTLY ? 50 + random().nextInt(50) : 10 + random().nextInt(10);
for (int i = 0; i < iters; ++i) {
// make extreme percents more likely
float percentSet2 = rarely() ? 0 : (float) Math.pow(random().nextDouble(), 2);
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Thu Nov 27 20:59:47 2014
@@ -110,7 +110,10 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.AssertingIndexSearcher;
+import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.FilterCachingPolicy;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryUtils.FCInvisibleMultiReader;
import org.apache.lucene.store.BaseDirectoryWrapper;
@@ -140,6 +143,7 @@ import org.junit.Test;
import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;
import org.junit.runner.RunWith;
+
import com.carrotsearch.randomizedtesting.JUnit4MethodProvider;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.carrotsearch.randomizedtesting.MixWithSuiteName;
@@ -466,7 +470,20 @@ public abstract class LuceneTestCase ext
CORE_DIRECTORIES = new ArrayList<>(FS_DIRECTORIES);
CORE_DIRECTORIES.add("RAMDirectory");
};
-
+
+ /** A {@link FilterCachingPolicy} that randomly caches. */
+ public static final FilterCachingPolicy MAYBE_CACHE_POLICY = new FilterCachingPolicy() {
+
+ @Override
+ public void onCache(Filter filter) {}
+
+ @Override
+ public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
+ return random().nextBoolean();
+ }
+
+ };
+
// -----------------------------------------------------------------
// Fields initialized in class or instance rules.
// -----------------------------------------------------------------
@@ -1285,6 +1302,10 @@ public abstract class LuceneTestCase ext
String fsdirClass = TEST_DIRECTORY;
if (fsdirClass.equals("random")) {
fsdirClass = RandomPicks.randomFrom(random(), FS_DIRECTORIES);
+ if (fsdirClass.equals("SimpleFSDirectory")) {
+ // pick again
+ fsdirClass = RandomPicks.randomFrom(random(), FS_DIRECTORIES);
+ }
}
Class<? extends FSDirectory> clazz;
@@ -1324,7 +1345,7 @@ public abstract class LuceneTestCase ext
directory = new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble());
}
- if (rarely(random) && !bare) {
+ if (TEST_NIGHTLY && rarely(random) && !bare) {
final double maxMBPerSec = TestUtil.nextInt(random, 20, 40);
if (LuceneTestCase.VERBOSE) {
System.out.println("LuceneTestCase: will rate limit output IndexOutput to " + maxMBPerSec + " MB/sec");
@@ -1513,6 +1534,10 @@ public abstract class LuceneTestCase ext
if (clazzName.equals("random")) {
if (rarely(random)) {
clazzName = RandomPicks.randomFrom(random, CORE_DIRECTORIES);
+ if (clazzName.equals("SimpleFSDirectory")) {
+ // pick again
+ clazzName = RandomPicks.randomFrom(random, CORE_DIRECTORIES);
+ }
} else {
clazzName = "RAMDirectory";
}
Modified: lucene/dev/branches/lucene6005/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/CHANGES.txt?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6005/solr/CHANGES.txt Thu Nov 27 20:59:47 2014
@@ -217,6 +217,8 @@ New Features
* SOLR-6533: Support editing common solrconfig.xml values (Noble Paul)
+* SOLR-6607: Managing requesthandlers throuh API (Noble Paul)
+
Bug Fixes
----------------------
@@ -445,6 +447,12 @@ Other Changes
* SOLR-6799: Update Saxon-HE to 9.6.0-2. (Mark Miller)
+* SOLR-6454: Suppress EOFExceptions in SolrDispatchFilter.
+ (Ramkumar Aiyengar via Mark Miller)
+
+* SOLR-6370: Allow tests to report/fail on many ZK watches being parallelly
+ requested on the same data (Ramkumar Aiyengar via Timothy Potter)
+
================== 4.10.3 ==================
Bug Fixes
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java Thu Nov 27 20:59:47 2014
@@ -2235,17 +2235,19 @@ public final class ZkController {
log.info("Watcher on {} is removed ", zkDir);
return;
}
- final Set<Runnable> listeners = confDirectoryListeners.get(zkDir);
+ Set<Runnable> listeners = confDirectoryListeners.get(zkDir);
if (listeners != null && !listeners.isEmpty()) {
+ final Set<Runnable> listenersCopy = new HashSet<>(listeners);
new Thread() {
//run these in a separate thread because this can be long running
public void run() {
- for (final Runnable listener : listeners)
+ for (final Runnable listener : listenersCopy) {
try {
listener.run();
} catch (Exception e) {
log.warn("listener throws error", e);
}
+ }
}
}.start();
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java Thu Nov 27 20:59:47 2014
@@ -28,17 +28,22 @@ import java.util.Map;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.request.SolrRequestHandler;
import org.noggit.CharArr;
import org.noggit.JSONParser;
import org.noggit.JSONWriter;
import org.noggit.ObjectBuilder;
+import static org.apache.solr.common.params.CoreAdminParams.NAME;
+
public class ConfigOverlay implements MapSerializable{
private final int znodeVersion ;
private Map<String, Object> data;
private Map<String,Object> props;
private Map<String,Object> userProps;
+ private Map<String, Map> reqHandlers;
public ConfigOverlay(Map<String,Object> jsonObj, int znodeVersion){
if(jsonObj == null) jsonObj= Collections.EMPTY_MAP;
@@ -48,6 +53,8 @@ public class ConfigOverlay implements Ma
if(props == null) props= Collections.EMPTY_MAP;
userProps = (Map<String, Object>) data.get("userProps");
if(userProps == null) userProps= Collections.EMPTY_MAP;
+ reqHandlers = (Map<String, Map>) data.get(SolrRequestHandler.TYPE);
+ if(reqHandlers == null) reqHandlers = new LinkedHashMap<>();
}
public Object getXPathProperty(String xpath){
@@ -255,4 +262,28 @@ public class ConfigOverlay implements Ma
result.putAll(data);
return result;
}
+
+ public Map<String, Map> getReqHandlers() {
+ return Collections.unmodifiableMap(reqHandlers);
+ }
+
+ public ConfigOverlay addReqHandler(Map<String, Object> info) {
+ ConfigOverlay copy = copyOverLayWithReqHandler();
+ copy.reqHandlers.put((String)info.get(NAME) , info);
+ return copy;
+ }
+
+ private ConfigOverlay copyOverLayWithReqHandler() {
+ LinkedHashMap<String, Object> newmap = new LinkedHashMap<>(data);
+ ConfigOverlay copy = new ConfigOverlay(newmap, znodeVersion);
+ newmap.put(SolrRequestHandler.TYPE, copy.reqHandlers = new LinkedHashMap<>(reqHandlers));
+ return copy;
+ }
+
+ public ConfigOverlay deleteHandler(String name) {
+ ConfigOverlay copy = copyOverLayWithReqHandler();
+ copy.reqHandlers.remove(name);
+ return copy;
+
+ }
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java Thu Nov 27 20:59:47 2014
@@ -22,8 +22,12 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import java.util.*;
+
+import static java.util.Arrays.asList;
import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableMap;
+import static org.apache.solr.common.params.CoreAdminParams.NAME;
+import static org.apache.solr.schema.FieldType.CLASS_NAME;
/**
* An Object which represents a Plugin of any type
@@ -38,8 +42,8 @@ public class PluginInfo implements MapSe
public PluginInfo(String type, Map<String, String> attrs ,NamedList initArgs, List<PluginInfo> children) {
this.type = type;
- this.name = attrs.get("name");
- this.className = attrs.get("class");
+ this.name = attrs.get(NAME);
+ this.className = attrs.get(CLASS_NAME);
this.initArgs = initArgs;
attributes = unmodifiableMap(attrs);
this.children = children == null ? Collections.<PluginInfo>emptyList(): unmodifiableList(children);
@@ -49,14 +53,27 @@ public class PluginInfo implements MapSe
public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) {
type = node.getNodeName();
- name = DOMUtil.getAttr(node, "name", requireName ? err : null);
- className = DOMUtil.getAttr(node, "class", requireClass ? err : null);
+ name = DOMUtil.getAttr(node, NAME, requireName ? err : null);
+ className = DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null);
initArgs = DOMUtil.childNodesToNamedList(node);
attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes()));
children = loadSubPlugins(node);
isFromSolrConfig = true;
}
+ public PluginInfo(String type, Map<String,Object> map) {
+ LinkedHashMap m = new LinkedHashMap<>(map);
+ NamedList nl = new NamedList();
+ for (String s : asList(DEFAULTS, APPENDS, INVARIANTS)) if (m.get(s) != null) nl.add(s, map.remove(s));
+ this.type = type;
+ this.name = (String) m.get(NAME);
+ this.className = (String) m.get(CLASS_NAME);
+ this.initArgs = nl;
+ attributes = unmodifiableMap(m);
+ this.children = Collections.<PluginInfo>emptyList();
+ isFromSolrConfig = true;
+ }
+
private List<PluginInfo> loadSubPlugins(Node node) {
List<PluginInfo> children = new ArrayList<>();
//if there is another sub tag with a non namedlist tag that has to be another plugin
@@ -131,11 +148,11 @@ public class PluginInfo implements MapSe
public static final PluginInfo EMPTY_INFO = new PluginInfo("",Collections.<String,String>emptyMap(), new NamedList(),Collections.<PluginInfo>emptyList());
private static final HashSet<String> NL_TAGS = new HashSet<>
- (Arrays.asList("lst", "arr",
- "bool",
- "str",
- "int","long",
- "float","double"));
+ (asList("lst", "arr",
+ "bool",
+ "str",
+ "int", "long",
+ "float", "double"));
public static final String DEFAULTS = "defaults";
public static final String APPENDS = "appends";
public static final String INVARIANTS = "invariants";
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java Thu Nov 27 20:59:47 2014
@@ -143,12 +143,15 @@ public final class RequestHandlers {
List<PluginInfo> implicits = PluginsRegistry.getHandlers(core);
// use link map so we iterate in the same order
Map<PluginInfo,SolrRequestHandler> handlers = new LinkedHashMap<>();
- Map<String, PluginInfo> implicitInfoMap= new HashMap<>();
+ Map<String, PluginInfo> infoMap= new LinkedHashMap<>();
//deduping implicit and explicit requesthandlers
- for (PluginInfo info : implicits) implicitInfoMap.put(info.name,info);
+ for (PluginInfo info : implicits) infoMap.put(info.name,info);
for (PluginInfo info : config.getPluginInfos(SolrRequestHandler.class.getName()))
- if(implicitInfoMap.containsKey(info.name)) implicitInfoMap.remove(info.name);
- ArrayList<PluginInfo> infos = new ArrayList<>(implicitInfoMap.values());
+ if(infoMap.containsKey(info.name)) infoMap.remove(info.name);
+ for (Map.Entry e : core.getSolrConfig().getOverlay().getReqHandlers().entrySet())
+ infoMap.put((String)e.getKey(), new PluginInfo(SolrRequestHandler.TYPE, (Map)e.getValue()));
+
+ ArrayList<PluginInfo> infos = new ArrayList<>(infoMap.values());
infos.addAll(config.getPluginInfos(SolrRequestHandler.class.getName()));
for (PluginInfo info : infos) {
try {
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java Thu Nov 27 20:59:47 2014
@@ -287,7 +287,7 @@ public class SolrConfig extends Config i
}
public static List<SolrPluginInfo> plugins = ImmutableList.<SolrPluginInfo>builder()
- .add(new SolrPluginInfo(SolrRequestHandler.class, "requestHandler", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
+ .add(new SolrPluginInfo(SolrRequestHandler.class, SolrRequestHandler.TYPE, REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(QParserPlugin.class, "queryParser", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(QueryResponseWriter.class, "queryResponseWriter", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(ValueSourceParser.class, "valueSourceParser", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
@@ -334,13 +334,14 @@ public class SolrConfig extends Config i
in = loader.openResource(ConfigOverlay.RESOURCE_NAME);
} catch (IOException e) {
//no problem no overlay.json file
- return new ConfigOverlay(Collections.EMPTY_MAP,0);
+ return new ConfigOverlay(Collections.EMPTY_MAP,-1);
}
try {
int version = 0; //will be always 0 for file based resourceloader
if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {
version = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion();
+ log.info("config overlay loaded . version : {} ", version);
}
Map m = (Map) ObjectBuilder.getVal(new JSONParser(new InputStreamReader(in, StandardCharsets.UTF_8)));
return new ConfigOverlay(m,version);
@@ -731,6 +732,9 @@ public class SolrConfig extends Config i
if(plugin.options.contains(PluginOpts.REQUIRE_NAME)){
LinkedHashMap items = new LinkedHashMap();
for (PluginInfo info : infos) items.put(info.name, info.toMap());
+ if(tag.equals(SolrRequestHandler.TYPE)){
+ for (Map.Entry e : overlay.getReqHandlers().entrySet()) items.put(e.getKey(),e.getValue());
+ }
result.put(tag,items);
} else {
if(plugin.options.contains(MULTI_OK)){
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java Thu Nov 27 20:59:47 2014
@@ -51,6 +51,7 @@ import org.apache.solr.request.LocalSolr
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaManager;
import org.apache.solr.util.CommandOperation;
import org.apache.solr.util.plugin.SolrCoreAware;
@@ -63,8 +64,10 @@ import static java.text.MessageFormat.fo
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
+import static org.apache.solr.common.params.CoreAdminParams.NAME;
import static org.apache.solr.core.ConfigOverlay.NOT_EDITABLE;
import static org.apache.solr.core.PluginInfo.DEFAULTS;
+import static org.apache.solr.schema.FieldType.CLASS_NAME;
public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAware{
public static final Logger log = LoggerFactory.getLogger(SolrConfigHandler.class);
@@ -106,7 +109,7 @@ public class SolrConfigHandler extends R
return new Runnable() {
@Override
public void run() {
- log.info("config update_listener called");
+ log.info("config update listener called for core {}", coreName);
SolrZkClient zkClient = cc.getZkController().getZkClient();
int solrConfigversion,overlayVersion;
try (SolrCore core = cc.getCore(coreName)) {
@@ -117,7 +120,7 @@ public class SolrConfigHandler extends R
if (checkStale(zkClient, overlayPath, solrConfigversion) ||
checkStale(zkClient, solrConfigPath, overlayVersion)) {
- log.info("core reload");
+ log.info("core reload {}",coreName);
cc.reload(coreName);
}
}
@@ -128,7 +131,7 @@ public class SolrConfigHandler extends R
try {
Stat stat = zkClient.exists(zkPath, null, true);
if(stat == null){
- if(currentVersion>0) return true;
+ if(currentVersion > -1) return true;
return false;
}
if (stat.getVersion() > currentVersion) {
@@ -198,7 +201,7 @@ public class SolrConfigHandler extends R
for (CommandOperation op : ops) opsCopy.add(op.getCopy());
try {
handleCommands(opsCopy, overlay);
- break;
+ break;//succeeded . so no need to go over the loop again
} catch (ZkController.ResourceModifiedInZkException e) {
//retry
log.info("Race condition, the node is modified in ZK by someone else " +e.getMessage());
@@ -226,6 +229,13 @@ public class SolrConfigHandler extends R
case UNSET_USER_PROPERTY:
overlay = applyUnsetUserProp(op, overlay);
break;
+ case UPDATE_REQHANDLER:
+ case CREATE_REQHANDLER:
+ overlay = applyRequestHandler(op, overlay);
+ break;
+ case DELETE_REQHANDLER:
+ overlay = applyDeleteHandler(op,overlay);
+ break;
}
}
List errs = CommandOperation.captureErrors(ops);
@@ -246,6 +256,50 @@ public class SolrConfigHandler extends R
}
+ private ConfigOverlay applyDeleteHandler(CommandOperation op, ConfigOverlay overlay) {
+ String name = op.getStr(CommandOperation.ROOT_OBJ);
+ if(op.hasError()) return overlay;
+ if(overlay.getReqHandlers().containsKey(name)){
+ return overlay.deleteHandler(name);
+ } else {
+ op.addError(MessageFormat.format("NO such requestHandler ''{0}'' ",name));
+ return overlay;
+ }
+
+ }
+
+ private ConfigOverlay applyRequestHandler(CommandOperation op, ConfigOverlay overlay) {
+ String name=op.getStr(NAME);
+ op.getStr(CLASS_NAME);
+ op.getMap(PluginInfo.DEFAULTS, null);
+ op.getMap(PluginInfo.INVARIANTS,null);
+ op.getMap(PluginInfo.APPENDS,null);
+ if(op.hasError()) return overlay;
+
+
+ if(CREATE_REQHANDLER.equals(op.name)) {
+ if (overlay.getReqHandlers().containsKey(name)) {
+ op.addError(MessageFormat.format(" ''{0}'' already exists . Do an ''{1}'' , if you want to change it ", name, UPDATE_REQHANDLER));
+ return overlay;
+ } else {
+ return overlay.addReqHandler(op.getDataMap());
+ }
+ } else if(UPDATE_REQHANDLER.equals(op.name)){
+ if (!overlay.getReqHandlers().containsKey(name)) {
+ op.addError(MessageFormat.format(" ''{0}'' does not exist . Do an ''{1}'' , if you want to create it ", name, CREATE_REQHANDLER));
+ return overlay;
+ } else {
+ return overlay.addReqHandler(op.getDataMap());
+
+ }
+ }
+
+ return overlay;
+
+
+
+ }
+
private ConfigOverlay applySetUserProp(CommandOperation op, ConfigOverlay overlay) {
Map<String, Object> m = op.getDataMap();
if(op.hasError()) return overlay;
@@ -351,7 +405,8 @@ public class SolrConfigHandler extends R
public static final String UNSET_PROPERTY = "unset-property";
public static final String SET_USER_PROPERTY = "set-user-property";
public static final String UNSET_USER_PROPERTY = "unset-user-property";
-
-
+ public static final String CREATE_REQHANDLER = "create-requesthandler";
+ public static final String DELETE_REQHANDLER = "delete-requesthandler";
+ public static final String UPDATE_REQHANDLER = "update-requesthandler";
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java Thu Nov 27 20:59:47 2014
@@ -49,7 +49,6 @@ import org.apache.solr.common.params.Com
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
@@ -64,7 +63,6 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
-import org.apache.solr.response.BinaryQueryResponseWriter;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.QueryResponseWriterUtil;
import org.apache.solr.response.SolrQueryResponse;
@@ -83,6 +81,7 @@ import javax.servlet.http.HttpServletReq
import javax.servlet.http.HttpServletResponse;
import java.io.ByteArrayInputStream;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -203,15 +202,17 @@ public class SolrDispatchFilter extends
}
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain, boolean retry) throws IOException, ServletException {
- if( abortErrorMessage != null ) {
- ((HttpServletResponse)response).sendError( 500, abortErrorMessage );
+
+ if (abortErrorMessage != null) {
+ sendError((HttpServletResponse) response, 500, abortErrorMessage);
return;
}
-
+
if (this.cores == null) {
- ((HttpServletResponse)response).sendError( 503, "Server is shutting down or failed to initialize" );
+ sendError((HttpServletResponse) response, 503, "Server is shutting down or failed to initialize");
return;
}
+
CoreContainer cores = this.cores;
SolrCore core = null;
SolrQueryRequest solrReq = null;
@@ -298,7 +299,7 @@ public class SolrDispatchFilter extends
if (core == null && cores.isZooKeeperAware()) {
// we couldn't find the core - lets make sure a collection was not specified instead
- core = getCoreByCollection(cores, corename, path);
+ core = getCoreByCollection(cores, corename);
if (core != null) {
// we found a core, update the path
@@ -520,7 +521,7 @@ public class SolrDispatchFilter extends
String queryString = req.getQueryString();
urlstr += queryString == null ? "" : "?" + queryString;
-
+
URL url = new URL(urlstr);
boolean isPostOrPutRequest = "POST".equals(req.getMethod()) || "PUT".equals(req.getMethod());
@@ -634,9 +635,7 @@ public class SolrDispatchFilter extends
boolean byCoreName, boolean activeReplicas) {
String coreUrl;
Set<String> liveNodes = clusterState.getLiveNodes();
- Iterator<Slice> it = slices.iterator();
- while (it.hasNext()) {
- Slice slice = it.next();
+ for (Slice slice : slices) {
Map<String,Replica> sliceShards = slice.getReplicasMap();
for (ZkNodeProps nodeProps : sliceShards.values()) {
ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
@@ -680,13 +679,12 @@ public class SolrDispatchFilter extends
}
return slices;
}
-
- private SolrCore getCoreByCollection(CoreContainer cores, String corename, String path) {
- String collection = corename;
+
+ private SolrCore getCoreByCollection(CoreContainer cores, String corename) {
ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();
ClusterState clusterState = zkStateReader.getClusterState();
- Map<String,Slice> slices = clusterState.getActiveSlicesMap(collection);
+ Map<String,Slice> slices = clusterState.getActiveSlicesMap(corename);
if (slices == null) {
return null;
}
@@ -696,9 +694,9 @@ public class SolrDispatchFilter extends
done:
for (Entry<String,Slice> entry : entries) {
// first see if we have the leader
- ZkNodeProps leaderProps = clusterState.getLeader(collection, entry.getKey());
+ ZkNodeProps leaderProps = clusterState.getLeader(corename, entry.getKey());
if (leaderProps != null) {
- core = checkProps(cores, path, leaderProps);
+ core = checkProps(cores, leaderProps);
}
if (core != null) {
break done;
@@ -709,7 +707,7 @@ public class SolrDispatchFilter extends
Set<Entry<String,Replica>> shardEntries = shards.entrySet();
for (Entry<String,Replica> shardEntry : shardEntries) {
Replica zkProps = shardEntry.getValue();
- core = checkProps(cores, path, zkProps);
+ core = checkProps(cores, zkProps);
if (core != null) {
break done;
}
@@ -718,8 +716,7 @@ public class SolrDispatchFilter extends
return core;
}
- private SolrCore checkProps(CoreContainer cores, String path,
- ZkNodeProps zkProps) {
+ private SolrCore checkProps(CoreContainer cores, ZkNodeProps zkProps) {
String corename;
SolrCore core = null;
if (cores.getZkController().getNodeName().equals(zkProps.getStr(ZkStateReader.NODE_NAME_PROP))) {
@@ -746,23 +743,27 @@ public class SolrDispatchFilter extends
private void writeResponse(SolrQueryResponse solrRsp, ServletResponse response,
QueryResponseWriter responseWriter, SolrQueryRequest solrReq, Method reqMethod)
throws IOException {
+ try {
+ // Now write it out
+ final String ct = responseWriter.getContentType(solrReq, solrRsp);
+ // don't call setContentType on null
+ if (null != ct) response.setContentType(ct);
- // Now write it out
- final String ct = responseWriter.getContentType(solrReq, solrRsp);
- // don't call setContentType on null
- if (null != ct) response.setContentType(ct);
-
- if (solrRsp.getException() != null) {
- NamedList info = new SimpleOrderedMap();
- int code = ResponseUtils.getErrorInfo(solrRsp.getException(), info, log);
- solrRsp.add("error", info);
- ((HttpServletResponse) response).setStatus(code);
+ if (solrRsp.getException() != null) {
+ NamedList info = new SimpleOrderedMap();
+ int code = ResponseUtils.getErrorInfo(solrRsp.getException(), info, log);
+ solrRsp.add("error", info);
+ ((HttpServletResponse) response).setStatus(code);
+ }
+
+ if (Method.HEAD != reqMethod) {
+ QueryResponseWriterUtil.writeQueryResponse(response.getOutputStream(), responseWriter, solrReq, solrRsp, ct);
+ }
+ //else http HEAD request, nothing to write out, waited this long just to get ContentType
}
-
- if (Method.HEAD != reqMethod) {
- QueryResponseWriterUtil.writeQueryResponse(response.getOutputStream(), responseWriter, solrReq, solrRsp, ct);
+ catch (EOFException e) {
+ log.info("Unable to write response, client closed connection or we are shutting down", e);
}
- //else http HEAD request, nothing to write out, waited this long just to get ContentType
}
protected void execute( HttpServletRequest req, SolrRequestHandler handler, SolrQueryRequest sreq, SolrQueryResponse rsp) {
@@ -773,7 +774,16 @@ public class SolrDispatchFilter extends
sreq.getCore().execute( handler, sreq, rsp );
}
- protected void sendError(SolrCore core,
+ private void sendError(HttpServletResponse response, int code, String message) throws IOException {
+ try {
+ response.sendError(code, message);
+ }
+ catch (EOFException e) {
+ log.info("Unable to write error response, client closed connection or we are shutting down", e);
+ }
+ }
+
+ protected void sendError(SolrCore core,
SolrQueryRequest req,
ServletRequest request,
HttpServletResponse response,
@@ -814,7 +824,7 @@ public class SolrDispatchFilter extends
if (exp != null) {
SimpleOrderedMap info = new SimpleOrderedMap();
int code = ResponseUtils.getErrorInfo(ex, info, log);
- response.sendError(code, info.toString());
+ sendError(response, code, info.toString());
}
} finally {
if (core == null && localCore != null) {
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/util/CommandOperation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/util/CommandOperation.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/util/CommandOperation.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/util/CommandOperation.java Thu Nov 27 20:59:47 2014
@@ -214,4 +214,15 @@ public class CommandOperation {
return new CommandOperation(name,commandData);
}
+ public Map getMap(String key, Map def) {
+ Object o =getMapVal(key);
+ if(o==null) return def;
+ if ( !(o instanceof Map)) {
+ addError(MessageFormat.format("''{0}'' must be a map", key));
+ return def;
+ } else {
+ return (Map) o;
+
+ }
+ }
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java Thu Nov 27 20:59:47 2014
@@ -19,15 +19,25 @@ package org.apache.solr.core;
import java.io.File;
+import java.io.IOException;
import java.io.StringReader;
-import java.util.List;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
import com.google.common.collect.ImmutableList;
import org.apache.commons.io.FileUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.util.EntityUtils;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.handler.TestSolrConfigHandlerConcurrent;
import org.apache.solr.util.RestTestBase;
import org.apache.solr.util.RestTestHarness;
import org.eclipse.jetty.servlet.ServletHolder;
@@ -77,13 +87,11 @@ public class TestSolrConfigHandler exten
public void testProperty() throws Exception{
+ RestTestHarness harness = restTestHarness;
String payload= "{\n" +
" 'set-property' : { 'updateHandler.autoCommit.maxDocs':100, 'updateHandler.autoCommit.maxTime':10 } \n" +
" }";
- RestTestHarness harness = restTestHarness;
- String response = harness.post("/config?wt=json", SolrTestCaseJ4.json(payload));
- Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ runConfigCommand( harness,"/config?wt=json", payload);
Map m = (Map) getRespMap("/config/overlay?wt=json" ,harness).get("overlay");
Map props = (Map) m.get("props");
@@ -99,9 +107,7 @@ public class TestSolrConfigHandler exten
payload= "{\n" +
" 'unset-property' : 'updateHandler.autoCommit.maxDocs'} \n" +
" }";
- response = harness.post("/config?wt=json", SolrTestCaseJ4.json(payload));
- map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ runConfigCommand(harness, "/config?wt=json", payload);
m = (Map) getRespMap("/config/overlay?wt=json" ,harness).get("overlay");
props = (Map) m.get("props");
@@ -111,14 +117,12 @@ public class TestSolrConfigHandler exten
}
public void testUserProp() throws Exception{
+ RestTestHarness harness = restTestHarness;
String payload= "{\n" +
" 'set-user-property' : { 'my.custom.variable.a':'MODIFIEDA'," +
" 'my.custom.variable.b':'MODIFIEDB' } \n" +
" }";
- RestTestHarness harness = restTestHarness;
- String response = harness.post("/config?wt=json", SolrTestCaseJ4.json(payload));
- Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertNull(response, map.get("errors"));
+ runConfigCommand(harness,"/config?wt=json", payload);
Map m = (Map) getRespMap("/config/overlay?wt=json" ,harness).get("overlay");
Map props = (Map) m.get("userProps");
@@ -132,14 +136,93 @@ public class TestSolrConfigHandler exten
assertEquals("MODIFIEDA", m.get("a"));
assertEquals("MODIFIEDB", m.get("b"));
+ }
+ public void testReqHandlerAPIs() throws Exception {
+ reqhandlertests(restTestHarness, null,null);
+ }
+ private static void runConfigCommand(RestTestHarness harness, String uri, String payload) throws IOException {
+ String response = harness.post(uri, SolrTestCaseJ4.json(payload));
+ Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ assertNull(response, map.get("errors"));
}
+ public static void reqhandlertests(RestTestHarness writeHarness,String testServerBaseUrl, CloudSolrServer cloudSolrServer) throws Exception {
+ String payload = "{\n" +
+ "'create-requesthandler' : { 'name' : '/x', 'class': 'org.apache.solr.handler.DumpRequestHandler' , 'startup' : 'lazy'}\n" +
+ "}";
+ runConfigCommand(writeHarness,"/config?wt=json", payload);
+
+ boolean success = false;
+ long startTime = System.nanoTime();
+ long maxTimeoutSeconds = 10;
+ while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
+ String uri = "/config/overlay?wt=json";
+ Map m = testServerBaseUrl ==null? getRespMap(uri,writeHarness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl+uri ,cloudSolrServer) ;
+ if("lazy".equals( ConfigOverlay.getObjectByPath(m, true, Arrays.asList("overlay", "requestHandler", "/x","startup")))) {
+ Map map = getRespMap("/x?wt=json",writeHarness);
+ if(map.containsKey("params")) {
+ success = true;
+ break;
+ }
+ }
+ Thread.sleep(100);
+
+ }
+
+ assertTrue( "Could not register requestHandler ", success);
+
+ payload = "{\n" +
+ "'update-requesthandler' : { 'name' : '/x', 'class': 'org.apache.solr.handler.DumpRequestHandler' , 'startup' : 'lazy' , 'a':'b'}\n" +
+ "}";
+ runConfigCommand(writeHarness,"/config?wt=json", payload);
+
+ success = false;
+ startTime = System.nanoTime();
+ maxTimeoutSeconds = 10;
+ while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
+ String uri = "/config/overlay?wt=json";
+ Map m = testServerBaseUrl ==null? getRespMap(uri,writeHarness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl+uri ,cloudSolrServer) ;
+ if("b".equals( ConfigOverlay.getObjectByPath(m, true, Arrays.asList("overlay", "requestHandler", "/x","a")))) {
+ success = true;
+ break;
+ }
+ Thread.sleep(100);
+
+ }
+
+ assertTrue( "Could not update requestHandler ", success);
+
+ payload = "{\n" +
+ "'delete-requesthandler' : '/x'" +
+ "}";
+ runConfigCommand(writeHarness,"/config?wt=json", payload);
+ success = false;
+ startTime = System.nanoTime();
+ maxTimeoutSeconds = 10;
+ while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
+ String uri = "/config/overlay?wt=json";
+ Map m = testServerBaseUrl ==null? getRespMap(uri,writeHarness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl+uri ,cloudSolrServer) ;
+ if(null == ConfigOverlay.getObjectByPath(m, true, Arrays.asList("overlay", "requestHandler", "/x","a"))) {
+ success = true;
+ break;
+ }
+ Thread.sleep(100);
+
+ }
+ assertTrue( "Could not delete requestHandler ", success);
+
+ }
+
public static Map getRespMap(String path, RestTestHarness restHarness) throws Exception {
String response = restHarness.query(path);
- return (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ try {
+ return (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ } catch (JSONParser.ParseException e) {
+ return Collections.emptyMap();
+ }
}
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java?rev=1642230&r1=1642229&r2=1642230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java Thu Nov 27 20:59:47 2014
@@ -33,6 +33,7 @@ import org.apache.http.client.methods.Ht
import org.apache.http.util.EntityUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.cloud.DocCollection;
@@ -164,7 +165,7 @@ public class TestSolrConfigHandlerConcur
while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
Thread.sleep(100);
errmessages.clear();
- Map respMap = getAsMap(url+"/config/overlay?wt=json");
+ Map respMap = getAsMap(url+"/config/overlay?wt=json", cloudClient);
Map m = (Map) respMap.get("overlay");
if(m!= null) m = (Map) m.get("props");
if(m == null) {
@@ -191,7 +192,7 @@ public class TestSolrConfigHandlerConcur
}
- private Map getAsMap(String uri) throws Exception {
+ public static Map getAsMap(String uri, CloudSolrServer cloudClient) throws Exception {
HttpGet get = new HttpGet(uri) ;
HttpEntity entity = null;
try {