You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/07/12 18:40:44 UTC
[lucene-solr] branch reference_impl updated: #95 Knock down some
tests, close a SolrCore leak.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/reference_impl by this push:
new e5cfb2f #95 Knock down some tests, close a SolrCore leak.
e5cfb2f is described below
commit e5cfb2f32259931e9b6429c5e8f87bd76d0605a8
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Sun Jul 12 13:40:09 2020 -0500
#95 Knock down some tests, close a SolrCore leak.
---
.../org/apache/solr/ltr/TestLTROnSolrCloud.java | 8 ++-
.../solr/handler/admin/CoreAdminOperation.java | 7 ++-
.../org/apache/solr/TestDistributedGrouping.java | 25 +++++----
.../org/apache/solr/cloud/TestCloudRecovery2.java | 2 +
.../cloud/TestDynamicFieldNamesIndexCorrectly.java | 2 +-
.../solr/cloud/TestRandomRequestDistribution.java | 9 +---
.../TestTolerantUpdateProcessorRandomCloud.java | 6 +--
...stributedFacetSimpleRefinementLongTailTest.java | 2 +
.../embedded/SolrExampleStreamingBinaryTest.java | 2 +
.../apache/solr/BaseDistributedSearchTestCase.java | 62 ++++++++++++++--------
10 files changed, 73 insertions(+), 52 deletions(-)
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
index 9000113..0167c55 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
@@ -51,15 +51,13 @@ public class TestLTROnSolrCloud extends TestRerankBase {
extraServlets = setupTestInit(solrconfig, schema, true);
System.setProperty("enable.update.log", "true");
- int numberOfShards = random().nextInt(TEST_NIGHTLY ? 4 : 2)+1;
- int numberOfReplicas = random().nextInt(TEST_NIGHTLY ? 2 : 1)+1;
- int maxShardsPerNode = random().nextInt(4)+1;
+ int numberOfShards = TEST_NIGHTLY ? random().nextInt(TEST_NIGHTLY ? 4 : 2)+1 : 2;
+ int numberOfReplicas = TEST_NIGHTLY ? random().nextInt(TEST_NIGHTLY ? 2 : 1)+1 : 2;
+ int maxShardsPerNode = TEST_NIGHTLY ? random().nextInt(4)+1 : 4;
int numberOfNodes = (numberOfShards*numberOfReplicas + (maxShardsPerNode-1))/maxShardsPerNode;
setupSolrCluster(numberOfShards, numberOfReplicas, numberOfNodes, maxShardsPerNode);
-
-
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
index 3036ced..3f963c4 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
@@ -26,6 +26,7 @@ import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkController;
+import org.apache.solr.common.AlreadyClosedException;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CoreAdminParams;
@@ -91,6 +92,10 @@ enum CoreAdminOperation implements CoreAdminOp {
}
boolean newCollection = params.getBool(CoreAdminParams.NEW_COLLECTION, false);
+ if (coreContainer.isShutDown()) {
+ log().warn("Will not create SolrCore, CoreContainer is shutdown");
+ throw new AlreadyClosedException("Will not create SolrCore, CoreContainer is shutdown");
+ }
coreContainer.create(coreName, instancePath, coreParams, newCollection);
@@ -288,8 +293,6 @@ enum CoreAdminOperation implements CoreAdminOp {
}
-
-
/**
* Returns the core status for a particular core.
* @param cores - the enclosing core container
diff --git a/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java b/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
index 7763658..d25b67c 100644
--- a/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
+++ b/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
@@ -49,6 +49,9 @@ public class TestDistributedGrouping extends BaseDistributedSearchTestCase {
public TestDistributedGrouping() {
// SOLR-10844: Even with points suppressed, this test breaks if we (randomize) docvalues="true" on trie fields?!?!?!!?
System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"false");
+ if (!TEST_NIGHTLY) {
+ fixShardCount(2);
+ }
}
String t1="a_t";
@@ -242,8 +245,8 @@ public class TestDistributedGrouping extends BaseDistributedSearchTestCase {
query("q", "*:*", "fl", "id," + i1dv, "group", "true", "group.field", i1dv, "group.limit", 10, "sort", i1 + " asc, id asc");
-
- // SOLR-4150: what if group.query has no matches,
+
+ // SOLR-4150: what if group.query has no matches,
// or only matches on one shard
query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true",
"group.query", t1 + ":kings OR " + t1 + ":eggs",
@@ -256,13 +259,13 @@ public class TestDistributedGrouping extends BaseDistributedSearchTestCase {
// SOLR-4164: main query matches nothing, or only matches on one shard
query("q", "bogus_s:nothing", // no docs match
- "group", "true",
+ "group", "true",
"group.query", t1 + ":this_will_never_match",
- "group.field", i1,
+ "group.field", i1,
"fl", "id", "group.limit", "2", "group.format", "simple");
query("q", "id:5", // one doc matches, so only one shard
- "rows", 100, "fl", "id," + i1, "group", "true",
- "group.query", t1 + ":kings OR " + t1 + ":eggs",
+ "rows", 100, "fl", "id," + i1, "group", "true",
+ "group.query", t1 + ":kings OR " + t1 + ":eggs",
"group.field", i1,
"group.limit", 10, "sort", i1 + " asc, id asc");
@@ -339,9 +342,9 @@ public class TestDistributedGrouping extends BaseDistributedSearchTestCase {
// SOLR-3960 - include a postfilter
for (String facet : new String[] { "false", "true"}) {
for (String fcache : new String[] { "", " cache=false cost=200"}) {
- query("q", "*:*", "rows", 100, "fl", "id," + i1,
+ query("q", "*:*", "rows", 100, "fl", "id," + i1,
"group.limit", 10, "sort", i1 + " asc, id asc",
- "group", "true", "group.field", i1,
+ "group", "true", "group.field", i1,
"fq", "{!frange l=50 "+fcache+"}"+tlong,
"facet.field", t1,
"facet", facet
@@ -538,7 +541,7 @@ public class TestDistributedGrouping extends BaseDistributedSearchTestCase {
assertTrue(docs.toString(), 8 <= docs.get(4).getFieldNames().size());
}
}
-
+
// grouping on boolean non-stored docValued enabled field
rsp = query("q", b1dv + ":*", "fl", "id," + b1dv, "group", "true", "group.field",
b1dv, "group.limit", 10, "sort", b1dv + " asc, id asc");
@@ -550,10 +553,10 @@ public class TestDistributedGrouping extends BaseDistributedSearchTestCase {
assertEquals(rsp.toString(), false, nl.get("groupValue"));
SolrDocumentList docs = (SolrDocumentList) nl.get("doclist");
assertEquals(docs.toString(), 4, docs.getNumFound());
-
+
// Can't validate the response, but can check if no errors occur.
simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.query", t1 + ":kings OR " + t1 + ":eggs", "group.limit", 10, "sort", i1 + " asc, id asc", CommonParams.TIME_ALLOWED, 1);
-
+
//Debug
simpleQuery("q", "*:*", "rows", 10, "fl", "id," + i1, "group", "true", "group.field", i1, "debug", "true");
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery2.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery2.java
index 6ac85e9..64057e8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery2.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery2.java
@@ -26,10 +26,12 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.cloud.Replica;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+@Ignore // nocommit flakey
public class TestCloudRecovery2 extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String COLLECTION = "collection1";
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestDynamicFieldNamesIndexCorrectly.java b/solr/core/src/test/org/apache/solr/cloud/TestDynamicFieldNamesIndexCorrectly.java
index d483253..6226c48 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestDynamicFieldNamesIndexCorrectly.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestDynamicFieldNamesIndexCorrectly.java
@@ -56,7 +56,7 @@ public class TestDynamicFieldNamesIndexCorrectly extends AbstractFullDistribZkTe
@BaseDistributedSearchTestCase.ShardsFixed(num = 3)
public void test() throws Exception {
createCollection(COLLECTION, "_default", 4, 1, 4);
- final int numRuns = 10;
+ final int numRuns = TEST_NIGHTLY ? 10 : 2;
populateIndex(numRuns);
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java b/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java
index 3c88a90..37a134e 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java
@@ -82,11 +82,6 @@ public class TestRandomRequestDistribution extends AbstractFullDistribZkTestBase
.setCreateNodeSet(nodeNames.get(2))
.process(cloudClient);
- waitForRecoveriesToFinish("a1x2", true);
- waitForRecoveriesToFinish("b1x1", true);
-
- cloudClient.getZkStateReader().forceUpdateCollection("b1x1");
-
// get direct access to the metrics counters for each core/replica we're interested to monitor them
final Map<String,Counter> counters = new LinkedHashMap<>();
for (JettySolrRunner runner : jettys) {
@@ -224,7 +219,7 @@ public class TestRandomRequestDistribution extends AbstractFullDistribZkTestBase
if (c == 1) {
break; // cluster state has got update locally
} else {
- Thread.sleep(100);
+ Thread.sleep(10);
}
if (count > 10000) {
@@ -234,7 +229,7 @@ public class TestRandomRequestDistribution extends AbstractFullDistribZkTestBase
// Now we fire a few additional queries and make sure ALL of them
// are served by the active replica
- int moreQueries = TestUtil.nextInt(random(), 4, 10);
+ int moreQueries = TEST_NIGHTLY ? TestUtil.nextInt(random(), 4, 10) : 2;
count = 1; // Since 1 query has already hit the leader
for (int i = 0; i < moreQueries; i++) {
client.query(new SolrQuery("*:*"));
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java
index bee8a67..6a270dc 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java
@@ -86,7 +86,7 @@ public class TestTolerantUpdateProcessorRandomCloud extends SolrCloudTestCase {
final String configName = "solrCloudCollectionConfig";
final File configDir = new File(TEST_HOME() + File.separator + "collection1" + File.separator + "conf");
- final int numShards = TestUtil.nextInt(random(), 2, TEST_NIGHTLY ? 5 : 3);
+ final int numShards = TEST_NIGHTLY ? TestUtil.nextInt(random(), 2, 5) : 2;
final int repFactor = TestUtil.nextInt(random(), 2, TEST_NIGHTLY ? 5 : 3);
// at least one server won't have any replicas
final int numServers = 1 + (numShards * repFactor);
@@ -143,10 +143,10 @@ public class TestTolerantUpdateProcessorRandomCloud extends SolrCloudTestCase {
}
public void testRandomUpdates() throws Exception {
- final int maxDocId = atLeast(10000);
+ final int maxDocId = atLeast(TEST_NIGHTLY ? 10000 : 1000);
final BitSet expectedDocIds = new BitSet(maxDocId+1);
- final int numIters = atLeast(50);
+ final int numIters = atLeast(TEST_NIGHTLY ? 50 : 15);
for (int i = 0; i < numIters; i++) {
log.info("BEGIN ITER #{}", i);
diff --git a/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java b/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java
index 744693a..88e482f 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java
@@ -20,6 +20,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.common.params.SolrParams;
@@ -40,6 +41,7 @@ import org.junit.Test;
* <code>facet.pivot</code> so the assertions in this test vary from that test.
* </p>
*/
+@LuceneTestCase.Nightly
public class DistributedFacetSimpleRefinementLongTailTest extends BaseDistributedSearchTestCase {
private static List<String> ALL_STATS = Arrays.asList("min", "max", "sum", "stddev", "avg", "sumsq", "unique",
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryTest.java
index bcab567..1a69b0d 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryTest.java
@@ -30,10 +30,12 @@ import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
+import org.junit.Ignore;
import org.junit.Test;
@Slow
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
+@Ignore // nocommit flakey
public class SolrExampleStreamingBinaryTest extends SolrExampleStreamingTest {
@Override
diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
index 0975bda..847f1d2 100644
--- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
@@ -344,30 +344,46 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
protected void createServers(int numShards) throws Exception {
System.setProperty("configSetBaseDir", getSolrHome());
-
- controlJetty = createControlJetty();
- controlClient = createNewSolrClient(controlJetty.getLocalPort());
-
- shardsArr = new String[numShards];
StringBuilder sb = new StringBuilder();
- for (int i = 0; i < numShards; i++) {
- if (sb.length() > 0) sb.append(',');
- final String shardname = "shard" + i;
- Path jettyHome = testDir.toPath().resolve(shardname);
- File jettyHomeFile = jettyHome.toFile();
- seedSolrHome(jettyHomeFile);
- seedCoreRootDirWithDefaultTestCore(jettyHome.resolve("cores"));
- JettySolrRunner j = createJetty(jettyHomeFile, null, null, getSolrConfigFile(), getSchemaFile());
- j.start();
- jettys.add(j);
- clients.add(createNewSolrClient(j.getLocalPort()));
- String shardStr = buildUrl(j.getLocalPort());
-
- if (shardStr.endsWith("/")) shardStr += DEFAULT_TEST_CORENAME;
- else shardStr += "/" + DEFAULT_TEST_CORENAME;
-
- shardsArr[i] = shardStr;
- sb.append(shardStr);
+ try (ParWork worker = new ParWork(this)) {
+ worker.collect(() -> {
+ try {
+ controlJetty = createControlJetty();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ controlClient = createNewSolrClient(controlJetty.getLocalPort());
+ });
+ shardsArr = new String[numShards];
+ for (int i = 0; i < numShards; i++) {
+ int finalI = i;
+ worker.collect(() -> {
+ if (sb.length() > 0) sb.append(',');
+ final String shardname = "shard" + finalI;
+ Path jettyHome = testDir.toPath().resolve(shardname);
+ File jettyHomeFile = jettyHome.toFile();
+ try {
+ seedSolrHome(jettyHomeFile);
+
+ seedCoreRootDirWithDefaultTestCore(jettyHome.resolve("cores"));
+ JettySolrRunner j = createJetty(jettyHomeFile, null, null, getSolrConfigFile(), getSchemaFile());
+ j.start();
+ jettys.add(j);
+ clients.add(createNewSolrClient(j.getLocalPort()));
+ String shardStr = buildUrl(j.getLocalPort());
+
+ if (shardStr.endsWith("/")) shardStr += DEFAULT_TEST_CORENAME;
+ else shardStr += "/" + DEFAULT_TEST_CORENAME;
+
+ shardsArr[finalI] = shardStr;
+ sb.append(shardStr);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ worker.addCollect("startJettys");
+ }
+
}
shards = sb.toString();