You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/07/10 04:54:21 UTC
[lucene-solr] branch reference_impl updated: #46 Knock down a few
long tail tests.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/reference_impl by this push:
new a917b6c #46 Knock down a few long tail tests.
a917b6c is described below
commit a917b6c957f77bdba9f85dbd4310228b73cbc561
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Jul 9 23:54:02 2020 -0500
#46 Knock down a few long tail tests.
---
.../apache/solr/cloud/DocValuesNotIndexedTest.java | 2 -
.../org/apache/solr/cloud/TestSegmentSorting.java | 2 +-
.../cloud/TestStressCloudBlindAtomicUpdates.java | 5 +-
...ava => CollectionsAPIDistClusterPerZkTest.java} | 153 ++----
.../CollectionsAPIDistributedZkTest.java | 529 +--------------------
.../HdfsCollectionsAPIDistributedZkTest.java | 4 +-
.../src/test/org/apache/solr/pkg/TestPackages.java | 2 +
.../search/facet/TestCloudJSONFacetJoinDomain.java | 23 +-
.../apache/solr/search/facet/TestJsonFacets.java | 6 +-
9 files changed, 61 insertions(+), 665 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java b/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
index 793907d..b6252fa 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
@@ -88,8 +88,6 @@ public class DocValuesNotIndexedTest extends SolrCloudTestCase {
CollectionAdminRequest.createCollection(COLLECTION, "conf1", 4, 1)
.setMaxShardsPerNode(2)
.process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(COLLECTION, 4, 4);
fieldsToTestSingle =
Collections.unmodifiableList(Arrays.asList(
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java
index 3551394..c36bd28 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java
@@ -80,7 +80,7 @@ public class TestSegmentSorting extends SolrCloudTestCase {
.setProperties(collectionProperties);
assertTrue( cmd.process(cloudSolrClient).isSuccess() );
-
+
cloudSolrClient.setDefaultCollection(collectionName);
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java b/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java
index 15d7c46..2d7b33a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java
@@ -134,8 +134,6 @@ public class TestStressCloudBlindAtomicUpdates extends SolrCloudTestCase {
.withProperty("schema", "schema-minimal-atomic-stress.xml")
.process(CLOUD_CLIENT);
- waitForRecoveriesToFinish(CLOUD_CLIENT);
-
CLIENTS.clear();
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
assertNotNull("Cluster contains null jetty?", jetty);
@@ -178,7 +176,6 @@ public class TestStressCloudBlindAtomicUpdates extends SolrCloudTestCase {
@Before
private void clearCloudCollection() throws Exception {
TestInjection.reset();
- waitForRecoveriesToFinish(CLOUD_CLIENT);
assertEquals(0, CLOUD_CLIENT.deleteByQuery("*:*").getStatus());
assertEquals(0, CLOUD_CLIENT.optimize().getStatus());
@@ -266,7 +263,7 @@ public class TestStressCloudBlindAtomicUpdates extends SolrCloudTestCase {
final CountDownLatch abortLatch = new CountDownLatch(1);
- final int numDocsToCheck = atLeast(37);
+ final int numDocsToCheck = atLeast(TEST_NIGHTLY ? 37 : 7);
final int numDocsInIndex = (numDocsToCheck * DOC_ID_INCR);
final AtomicLong[] expected = new AtomicLong[numDocsToCheck];
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistClusterPerZkTest.java
similarity index 86%
copy from solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
copy to solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistClusterPerZkTest.java
index 61ce71b..1465b91 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistClusterPerZkTest.java
@@ -16,30 +16,7 @@
*/
package org.apache.solr.cloud.api.collections;
-import javax.management.MBeanServer;
-import javax.management.MBeanServerFactory;
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.lang.management.ManagementFactory;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
import com.google.common.collect.ImmutableList;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.client.solrj.SolrQuery;
@@ -75,7 +52,7 @@ import org.apache.solr.core.SolrInfoBean.Category;
import org.apache.solr.util.TestInjection;
import org.apache.solr.util.TimeOut;
import org.junit.After;
-import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
@@ -83,21 +60,42 @@ import org.slf4j.LoggerFactory;
import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
+import javax.management.MBeanServer;
+import javax.management.MBeanServerFactory;
+import javax.management.ObjectName;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.lang.management.ManagementFactory;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
/**
* Tests the Cloud Collections API.
*/
@Slow
//@LuceneTestCase.Nightly // nocommit speed up, though prob requires overseer perf boost
-public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase {
+public class CollectionsAPIDistClusterPerZkTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- protected String getConfigSet() {
- return "cloud-minimal";
+ protected static volatile String configSet = "cloud-minimal";
+ protected static String getConfigSet() {
+ return configSet;
}
- @Before
- public void setupCluster() throws Exception {
+ @BeforeClass
+ public static void setupCluster() throws Exception {
// we don't want this test to have zk timeouts
System.setProperty("zkClientTimeout", "60000");
if (TEST_NIGHTLY) {
@@ -117,70 +115,7 @@ public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase {
@After
public void tearDownCluster() throws Exception {
- try {
- shutdownCluster();
- } finally {
- System.clearProperty("createCollectionWaitTimeTillActive");
- super.tearDown();
- }
- }
-
- @Test
- @Ignore // nocommit debug
- public void testCreationAndDeletion() throws Exception {
- String collectionName = "created_and_deleted";
-
- CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1).process(cluster.getSolrClient());
- assertTrue(CollectionAdminRequest.listCollections(cluster.getSolrClient())
- .contains(collectionName));
-
- CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
- assertFalse(CollectionAdminRequest.listCollections(cluster.getSolrClient())
- .contains(collectionName));
-
- assertFalse(cluster.getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true));
- }
-
- @Test
- public void deleteCollectionRemovesStaleZkCollectionsNode() throws Exception {
- String collectionName = "out_of_sync_collection";
-
- // manually create a collections zknode
- cluster.getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true);
-
- CollectionAdminRequest.deleteCollection(collectionName)
- .process(cluster.getSolrClient());
-
- assertFalse(CollectionAdminRequest.listCollections(cluster.getSolrClient())
- .contains(collectionName));
-
- assertFalse(cluster.getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true));
- }
-
- @Test
- @Ignore // nocommit - this can be faster
- public void deletePartiallyCreatedCollection() throws Exception {
- final String collectionName = "halfdeletedcollection";
-
- assertEquals(0, CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1)
- .setCreateNodeSet("")
- .process(cluster.getSolrClient()).getStatus());
- String dataDir = createTempDir().toFile().getAbsolutePath();
- // create a core that simulates something left over from a partially-deleted collection
- assertTrue(CollectionAdminRequest
- .addReplicaToShard(collectionName, "shard1")
- .setDataDir(dataDir)
- .process(cluster.getSolrClient()).isSuccess());
-
- CollectionAdminRequest.deleteCollection(collectionName)
- .process(cluster.getSolrClient());
-
- assertFalse(CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName));
-
- CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1)
- .process(cluster.getSolrClient());
-
- assertTrue(CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName));
+ cluster.deleteAllCollections();
}
@Test
@@ -201,36 +136,6 @@ public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase {
}
@Test
- public void testBadActionNames() {
- // try a bad action
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", "BADACTION");
- String collectionName = "badactioncollection";
- params.set("name", collectionName);
- params.set("numShards", 2);
- final QueryRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- expectThrows(Exception.class, () -> {
- cluster.getSolrClient().request(request);
- });
- }
-
- @Test
- public void testMissingRequiredParameters() {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", CollectionAction.CREATE.toString());
- params.set("numShards", 2);
- // missing required collection parameter
- final SolrRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- expectThrows(Exception.class, () -> {
- cluster.getSolrClient().request(request);
- });
- }
-
- @Test
public void testTooManyReplicas() {
CollectionAdminRequest req = CollectionAdminRequest.createCollection("collection", "conf", 2, 10);
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
index 61ce71b..eb3d4ca 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
@@ -76,6 +76,7 @@ import org.apache.solr.util.TestInjection;
import org.apache.solr.util.TimeOut;
import org.junit.After;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
@@ -87,13 +88,11 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
/**
* Tests the Cloud Collections API.
*/
-@Slow
-//@LuceneTestCase.Nightly // nocommit speed up, though prob requires overseer perf boost
public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- protected String getConfigSet() {
- return "cloud-minimal";
+ protected static volatile String configSet = "cloud-minimal";
+ protected static String getConfigSet() {
+ return configSet;
}
@Before
@@ -117,28 +116,7 @@ public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase {
@After
public void tearDownCluster() throws Exception {
- try {
- shutdownCluster();
- } finally {
- System.clearProperty("createCollectionWaitTimeTillActive");
- super.tearDown();
- }
- }
-
- @Test
- @Ignore // nocommit debug
- public void testCreationAndDeletion() throws Exception {
- String collectionName = "created_and_deleted";
-
- CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1).process(cluster.getSolrClient());
- assertTrue(CollectionAdminRequest.listCollections(cluster.getSolrClient())
- .contains(collectionName));
-
- CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
- assertFalse(CollectionAdminRequest.listCollections(cluster.getSolrClient())
- .contains(collectionName));
-
- assertFalse(cluster.getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true));
+ cluster.shutdown();
}
@Test
@@ -153,51 +131,8 @@ public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase {
assertFalse(CollectionAdminRequest.listCollections(cluster.getSolrClient())
.contains(collectionName));
-
- assertFalse(cluster.getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true));
- }
-
- @Test
- @Ignore // nocommit - this can be faster
- public void deletePartiallyCreatedCollection() throws Exception {
- final String collectionName = "halfdeletedcollection";
-
- assertEquals(0, CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1)
- .setCreateNodeSet("")
- .process(cluster.getSolrClient()).getStatus());
- String dataDir = createTempDir().toFile().getAbsolutePath();
- // create a core that simulates something left over from a partially-deleted collection
- assertTrue(CollectionAdminRequest
- .addReplicaToShard(collectionName, "shard1")
- .setDataDir(dataDir)
- .process(cluster.getSolrClient()).isSuccess());
-
- CollectionAdminRequest.deleteCollection(collectionName)
- .process(cluster.getSolrClient());
-
- assertFalse(CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName));
-
- CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1)
- .process(cluster.getSolrClient());
- assertTrue(CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName));
- }
-
- @Test
- public void deleteCollectionOnlyInZk() throws Exception {
- final String collectionName = "onlyinzk";
-
- // create the collections node, but nothing else
- cluster.getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true);
-
- // delete via API - should remove collections node
- CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
- assertFalse(CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName));
-
- // now creating that collection should work
- CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1)
- .process(cluster.getSolrClient());
- assertTrue(CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName));
+ assertFalse(cluster.getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true));
}
@Test
@@ -230,456 +165,4 @@ public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase {
});
}
- @Test
- public void testTooManyReplicas() {
- CollectionAdminRequest req = CollectionAdminRequest.createCollection("collection", "conf", 2, 10);
-
- expectThrows(Exception.class, () -> {
- cluster.getSolrClient().request(req);
- });
- }
-
- @Test
- public void testMissingNumShards() {
- // No numShards should fail
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", CollectionAction.CREATE.toString());
- params.set("name", "acollection");
- params.set(REPLICATION_FACTOR, 10);
- params.set("collection.configName", "conf");
-
- final SolrRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- expectThrows(Exception.class, () -> {
- cluster.getSolrClient().request(request);
- });
- }
-
- @Test
- public void testZeroNumShards() {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", CollectionAction.CREATE.toString());
- params.set("name", "acollection");
- params.set(REPLICATION_FACTOR, 10);
- params.set("numShards", 0);
- params.set("collection.configName", "conf");
-
- final SolrRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
- expectThrows(Exception.class, () -> {
- cluster.getSolrClient().request(request);
- });
- }
-
- @Test
- @Ignore // nocommit we can speed this up
- public void testCreateShouldFailOnExistingCore() throws Exception {
- assertEquals(0, CollectionAdminRequest.createCollection("halfcollectionblocker", "conf", 1, 1)
- .setCreateNodeSet("")
- .process(cluster.getSolrClient()).getStatus());
- assertTrue(CollectionAdminRequest.addReplicaToShard("halfcollectionblocker", "shard1")
- .setNode(cluster.getJettySolrRunner(0).getNodeName())
- .setCoreName("halfcollection_shard1_replica_n1")
- .process(cluster.getSolrClient()).isSuccess());
-
- assertEquals(0, CollectionAdminRequest.createCollection("halfcollectionblocker2", "conf",1, 1)
- .setCreateNodeSet("")
- .process(cluster.getSolrClient()).getStatus());
- assertTrue(CollectionAdminRequest.addReplicaToShard("halfcollectionblocker2", "shard1")
- .setNode(cluster.getJettySolrRunner(1).getNodeName())
- .setCoreName("halfcollection_shard1_replica_n1")
- .process(cluster.getSolrClient()).isSuccess());
-
- String nn1 = cluster.getJettySolrRunner(0).getNodeName();
- String nn2 = cluster.getJettySolrRunner(1).getNodeName();
-
- expectThrows(BaseHttpSolrClient.RemoteSolrException.class, () -> {
- CollectionAdminResponse resp = CollectionAdminRequest.createCollection("halfcollection", "conf", 2, 1)
- .setCreateNodeSet(nn1 + "," + nn2)
- .process(cluster.getSolrClient());
- });
- }
-
- @Test
- public void testNoConfigSetExist() throws Exception {
- expectThrows(Exception.class, () -> {
- CollectionAdminRequest.createCollection("noconfig", "conf123", 1, 1)
- .process(cluster.getSolrClient());
- });
-
- // in both cases, the collection should have default to the core name
- //cluster.getSolrClient().getZkStateReader().forceUpdateCollection("noconfig");
- assertFalse(CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains("noconfig"));
- }
-
- @Test
- public void testCoresAreDistributedAcrossNodes() throws Exception {
- CollectionAdminRequest.createCollection("nodes_used_collection", "conf", 2, 2)
- .process(cluster.getSolrClient());
-
- Set<String> liveNodes = cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes();
-
- List<String> createNodeList = new ArrayList<>(liveNodes);
-
- DocCollection collection = getCollectionState("nodes_used_collection");
- for (Slice slice : collection.getSlices()) {
- for (Replica replica : slice.getReplicas()) {
- createNodeList.remove(replica.getNodeName());
- }
- }
-
- assertEquals(createNodeList.toString(), 0, createNodeList.size());
- }
-
- @Test
- public void testDeleteNonExistentCollection() throws Exception {
-
- expectThrows(SolrException.class, () -> {
- CollectionAdminRequest.deleteCollection("unknown_collection").process(cluster.getSolrClient());
- });
-
- // create another collection should still work
- CollectionAdminRequest.createCollection("acollectionafterbaddelete", "conf", 1, 2)
- .process(cluster.getSolrClient());
- waitForState("Collection creation after a bad delete failed", "acollectionafterbaddelete",
- (n, c) -> DocCollection.isFullyActive(n, c, 1, 2));
- }
-
- @Test
- @Ignore // nocommit slow
- public void testSpecificConfigsets() throws Exception {
- CollectionAdminRequest.createCollection("withconfigset2", "conf2", 1, 1).process(cluster.getSolrClient());
- byte[] data = zkClient().getData(ZkStateReader.COLLECTIONS_ZKNODE + "/" + "withconfigset2", null, null, true);
- assertNotNull(data);
- ZkNodeProps props = ZkNodeProps.load(data);
- String configName = props.getStr(ZkController.CONFIGNAME_PROP);
- assertEquals("conf2", configName);
- }
-
- @Test
- public void testMaxNodesPerShard() {
- int numLiveNodes = cluster.getJettySolrRunners().size();
- int numShards = (numLiveNodes/2) + 1;
- int replicationFactor = 2;
-
- expectThrows(SolrException.class, () -> {
- CollectionAdminRequest.createCollection("oversharded", "conf", numShards, replicationFactor)
- .process(cluster.getSolrClient());
- });
- }
-
- @Test
- @Ignore // nocommit debug seems to random fail
- public void testCreateNodeSet() throws Exception {
- JettySolrRunner jetty1 = cluster.getRandomJetty(random());
- JettySolrRunner jetty2 = cluster.getRandomJetty(random());
-
- List<String> baseUrls = ImmutableList.of(jetty1.getCoreContainer().getZkController().getNodeName(), jetty2.getCoreContainer().getZkController().getNodeName());
-
- CollectionAdminRequest.createCollection("nodeset_collection", "conf", 2, 1)
- .setCreateNodeSet(baseUrls.get(0) + "," + baseUrls.get(1))
- .process(cluster.getSolrClient());
-
- DocCollection collectionState = getCollectionState("nodeset_collection");
- for (Replica replica : collectionState.getReplicas()) {
- String node = replica.getNodeName();
- boolean matchingJetty = false;
- for (String jettyNode : baseUrls) {
- if (node.equals(jettyNode)) {
- matchingJetty = true;
- }
- }
- if (matchingJetty == false) {
- fail("Expected replica to be on " + baseUrls + " but was on " + node);
- }
- }
- }
-
- @Test
- //28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
- // See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally
- // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018 SOLR-12028
- public void testCollectionsAPI() throws Exception {
-
- // create new collections rapid fire
- int cnt = random().nextInt(TEST_NIGHTLY ? 3 : 1) + 1;
- CollectionAdminRequest.Create[] createRequests = new CollectionAdminRequest.Create[cnt];
-
- class Coll {
- String name;
- int numShards;
- int replicationFactor;
- }
-
- List<Coll> colls = new ArrayList<>();
-
- for (int i = 0; i < cnt; i++) {
-
- int numShards = TestUtil.nextInt(random(), 0, cluster.getJettySolrRunners().size()) + 1;
- int replicationFactor = TestUtil.nextInt(random(), 0, 3) + 1;
- int maxShardsPerNode = (((numShards * replicationFactor) / cluster.getJettySolrRunners().size())) + 1;
-
- createRequests[i]
- = CollectionAdminRequest.createCollection("awhollynewcollection_" + i, "conf2", numShards, replicationFactor)
- .setMaxShardsPerNode(maxShardsPerNode);
- createRequests[i].processAsync(cluster.getSolrClient());
-
- Coll coll = new Coll();
- coll.name = "awhollynewcollection_" + i;
- coll.numShards = numShards;
- coll.replicationFactor = replicationFactor;
- colls.add(coll);
- }
-
- for (Coll coll : colls) {
- cluster.waitForActiveCollection(coll.name, coll.numShards, coll.numShards * coll.replicationFactor);
- }
-
- waitForStable(cnt, createRequests);
-
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- checkInstanceDirs(cluster.getJettySolrRunner(i));
- }
-
- String collectionName = createRequests[random().nextInt(createRequests.length)].getCollectionName();
-
- new UpdateRequest()
- .add("id", "6")
- .add("id", "7")
- .add("id", "8")
- .commit(cluster.getSolrClient(), collectionName);
- long numFound = 0;
- TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while (!timeOut.hasTimedOut()) {
-
- numFound = cluster.getSolrClient().query(collectionName, new SolrQuery("*:*")).getResults().getNumFound();
- if (numFound == 3) {
- break;
- }
-
- Thread.sleep(500);
- }
-
- if (timeOut.hasTimedOut()) {
- fail("Timeout waiting to see 3 found, instead saw " + numFound + " for collection " + collectionName);
- }
-
- // checkNoTwoShardsUseTheSameIndexDir();
- }
-
- private void waitForStable(int cnt, CollectionAdminRequest.Create[] createRequests) throws InterruptedException {
- for (int i = 0; i < cnt; i++) {
- String collectionName = "awhollynewcollection_" + i;
- final int j = i;
- waitForState("Expected to see collection " + collectionName, collectionName,
- (n, c) -> {
- CollectionAdminRequest.Create req = createRequests[j];
- return DocCollection.isFullyActive(n, c, req.getNumShards(), req.getReplicationFactor());
- });
-
- ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
- // make sure we have leaders for each shard
- for (int z = 1; z < createRequests[j].getNumShards(); z++) {
- zkStateReader.getLeaderRetry(collectionName, "shard" + z, 10000);
- } // make sure we again have leaders for each shard
- }
- }
-
- @Test
- @Ignore // nocommit have to fix reload again, ug, its a pain, I don't recall the exact incantation
- public void testCollectionReload() throws Exception {
- final String collectionName = "reloaded_collection";
- CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2).process(cluster.getSolrClient());
-
- // get core open times
- Map<String, Long> urlToTimeBefore = new HashMap<>();
- collectStartTimes(collectionName, urlToTimeBefore);
- assertTrue(urlToTimeBefore.size() > 0);
-
- Thread.sleep(200);
-
- CollectionAdminRequest.reloadCollection(collectionName).processAsync(cluster.getSolrClient());
-
- // reloads make take a short while
- boolean allTimesAreCorrect = waitForReloads(collectionName, urlToTimeBefore);
- assertTrue("some core start times did not change on reload", allTimesAreCorrect);
- }
-
- private void checkInstanceDirs(JettySolrRunner jetty) throws IOException {
- CoreContainer cores = jetty.getCoreContainer();
- Collection<SolrCore> theCores = cores.getCores();
- for (SolrCore core : theCores) {
- // look for core props file
- Path instancedir = core.getInstancePath();
- assertTrue("Could not find expected core.properties file", Files.exists(instancedir.resolve("core.properties")));
-
- Path expected = Paths.get(jetty.getSolrHome()).toAbsolutePath().resolve(core.getName());
-
- assertTrue("Expected: " + expected + "\nFrom core stats: " + instancedir, Files.isSameFile(expected, instancedir));
- }
- }
-
- private boolean waitForReloads(String collectionName, Map<String,Long> urlToTimeBefore) throws SolrServerException, IOException {
- TimeOut timeout = new TimeOut(45, TimeUnit.SECONDS, TimeSource.NANO_TIME);
-
- boolean allTimesAreCorrect = false;
- while (! timeout.hasTimedOut()) {
- Map<String,Long> urlToTimeAfter = new HashMap<>();
- collectStartTimes(collectionName, urlToTimeAfter);
-
- boolean retry = false;
- Set<Entry<String,Long>> entries = urlToTimeBefore.entrySet();
- for (Entry<String,Long> entry : entries) {
- Long beforeTime = entry.getValue();
- Long afterTime = urlToTimeAfter.get(entry.getKey());
- assertNotNull(afterTime);
- if (afterTime <= beforeTime) {
- retry = true;
- break;
- }
-
- }
- if (!retry) {
- allTimesAreCorrect = true;
- break;
- }
- }
- return allTimesAreCorrect;
- }
-
- private void collectStartTimes(String collectionName, Map<String,Long> urlToTime)
- throws SolrServerException, IOException {
-
- DocCollection collectionState = getCollectionState(collectionName);
- if (collectionState != null) {
- for (Slice shard : collectionState) {
- for (Replica replica : shard) {
- ZkCoreNodeProps coreProps = new ZkCoreNodeProps(replica);
- CoreStatus coreStatus;
- try (HttpSolrClient server = getHttpSolrClient(coreProps.getBaseUrl())) {
- coreStatus = CoreAdminRequest.getCoreStatus(coreProps.getCoreName(), false, server);
- }
- long before = coreStatus.getCoreStartTime().getTime();
- urlToTime.put(coreProps.getCoreUrl(), before);
- }
- }
- } else {
- throw new IllegalArgumentException("Could not find collection " + collectionName);
- }
- }
-
- private void checkNoTwoShardsUseTheSameIndexDir() {
- Map<String, Set<String>> indexDirToShardNamesMap = new HashMap<>();
-
- List<MBeanServer> servers = new LinkedList<>();
- servers.add(ManagementFactory.getPlatformMBeanServer());
- servers.addAll(MBeanServerFactory.findMBeanServer(null));
- for (final MBeanServer server : servers) {
- Set<ObjectName> mbeans = new HashSet<>(server.queryNames(null, null));
- for (final ObjectName mbean : mbeans) {
- try {
- Map<String, String> props = mbean.getKeyPropertyList();
- String category = props.get("category");
- String name = props.get("name");
- if ((category != null && category.equals(Category.CORE.toString())) &&
- (name != null && name.equals("indexDir"))) {
- String indexDir = server.getAttribute(mbean, "Value").toString();
- String key = props.get("dom2") + "." + props.get("dom3") + "." + props.get("dom4");
- if (!indexDirToShardNamesMap.containsKey(indexDir)) {
- indexDirToShardNamesMap.put(indexDir, new HashSet<>());
- }
- indexDirToShardNamesMap.get(indexDir).add(key);
- }
- } catch (Exception e) {
- // ignore, just continue - probably a "Value" attribute
- // not found
- }
- }
- }
-
- assertTrue(
- "Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of "
- + SolrCore.class.getSimpleName() + " : " + indexDirToShardNamesMap,
- indexDirToShardNamesMap.size() > 0);
- for (Entry<String,Set<String>> entry : indexDirToShardNamesMap.entrySet()) {
- if (entry.getValue().size() > 1) {
- fail("We have shards using the same indexDir. E.g. shards "
- + entry.getValue().toString() + " all use indexDir "
- + entry.getKey());
- }
- }
- }
-
- @Test
- public void addReplicaTest() throws Exception {
- String collectionName = "addReplicaColl";
-
- CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2)
- .setMaxShardsPerNode(4)
- .process(cluster.getSolrClient());
- cluster.waitForActiveCollection(collectionName, 2, 4);
-
- ArrayList<String> nodeList
- = new ArrayList<>(cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes());
- Collections.shuffle(nodeList, random());
-
- CollectionAdminResponse response = CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
- .setNode(nodeList.get(0))
- .process(cluster.getSolrClient());
- Replica newReplica = grabNewReplica(response, getCollectionState(collectionName));
-
- assertEquals("Replica should be created on the right node",
- cluster.getSolrClient().getZkStateReader().getBaseUrlForNodeName(nodeList.get(0)),
- newReplica.getStr(ZkStateReader.BASE_URL_PROP));
-
- Path instancePath = createTempDir();
- response = CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
- .withProperty(CoreAdminParams.INSTANCE_DIR, instancePath.toString())
- .process(cluster.getSolrClient());
- newReplica = grabNewReplica(response, getCollectionState(collectionName));
- assertNotNull(newReplica);
-
- try (HttpSolrClient coreclient = getHttpSolrClient(newReplica.getStr(ZkStateReader.BASE_URL_PROP))) {
- CoreAdminResponse status = CoreAdminRequest.getStatus(newReplica.getStr("core"), coreclient);
- NamedList<Object> coreStatus = status.getCoreStatus(newReplica.getStr("core"));
- String instanceDirStr = (String) coreStatus.get("instanceDir");
- assertEquals(instanceDirStr, instancePath.toString());
- }
-
- //Test to make sure we can't create another replica with an existing core_name of that collection
- String coreName = newReplica.getStr(CORE_NAME_PROP);
- SolrException e = expectThrows(SolrException.class, () -> {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", "addreplica");
- params.set("collection", collectionName);
- params.set("shard", "shard1");
- params.set("name", coreName);
- QueryRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
- cluster.getSolrClient().request(request);
- });
-
- assertTrue(e.getMessage().contains("Another replica with the same core name already exists for this collection"));
-
- // Check that specifying property.name works. DO NOT remove this when the "name" property is deprecated
- // for ADDREPLICA, this is "property.name". See SOLR-7132
- response = CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
- .withProperty(CoreAdminParams.NAME, "propertyDotName")
- .process(cluster.getSolrClient());
-
- newReplica = grabNewReplica(response, getCollectionState(collectionName));
- assertEquals("'core' should be 'propertyDotName' ", "propertyDotName", newReplica.getStr("core"));
- }
-
- private Replica grabNewReplica(CollectionAdminResponse response, DocCollection docCollection) {
- String replicaName = response.getCollectionCoresStatus().keySet().iterator().next();
- Optional<Replica> optional = docCollection.getReplicas().stream()
- .filter(replica -> replicaName.equals(replica.getCoreName()))
- .findAny();
- if (optional.isPresent()) {
- return optional.get();
- }
- throw new AssertionError("Can not find " + replicaName + " from " + docCollection);
- }
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
index 0297385..cba0566 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
@@ -42,6 +42,7 @@ public class HdfsCollectionsAPIDistributedZkTest extends CollectionsAPIDistribut
@BeforeClass
public static void setupClass() throws Exception {
+ configSet = getConfigSet();
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@@ -54,8 +55,7 @@ public class HdfsCollectionsAPIDistributedZkTest extends CollectionsAPIDistribut
}
}
- @Override
- protected String getConfigSet() {
+ protected static String getConfigSet() {
return "cloud-hdfs";
}
}
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index 5a5476e..fe6251c 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -25,6 +25,7 @@ import java.util.concurrent.Callable;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
@@ -77,6 +78,7 @@ import static org.hamcrest.CoreMatchers.containsString;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
//@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13822") // leaks files
+@LuceneTestCase.Nightly // nocommit debug - can be slow
public class TestPackages extends SolrCloudTestCase {
@Before
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java
index 70f4ad2..63941b7 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java
@@ -95,9 +95,18 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true");
// multi replicas should not matter...
- final int repFactor = usually() ? 1 : 2;
+ final int repFactor;
// ... but we definitely want to test multiple shards
- final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
+ final int numShards;
+
+ if (TEST_NIGHTLY) {
+ numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
+ repFactor = usually() ? 1 : 2;
+ } else {
+ numShards = 1;
+ repFactor = 1;
+ }
+
final int numNodes = (numShards * repFactor);
final String configName = DEBUG_LABEL + "_config-set";
@@ -115,13 +124,11 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
CLOUD_CLIENT = cluster.getSolrClient();
CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
- waitForRecoveriesToFinish(CLOUD_CLIENT);
-
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
}
- final int numDocs = atLeast(100);
+ final int numDocs = atLeast(TEST_NIGHTLY ? 100 : 25);
for (int id = 0; id < numDocs; id++) {
SolrInputDocument doc = sdoc("id", ""+id);
for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) {
@@ -367,7 +374,7 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
public void testTheTestRandomRefineParam() {
// sanity check that randomRefineParam never violates isRefinementNeeded
// (should be imposisble ... unless someone changes/breaks the randomization logic in the future)
- final int numIters = atLeast(100);
+ final int numIters = atLeast(TEST_NIGHTLY ? 100 : 10);
for (int iter = 0; iter < numIters; iter++) {
final Integer limit = TermFacet.randomLimitParam(random());
final Integer overrequest = TermFacet.randomOverrequestParam(random());
@@ -393,10 +400,10 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
// we get a really big one early on, we can test as much as possible, skip other iterations.
//
// (deeply nested facets may contain more buckets then the max, but we won't *check* all of them)
- final int maxBucketsAllowed = atLeast(2000);
+ final int maxBucketsAllowed = atLeast(TEST_NIGHTLY ? 2000 : 200);
final AtomicInteger maxBucketsToCheck = new AtomicInteger(maxBucketsAllowed);
- final int numIters = atLeast(20);
+ final int numIters = atLeast(TEST_NIGHTLY ? 20 : 3);
for (int iter = 0; iter < numIters && 0 < maxBucketsToCheck.get(); iter++) {
assertFacetCountsAreCorrect(maxBucketsToCheck, TermFacet.buildRandomFacets(), buildRandomQuery());
}
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
index f6103f4..9052378 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
@@ -228,6 +228,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
public void indexSimple(Client client) throws Exception {
client.deleteByQuery("*:*", null);
+ client.commit();
client.add(sdoc("id", "1", "cat_s", "A", "where_s", "NY", "num_d", "4", "num_i", "2",
"num_is", "4", "num_is", "2",
"val_b", "true", "sparse_s", "one"), null);
@@ -1056,6 +1057,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
}
@Test
+ @Nightly
public void testStats() throws Exception {
doStats(Client.localClient, params("debugQuery", Boolean.toString(random().nextBoolean()) ));
}
@@ -2538,7 +2540,8 @@ public class TestJsonFacets extends SolrTestCaseHS {
public void testPrelimSortingSingleNode() throws Exception {
doTestPrelimSortingSingleNode(false, false);
}
-
+
+ @Nightly
public void testPrelimSortingSingleNodeExtraStat() throws Exception {
doTestPrelimSortingSingleNode(true, false);
}
@@ -2967,6 +2970,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
@Test
+ @Nightly
public void testBigger() throws Exception {
ModifiableSolrParams p = params("rows", "0", "cat_s", "cat_ss", "where_s", "where_ss");
// doBigger(Client.localClient, p);