You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/07/17 11:45:29 UTC
lucene-solr:branch_7x: SOLR-12412: Fix test failure
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 6ed703bd8 -> eed5e7bb1
SOLR-12412: Fix test failure
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/eed5e7bb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/eed5e7bb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/eed5e7bb
Branch: refs/heads/branch_7x
Commit: eed5e7bb1c28fc99982a8d13b33c68425e99e21c
Parents: 6ed703b
Author: Cao Manh Dat <da...@apache.org>
Authored: Tue Jul 17 18:42:29 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Tue Jul 17 18:45:16 2018 +0700
----------------------------------------------------------------------
.../solr/cloud/LeaderTragicEventTest.java | 123 +++++++++++--------
.../apache/solr/cloud/MiniSolrCloudCluster.java | 1 +
.../apache/solr/core/MockDirectoryFactory.java | 6 +-
3 files changed, 77 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eed5e7bb/solr/core/src/test/org/apache/solr/cloud/LeaderTragicEventTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderTragicEventTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderTragicEventTest.java
index c838eff..747555b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/LeaderTragicEventTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/LeaderTragicEventTest.java
@@ -17,6 +17,7 @@
package org.apache.solr.cloud;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
@@ -34,6 +35,7 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.MockDirectoryFactory;
import org.apache.solr.core.SolrCore;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -43,64 +45,71 @@ import org.slf4j.LoggerFactory;
public class LeaderTragicEventTest extends SolrCloudTestCase {
- private static final String COLLECTION = "collection1";
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("solr.mscheduler", "org.apache.solr.core.MockConcurrentMergeScheduler");
+ System.setProperty(MockDirectoryFactory.SOLR_TESTS_USING_MOCK_DIRECTORY_WRAPPER, "true");
configureCluster(2)
.addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
.configure();
-
- cluster.getSolrClient().setDefaultCollection(COLLECTION);
}
@AfterClass
public static void cleanup() {
System.clearProperty("solr.mscheduler");
+ System.clearProperty(MockDirectoryFactory.SOLR_TESTS_USING_MOCK_DIRECTORY_WRAPPER);
}
@Test
public void test() throws Exception {
+ final String collection = "collection1";
+ cluster.getSolrClient().setDefaultCollection(collection);
CollectionAdminRequest
- .createCollection(COLLECTION, "config", 1, 2)
+ .createCollection(collection, "config", 1, 2)
.process(cluster.getSolrClient());
- ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), COLLECTION, 120000);
-
- List<String> addedIds = new ArrayList<>();
- Replica oldLeader = corruptLeader(addedIds);
-
- waitForState("Timeout waiting for new replica become leader", COLLECTION, (liveNodes, collectionState) -> {
- Slice slice = collectionState.getSlice("shard1");
-
- if (slice.getReplicas().size() != 2) return false;
- if (slice.getLeader().getName().equals(oldLeader.getName())) return false;
-
- return true;
- });
- ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), COLLECTION, 120000);
- Slice shard = getCollectionState(COLLECTION).getSlice("shard1");
- assertNotSame(shard.getLeader().getNodeName(), oldLeader.getNodeName());
- assertEquals(getNonLeader(shard).getNodeName(), oldLeader.getNodeName());
+ ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), collection, 120000);
+ try {
+ List<String> addedIds = new ArrayList<>();
+ Replica oldLeader = corruptLeader(collection, addedIds);
+
+ waitForState("Timeout waiting for new replica become leader", collection, (liveNodes, collectionState) -> {
+ Slice slice = collectionState.getSlice("shard1");
+
+ if (slice.getReplicas().size() != 2) return false;
+ if (slice.getLeader() == null) return false;
+ if (slice.getLeader().getName().equals(oldLeader.getName())) return false;
+
+ return true;
+ });
+ ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), collection, 120000);
+ Slice shard = getCollectionState(collection).getSlice("shard1");
+ assertNotSame(shard.getLeader().getNodeName(), oldLeader.getNodeName());
+ assertEquals(getNonLeader(shard).getNodeName(), oldLeader.getNodeName());
+
+ for (String id : addedIds) {
+ assertNotNull(cluster.getSolrClient().getById(collection,id));
+ }
+ log.info("The test success oldLeader:{} currentState:{}", oldLeader, getCollectionState(collection));
- for (String id : addedIds) {
- assertNotNull(cluster.getSolrClient().getById(COLLECTION,id));
+ } finally {
+ CollectionAdminRequest.deleteCollection(collection).process(cluster.getSolrClient());
}
- log.info("The test success oldLeader:{} currentState:{}", oldLeader, getCollectionState(COLLECTION));
-
- CollectionAdminRequest.deleteCollection(COLLECTION).process(cluster.getSolrClient());
}
- private Replica corruptLeader(List<String> addedIds) throws IOException {
- DocCollection dc = getCollectionState(COLLECTION);
+ private Replica corruptLeader(String collection, List<String> addedIds) throws IOException {
+ DocCollection dc = getCollectionState(collection);
Replica oldLeader = dc.getLeader("shard1");
+ log.info("Corrupt leader : {}", oldLeader);
+
CoreContainer leaderCC = cluster.getReplicaJetty(oldLeader).getCoreContainer();
SolrCore leaderCore = leaderCC.getCores().iterator().next();
- MockDirectoryWrapper dir = (MockDirectoryWrapper) leaderCore.getDirectoryFactory().get(leaderCore.getIndexDir(), DirectoryFactory.DirContext.DEFAULT, leaderCore.getSolrConfig().indexConfig.lockType);
- leaderCore.getDirectoryFactory().release(dir);
+ MockDirectoryWrapper mockDir = (MockDirectoryWrapper) leaderCore.getDirectoryFactory()
+ .get(leaderCore.getIndexDir(), DirectoryFactory.DirContext.DEFAULT, leaderCore.getSolrConfig().indexConfig.lockType);
+ leaderCore.getDirectoryFactory().release(mockDir);
try (HttpSolrClient solrClient = new HttpSolrClient.Builder(dc.getLeader("shard1").getCoreUrl()).build()) {
for (int i = 0; i < 100; i++) {
@@ -110,16 +119,21 @@ public class LeaderTragicEventTest extends SolrCloudTestCase {
solrClient.commit();
addedIds.add(i + "");
- for (String file : dir.listAll()) {
+ for (String file : mockDir.listAll()) {
if (file.contains("segments_")) continue;
if (file.endsWith("si")) continue;
if (file.endsWith("fnm")) continue;
if (random().nextBoolean()) continue;
- dir.corruptFiles(Collections.singleton(file));
+ try {
+ mockDir.corruptFiles(Collections.singleton(file));
+ } catch (RuntimeException | FileNotFoundException e) {
+ // merges can lead to this exception
+ }
}
}
} catch (Exception e) {
+ log.info("Corrupt leader ex: ",e);
// Expected
}
return oldLeader;
@@ -132,34 +146,39 @@ public class LeaderTragicEventTest extends SolrCloudTestCase {
@Test
public void testOtherReplicasAreNotActive() throws Exception {
+ final String collection = "collection2";
+ cluster.getSolrClient().setDefaultCollection(collection);
int numReplicas = random().nextInt(2) + 1;
// won't do anything if leader is the only one active replica in the shard
CollectionAdminRequest
- .createCollection(COLLECTION, "config", 1, numReplicas)
+ .createCollection(collection, "config", 1, numReplicas)
.process(cluster.getSolrClient());
- ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), COLLECTION, 120000);
-
- JettySolrRunner otherReplicaJetty = null;
- if (numReplicas == 2) {
- Slice shard = getCollectionState(COLLECTION).getSlice("shard1");
- otherReplicaJetty = cluster.getReplicaJetty(getNonLeader(shard));
- otherReplicaJetty.stop();
- waitForState("Timeout waiting for replica get down", COLLECTION, (liveNodes, collectionState) -> getNonLeader(collectionState.getSlice("shard1")).getState() != Replica.State.ACTIVE);
- }
+ ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), collection, 120000);
+
+ try {
+ JettySolrRunner otherReplicaJetty = null;
+ if (numReplicas == 2) {
+ Slice shard = getCollectionState(collection).getSlice("shard1");
+ otherReplicaJetty = cluster.getReplicaJetty(getNonLeader(shard));
+ log.info("Stop jetty node : {} state:{}", otherReplicaJetty.getBaseUrl(), getCollectionState(collection));
+ otherReplicaJetty.stop();
+ waitForState("Timeout waiting for replica get down", collection, (liveNodes, collectionState) -> getNonLeader(collectionState.getSlice("shard1")).getState() != Replica.State.ACTIVE);
+ }
- Replica oldLeader = corruptLeader(new ArrayList<>());
+ Replica oldLeader = corruptLeader(collection, new ArrayList<>());
- //TODO better way to test this
- Thread.sleep(5000);
- Replica leader = getCollectionState(COLLECTION).getSlice("shard1").getLeader();
- assertEquals(leader.getName(), oldLeader.getName());
+ //TODO better way to test this
+ Thread.sleep(5000);
+ Replica leader = getCollectionState(collection).getSlice("shard1").getLeader();
+ assertEquals(leader.getName(), oldLeader.getName());
- if (otherReplicaJetty != null) {
- // won't be able to do anything here, since this replica can't recovery from the leader
- otherReplicaJetty.start();
+ if (otherReplicaJetty != null) {
+ // won't be able to do anything here, since this replica can't recovery from the leader
+ otherReplicaJetty.start();
+ }
+ } finally {
+ CollectionAdminRequest.deleteCollection(collection).process(cluster.getSolrClient());
}
-
- CollectionAdminRequest.deleteCollection(COLLECTION).process(cluster.getSolrClient());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eed5e7bb/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
index 6c28eba..df458d7 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
@@ -536,6 +536,7 @@ public class MiniSolrCloudCluster {
*/
public JettySolrRunner getReplicaJetty(Replica replica) {
for (JettySolrRunner jetty : jettys) {
+ if (jetty.isStopped()) continue;
if (replica.getCoreUrl().startsWith(jetty.getBaseUrl().toString()))
return jetty;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eed5e7bb/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java b/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
index 06fac8e..07e1d0d 100644
--- a/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
+++ b/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
@@ -33,7 +33,9 @@ import org.apache.lucene.util.LuceneTestCase;
public class MockDirectoryFactory extends EphemeralDirectoryFactory {
public static final String SOLR_TESTS_ALLOW_READING_FILES_STILL_OPEN_FOR_WRITE = "solr.tests.allow_reading_files_still_open_for_write";
+ public static final String SOLR_TESTS_USING_MOCK_DIRECTORY_WRAPPER = "solr.tests.using_mock_directory_wrapper";
private boolean allowReadingFilesStillOpenForWrite = Boolean.getBoolean(SOLR_TESTS_ALLOW_READING_FILES_STILL_OPEN_FOR_WRITE);
+ private boolean useMockDirectoryWrapper = Boolean.getBoolean(SOLR_TESTS_USING_MOCK_DIRECTORY_WRAPPER);
@Override
protected LockFactory createLockFactory(String rawLockType) throws IOException {
@@ -42,7 +44,9 @@ public class MockDirectoryFactory extends EphemeralDirectoryFactory {
@Override
protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
- Directory dir = LuceneTestCase.newDirectory(); // we ignore the given lock factory
+ Directory dir;
+ if (useMockDirectoryWrapper) dir = LuceneTestCase.newMockDirectory();
+ else dir = LuceneTestCase.newDirectory(); // we ignore the given lock factory
Directory cdir = reduce(dir);
cdir = reduce(cdir);