You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/09/07 15:31:54 UTC
[lucene-solr] branch reference_impl_dev updated (99f8da7 -> 599b1d3)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a change to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 99f8da7 @777 Try to catch AddReplicaTest fail.
new 3494c27 @778 Make thread safe.
new 599b1d3 @779 Enable some more tests.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../client/solrj/embedded/JettySolrRunner.java | 3 +--
.../solr/cloud/api/collections/AddReplicaCmd.java | 19 ++++++++++++-------
.../apache/solr/cloud/SolrCloudBridgeTestCase.java | 5 ++---
.../apache/solr/cloud/UnloadDistributedZkTest.java | 22 ++++++++++------------
.../cloud/api/collections/TestCollectionAPI.java | 1 -
.../TestCollectionsAPIViaSolrCloudCluster.java | 14 +++++---------
.../apache/solr/cloud/MiniSolrCloudCluster.java | 3 ---
7 files changed, 30 insertions(+), 37 deletions(-)
[lucene-solr] 01/02: @778 Make thread safe.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 3494c2701d55801ac099fa4acfae3583d553a890
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Mon Sep 7 09:42:16 2020 -0500
@778 Make thread safe.
---
.../java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
index a32f26e..f3facf7 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
@@ -41,6 +41,8 @@ import java.util.EnumMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
@@ -206,7 +208,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
}
if (asyncId != null) {
- List<String> coreNodeNames = new ArrayList<>();
+ Set<String> coreNodeNames = ConcurrentHashMap.newKeySet(createReplicas.size());
for (CreateReplica replica : createReplicas) {
coreNodeNames.add(ocmh.waitForCoreNodeName(zkStateReader, collectionName, replica.node, replica.coreName));
}
@@ -247,7 +249,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
if (asyncId == null) {
- List<String> coreNodeNames = new ArrayList<>();
+ Set<String> coreNodeNames = ConcurrentHashMap.newKeySet(createReplicas.size());
for (CreateReplica replica : createReplicas) {
coreNodeNames.add(ocmh.waitForCoreNodeName(zkStateReader, collectionName, replica.node, replica.coreName));
}
[lucene-solr] 02/02: @779 Enable some more tests.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 599b1d33eb1adb2080dd51750dbf9513357263b5
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Mon Sep 7 10:31:29 2020 -0500
@779 Enable some more tests.
---
.../client/solrj/embedded/JettySolrRunner.java | 3 +--
.../solr/cloud/api/collections/AddReplicaCmd.java | 13 ++++++++-----
.../apache/solr/cloud/SolrCloudBridgeTestCase.java | 5 ++---
.../apache/solr/cloud/UnloadDistributedZkTest.java | 22 ++++++++++------------
.../cloud/api/collections/TestCollectionAPI.java | 1 -
.../TestCollectionsAPIViaSolrCloudCluster.java | 14 +++++---------
.../apache/solr/cloud/MiniSolrCloudCluster.java | 3 ---
7 files changed, 26 insertions(+), 35 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
index a7d6180..ee50dce 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
@@ -502,8 +502,7 @@ public class JettySolrRunner implements Closeable {
}
public boolean isStopped() {
- return (server.isStopped() && dispatchFilter == null) || (server.isStopped() && dispatchFilter.isStopped()
- && ((QueuedThreadPool) server.getThreadPool()).isStopped());
+ return (server.isStopped() && dispatchFilter == null) || (server.isStopped() && dispatchFilter.isStopped());
}
// ------------------------------------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
index f3facf7..d1452f1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
@@ -101,8 +101,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
throws IOException, InterruptedException, KeeperException {
log.info("addReplica() : {}", Utils.toJSONString(message));
-
-
+
String extCollectionName = message.getStr(COLLECTION_PROP);
boolean followAliases = message.getBool(FOLLOW_ALIASES, false);
String shard = message.getStr(SHARD_ID_PROP);
@@ -148,7 +147,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
}
if (totalReplicas > 1) {
if (node != null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot create " + totalReplicas + " replicas if 'name' parameter is specified");
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot create " + totalReplicas + " replicas if 'node' parameter is specified");
}
if (message.getStr(CoreAdminParams.CORE_NODE_NAME) != null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot create " + totalReplicas + " replicas if 'coreNodeName' parameter is specified");
@@ -159,7 +158,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
List<CreateReplica> createReplicas;
try {
- ocmh.zkStateReader.waitForState(collectionName, 3, TimeUnit.SECONDS, (liveNodes, collectionState) -> {
+ ocmh.zkStateReader.waitForState(collectionName, 5, TimeUnit.SECONDS, (liveNodes, collectionState) -> {
if (collectionState == null) {
return false;
}
@@ -185,6 +184,10 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
}
}
+ if (createReplicas.size() != totalReplicas) {
+ throw new IllegalStateException("Did not get enough positions to cover new replicas");
+ }
+
ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(ocmh.overseer.getCoreContainer().getUpdateShardHandler().getTheSharedHttpClient());
ZkStateReader zkStateReader = ocmh.zkStateReader;
@@ -216,7 +219,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
try {
zkStateReader.waitForState(collectionName, 10, TimeUnit.SECONDS, (liveNodes, collectionState) -> {
if (collectionState == null) {
- return true; // deleted collection
+ return false;
}
List<Replica> replicas = collectionState.getReplicas();
diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCloudBridgeTestCase.java b/solr/core/src/test/org/apache/solr/cloud/SolrCloudBridgeTestCase.java
index dbee754..0b41836 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SolrCloudBridgeTestCase.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SolrCloudBridgeTestCase.java
@@ -553,15 +553,14 @@ public abstract class SolrCloudBridgeTestCase extends SolrCloudTestCase {
final int freezeI = i;
executor.execute(() -> {
try {
- assertTrue(CollectionAdminRequest.addReplicaToShard(collection, "shard"+((freezeI%numShards)+1))
+ CollectionAdminRequest.addReplicaToShard(collection, "shard"+((freezeI%numShards)+1))
.setCoreName(collection + freezeI)
- .setNode(nodeName).process(client).isSuccess());
+ .setNode(nodeName).process(client);
} catch (SolrServerException | IOException e) {
throw new RuntimeException(e);
}
});
}
- cluster.waitForActiveCollection(collection, numShards, numReplicas);
}
protected boolean reloadCollection(Replica replica, String testCollectionName) throws Exception {
diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
index 451b8a3..7bffe7a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
@@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.solr.SolrTestCase;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -54,7 +55,6 @@ import org.junit.Test;
* work as expected.
*/
@SolrTestCase.SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
-@Ignore // nocommit debug
public class UnloadDistributedZkTest extends SolrCloudBridgeTestCase {
public UnloadDistributedZkTest() {
@@ -100,6 +100,7 @@ public class UnloadDistributedZkTest extends SolrCloudBridgeTestCase {
}
@Test
+ @Nightly
public void testUnloadShardAndCollection() throws Exception{
final int numShards = 2;
@@ -108,22 +109,18 @@ public class UnloadDistributedZkTest extends SolrCloudBridgeTestCase {
final String coreName1 = collection+"_1";
final String coreName2 = collection+"_2";
- assertEquals(0, CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1)
+ assertEquals(0, CollectionAdminRequest.createCollection(collection, "_default", numShards, 1)
.setCreateNodeSet("")
.process(cloudClient).getStatus());
- assertTrue(CollectionAdminRequest.addReplicaToShard(collection, "shard1")
+ CollectionAdminRequest.addReplicaToShard(collection, "shard1")
.setCoreName(coreName1)
.setNode(cluster.getJettySolrRunner(0).getNodeName())
- .process(cloudClient).isSuccess());
+ .process(cloudClient);
- assertTrue(CollectionAdminRequest.addReplicaToShard(collection, "shard2")
+ CollectionAdminRequest.addReplicaToShard(collection, "shard2")
.setCoreName(coreName2)
.setNode(cluster.getJettySolrRunner(0).getNodeName())
- .process(cloudClient).isSuccess());
-
-
- // does not mean they are active and up yet :*
- waitForRecoveriesToFinish(collection);
+ .process(cloudClient);
final boolean unloadInOrder = random().nextBoolean();
final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2);
@@ -168,7 +165,7 @@ public class UnloadDistributedZkTest extends SolrCloudBridgeTestCase {
JettySolrRunner jetty1 = cluster.getJettySolrRunner(0);
assertEquals(0, CollectionAdminRequest
- .createCollection("unloadcollection", "conf1", 1,1)
+ .createCollection("unloadcollection", "_default", 1,1)
.setCreateNodeSet(jetty1.getNodeName())
.process(cloudClient).getStatus());
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
@@ -330,9 +327,10 @@ public class UnloadDistributedZkTest extends SolrCloudBridgeTestCase {
}
@Test
+ @Ignore // nocommit - needs to be hardened
public void testUnloadLotsOfCores() throws Exception {
JettySolrRunner jetty = cluster.getJettySolrRunner(0);
- try (final HttpSolrClient adminClient = (HttpSolrClient) jetty.newClient(15000, 60000)) {
+ try (final Http2SolrClient adminClient = (Http2SolrClient) jetty.newClient(15000, 60000)) {
int numReplicas = atLeast(3);
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
index 0f67d1a..41d16fa 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
@@ -1055,7 +1055,6 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {
*/
@Test
@ShardsFixed(num = 2)
- @Ignore // debug
public void testRecreateCollectionAfterFailure() throws Exception {
// Upload a bad configset
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(), ZkTestServer.TIMEOUT,
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionsAPIViaSolrCloudCluster.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionsAPIViaSolrCloudCluster.java
index 52523ff..c9866ae 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionsAPIViaSolrCloudCluster.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionsAPIViaSolrCloudCluster.java
@@ -76,6 +76,7 @@ public class TestCollectionsAPIViaSolrCloudCluster extends SolrCloudTestCase {
@Override
public void tearDown() throws Exception {
cluster.shutdown();
+ cluster = null;
super.tearDown();
}
@@ -98,7 +99,7 @@ public class TestCollectionsAPIViaSolrCloudCluster extends SolrCloudTestCase {
}
@Test
- @Ignore // nocommit debug
+ @Ignore // nocommit "was expecting to find a node without a replica
public void testCollectionCreateSearchDelete() throws Exception {
final CloudHttp2SolrClient client = cluster.getSolrClient();
final String collectionName = "testcollection";
@@ -183,7 +184,6 @@ public class TestCollectionsAPIViaSolrCloudCluster extends SolrCloudTestCase {
@Test
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 09-Apr-2018
- @Ignore // nocommit - need to finish addressing async
public void testCollectionCreateWithoutCoresThenDelete() throws Exception {
final String collectionName = "testSolrCloudCollectionWithoutCores";
@@ -208,7 +208,6 @@ public class TestCollectionsAPIViaSolrCloudCluster extends SolrCloudTestCase {
}
@Test
- @Ignore // nocommit - good debug for parallel commit
public void testStopAllStartAll() throws Exception {
final String collectionName = "testStopAllStartAllCollection";
@@ -338,10 +337,9 @@ public class TestCollectionsAPIViaSolrCloudCluster extends SolrCloudTestCase {
}
}
} else {
+ log.info("START JETTY'S BACK UP");
try (ParWork worker = new ParWork(this)) {
- for (Integer ii : restartIndicesList) {
- final JettySolrRunner jetty = jettys.get(ii);
- if (!jetty.isRunning()) {
+ for (JettySolrRunner jetty : jettys) {
worker.collect("startJetties", () -> {
try {
cluster.startJettySolrRunner(jetty);
@@ -350,12 +348,10 @@ public class TestCollectionsAPIViaSolrCloudCluster extends SolrCloudTestCase {
}
assertTrue(jetty.isRunning());
});
-
- }
}
}
}
- cluster.waitForActiveCollection(collectionName, numShards, numShards * numReplicas);
+// cluster.waitForActiveCollection(collectionName, numShards, numShards * numReplicas);
// re-query collection
assertEquals(numDocs, client.query(collectionName, query).getResults().getNumFound());
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
index 8659212..7e1fb67 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
@@ -663,9 +663,6 @@ public class MiniSolrCloudCluster {
* Shut down the cluster, including all Solr nodes and ZooKeeper
*/
public synchronized void shutdown() throws Exception {
- if (this.isShutDown) {
- throw new AlreadyClosedException("This MiniSolrCloudCluster has already been shutdown");
- }
this.isShutDown = true;
// if (zkServer.getZkClient().isConnected()) {
// try {