You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2021/02/10 05:57:12 UTC
[lucene-solr] branch reference_impl_dev updated: @1337 Fix a dist
update bug and some other cleanup / fixes.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/reference_impl_dev by this push:
new ca760ca @1337 Fix a dist update bug and some other cleanup / fixes.
ca760ca is described below
commit ca760ca70c11ae893fd0bf7266f0bf3bdc12f515
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Feb 9 23:55:16 2021 -0600
@1337 Fix a dist update bug and some other cleanup / fixes.
---
solr/bin/jetty.sh | 2 +-
.../cloud/api/collections/CreateCollectionCmd.java | 2 +-
.../solr/cloud/overseer/CollectionMutator.java | 2 +-
.../java/org/apache/solr/core/CoreContainer.java | 4 +-
.../src/java/org/apache/solr/core/SolrCore.java | 73 ++++---
.../java/org/apache/solr/update/SolrCoreState.java | 1 +
.../processor/DistributedZkUpdateProcessor.java | 25 +--
.../solr/cloud/ChaosMonkeySafeLeaderTest.java | 189 +------------------
.../solr/cloud/FullSolrCloudDistribCmdsTest.java | 4 +-
.../src/java/org/apache/solr/cli/CliCommand.java | 7 +
...lusterCommand.java => ClusterCheckCommand.java} | 39 ++--
...lusterCommand.java => ClusterStateCommand.java} | 45 +++--
.../apache/solr/cli/CreateCollectionCommand.java | 110 +++++++++++
.../apache/solr/cli/DeleteCollectionCommand.java | 69 +++++++
.../src/java/org/apache/solr/cli/Stellar.java | 12 +-
.../client/solrj/impl/BaseCloudSolrClient.java | 5 +-
.../solr/client/solrj/impl/LBSolrClient.java | 3 +
.../solrj/request/CollectionAdminRequest.java | 6 +-
.../org/apache/solr/cloud/CloudInspectUtil.java | 8 +-
.../solr/common/cloud/ConnectionManager.java | 3 +-
.../org/apache/solr/common/cloud/SolrZkClient.java | 8 +-
.../apache/solr/common/cloud/ZkStateReader.java | 210 ++++++++++++++++++++-
.../apache/solr/cloud/MiniSolrCloudCluster.java | 31 ++-
.../src/resources/logconf/log4j2-std-debug.xml | 29 +--
24 files changed, 559 insertions(+), 328 deletions(-)
diff --git a/solr/bin/jetty.sh b/solr/bin/jetty.sh
index ad377dc..7a66559 100644
--- a/solr/bin/jetty.sh
+++ b/solr/bin/jetty.sh
@@ -326,7 +326,7 @@ CYGWIN*) JETTY_STATE="`cygpath -w $JETTY_STATE`";;
esac
-JETTY_ARGS=(${JETTY_ARGS[*]} "-Djetty.state=$JETTY_STATE")
+JETTY_ARGS=(${JETTY_ARGS[*]} "jetty.state=$JETTY_STATE")
##################################################
# Get the list of config.xml files from jetty.conf
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index 75126a9..81621f6 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -586,7 +586,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
// TODO need to make this makePath calls efficient and not use zkSolrClient#makePath
for (String shardName : slices.keySet()) {
try {
- stateManager.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + cName + "/" + shardName, null, CreateMode.PERSISTENT, false);
+ //stateManager.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + cName + "/" + shardName, null, CreateMode.PERSISTENT, false);
stateManager.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + cName + "/leader_elect", null, CreateMode.PERSISTENT, false);
stateManager.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + cName + "/leader_elect/" + shardName, null, CreateMode.PERSISTENT, false);
stateManager.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + cName + "/leader_elect/" + shardName + "/election", null, CreateMode.PERSISTENT, false);
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
index 57ca7bf..85dd0bd 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
@@ -84,7 +84,7 @@ public class CollectionMutator {
// TODO - fix, no makePath (ensure every path part exists), async, single node
try {
- stateManager.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName + "/" + shardId, null, CreateMode.PERSISTENT, false);
+ // stateManager.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName + "/" + shardId, null, CreateMode.PERSISTENT, false);
stateManager.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName + "/leader_elect/" + shardId, null, CreateMode.PERSISTENT, false);
stateManager.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName + "/leader_elect/" + shardId + "/election", null, CreateMode.PERSISTENT, false);
stateManager.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName + "/leaders/" + shardId, null, CreateMode.PERSISTENT, false);
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index e5e3b5f..cd413dc 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1286,10 +1286,10 @@ public class CoreContainer implements Closeable {
try {
Future<?> future = solrCoreExecutor.submit(() -> {
log.info("Closing replaced core {}", cd.getName());
- finalCore.closeAndWait();
+ finalCore.close();
});
} catch (RejectedExecutionException e) {
- finalCore.closeAndWait();
+ finalCore.close();
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index a8e2325..109ecd7 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -1713,15 +1713,15 @@ public final class SolrCore implements SolrInfoBean, Closeable {
}
int cnt = refCount.incrementAndGet();
- if (log.isDebugEnabled()) {
- RuntimeException e = new RuntimeException();
- StackTraceElement[] stack = e.getStackTrace();
- for (int i = 0; i < Math.min(8, stack.length - 1); i++) {
- log.debug(stack[i].toString());
- }
-
- log.debug("open refcount {} {} {}", this, name, cnt);
- }
+// if (log.isDebugEnabled()) {
+// RuntimeException e = new RuntimeException();
+// StackTraceElement[] stack = e.getStackTrace();
+// for (int i = 0; i < Math.min(8, stack.length - 1); i++) {
+// log.debug(stack[i].toString());
+// }
+//
+// log.debug("open refcount {} {} {}", this, name, cnt);
+// }
}
/**
@@ -1751,21 +1751,23 @@ public final class SolrCore implements SolrInfoBean, Closeable {
*/
@Override
public void close() {
- if (refCount.get() == -1) {
- return;
+ int cref = refCount.get();
+ if (cref == -1 || cref == 0) {
+ throw new IllegalStateException("Already closed " + cref);
}
- int count = refCount.decrementAndGet();
- if (log.isDebugEnabled()) {
- RuntimeException e = new RuntimeException();
- StackTraceElement[] stack = e.getStackTrace();
- for (int i = 0; i < Math.min(8, stack.length - 1); i++) {
- log.debug(stack[i].toString());
- }
+ int count = refCount.decrementAndGet();
- log.debug("close refcount after {} {} {}", this, name, count);
- }
+// if (log.isDebugEnabled()) {
+// RuntimeException e = new RuntimeException();
+// StackTraceElement[] stack = e.getStackTrace();
+// for (int i = 0; i < Math.min(8, stack.length - 1); i++) {
+// log.debug(stack[i].toString());
+// }
+//
+// log.debug("close refcount after {} {} {}", this, name, count);
+// }
if (count == 0) {
try {
@@ -1801,11 +1803,16 @@ public final class SolrCore implements SolrInfoBean, Closeable {
int timeouts = 30;
+ // MRM TODO: put this timeout in play again
TimeOut timeout = new TimeOut(timeouts, TimeUnit.SECONDS, TimeSource.NANO_TIME);
int cnt = 0;
while (!canBeClosed() || refCount.get() != -1) {
- if (cnt >= 2 && !closing) {
- throw new IllegalStateException();
+// if (cnt >= 2 && !closing) {
+// throw new IllegalStateException();
+// }
+ if (refCount.get() == 0 && !closing) {
+ doClose();
+ break;
}
synchronized (closeAndWait) {
try {
@@ -1814,25 +1821,8 @@ public final class SolrCore implements SolrInfoBean, Closeable {
throw new IllegalStateException();
}
}
+ if (log.isDebugEnabled()) log.debug("close count is {} {} closing={} isClosed={}", name, refCount.get(), closing, isClosed);
cnt++;
-// cnt++;
-// try {
-// if (!closing) {
-// synchronized (closeAndWait) {
-// closeAndWait.wait(250);
-// }
-// }
-// if (cnt >= 2 && !closing) {
-// close();
-// }
-// if (log.isTraceEnabled()) log.trace("close count is {} {} closing={} isClosed={}", name, refCount.get(), closing, isClosed);
-// } catch (InterruptedException e) {
-//
-// }
-// if (timeout.hasTimedOut()) {
-// throw new SolrException(ErrorCode.SERVER_ERROR, "Timeout waiting for SolrCore close timeout=" + timeouts + "s");
-// }
-
}
}
@@ -1840,6 +1830,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
try {
if (closing) {
+ this.closing = true;
while (!isClosed) {
synchronized (closeAndWait) {
try {
@@ -1854,7 +1845,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
log.info("CLOSING SolrCore {}", logid);
assert ObjectReleaseTracker.release(this);
- this.closing = true;
+
searcherReadyLatch.countDown();
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
index aa75fa9..469126c 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
@@ -58,6 +58,7 @@ public abstract class SolrCoreState {
protected int solrCoreStateRefCnt = 1;
public void increfSolrCoreState() {
+ if (log.isDebugEnabled()) log.debug("SolrCoreState ref count {}", solrCoreStateRefCnt);
synchronized (this) {
if (solrCoreStateRefCnt == 0) {
throw new CoreIsClosedException("IndexWriter has been closed");
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
index 484bb61..cae28c1 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
@@ -668,6 +668,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
return null;
}
+ clusterState = zkController.getClusterState();
DocCollection coll = clusterState.getCollection(collection);
Slice slice = coll.getRouter().getTargetSlice(id, doc, route, req.getParams(), coll);
@@ -697,35 +698,27 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
String shardId = slice.getName();
Replica leaderReplica = null;
- LeaderElector elector = zkController.getLeaderElector(desc.getName());
try {
- isLeader = elector != null && elector.isLeader();
+ // Not equivalent to getLeaderProps, which retries to find a leader.
+ // Replica leader = slice.getLeader();
+ leaderReplica = clusterState.getCollection(collection).getSlice(shardId).getLeader();
+ isLeader = leaderReplica.getName().equals(desc.getName());
if (log.isTraceEnabled()) log.trace("Are we leader for sending to replicas? {} phase={}", isLeader, phase);
if (!isLeader) {
isSubShardLeader = amISubShardLeader(coll, slice, id, doc);
if (isSubShardLeader) {
shardId = cloudDesc.getShardId();
+ leaderReplica = clusterState.getCollection(collection).getSlice(shardId).getLeader();
}
}
- if (!isLeader && !isSubShardLeader) {
- leaderReplica = slice.getLeader();
- if (desc.getName().equals(slice.getLeader().getName()) && slice.getReplicas().size() > 1) {
- assert !desc.getName().equals(slice.getLeader().getName()) : elector == null ? null : elector.getState();
- } else {
- isLeader = true;
- }
- } else {
- leaderReplica = clusterState.getCollection(collection).getReplica(req.getCore().getName());
- }
-
doDefensiveChecks(phase);
// if request is coming from another collection then we want it to be sent to all replicas
// even if its phase is FROMLEADER
String fromCollection = updateCommand.getReq().getParams().get(DISTRIB_FROM_COLLECTION);
- if (DistribPhase.FROMLEADER == phase && !isSubShardLeader && !isLeader && fromCollection == null) {
+ if (DistribPhase.FROMLEADER == phase && !isSubShardLeader && fromCollection == null) {
// we are coming from the leader, just go local - add no urls
forwardToLeader = false;
return null;
@@ -777,7 +770,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), replica, collection, shardId, maxRetriesToFollowers));
}
}
- if (log.isDebugEnabled()) log.debug("We are the leader, forward update to replicas.. {}", nodes);
+ if (log.isDebugEnabled()) log.debug("We are the leader {}, forward update to replicas.. {} {}", req.getCore().getName(), nodes);
return nodes;
} else {
@@ -1058,6 +1051,8 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
}
private void doDefensiveChecks(DistribPhase phase) {
+ // MRM TODO: could use LeaderElector elector = zkController.getLeaderElector(desc.getName());
+
boolean isReplayOrPeersync = (updateCommand.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
if (isReplayOrPeersync) return;
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
index 0dc7e8f..3ad0992 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
@@ -18,18 +18,9 @@ package org.apache.solr.cloud;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.SolrParams;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -40,7 +31,6 @@ import org.slf4j.LoggerFactory;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.TimeUnit;
@Slow
@@ -179,7 +169,8 @@ public class ChaosMonkeySafeLeaderTest extends SolrCloudBridgeTestCase {
commit();
- checkShardConsistency(batchSize == 1, true);
+ // MRM TODO: make test fail on compare fail
+ cluster.getSolrClient().getZkStateReader().checkShardConsistency(DEFAULT_COLLECTION, batchSize == 1, true);
if (VERBOSE) System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
@@ -226,180 +217,4 @@ public class ChaosMonkeySafeLeaderTest extends SolrCloudBridgeTestCase {
indexDoc(doc);
}
- /* Checks both shard replcia consistency and against the control shard.
- * The test will be failed if differences are found.
- */
- protected void checkShardConsistency() throws Exception {
- checkShardConsistency(true, false);
- }
-
- /* Checks shard consistency and optionally checks against the control shard.
- * The test will be failed if differences are found.
- */
- protected void checkShardConsistency(boolean checkVsControl, boolean verbose)
- throws Exception {
- checkShardConsistency(checkVsControl, verbose, null, null);
- }
-
- /* Checks shard consistency and optionally checks against the control shard.
- * The test will be failed if differences are found.
- */
- protected void checkShardConsistency(boolean checkVsControl, boolean verbose, Set<String> addFails, Set<String> deleteFails)
- throws Exception {
-
- Set<String> theShards = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getSlicesMap().keySet();
- String failMessage = null;
- for (String shard : theShards) {
- String shardFailMessage = checkShardConsistency(shard, false, verbose);
- if (shardFailMessage != null && failMessage == null) {
- failMessage = shardFailMessage;
- }
- }
-
- if (failMessage != null) {
- fail(failMessage);
- }
-
- if (!checkVsControl) return;
-
- SolrParams q = params("q","*:*","rows","0", "tests","checkShardConsistency(vsControl)"); // add a tag to aid in debugging via logs
-
- SolrDocumentList controlDocList = controlClient.query(q).getResults();
- long controlDocs = controlDocList.getNumFound();
-
- SolrDocumentList cloudDocList = cloudClient.query(q).getResults();
- long cloudClientDocs = cloudDocList.getNumFound();
-
-
- // now check that the right # are on each shard
-// theShards = shardToJetty.keySet();
-// int cnt = 0;
-// for (String s : theShards) {
-// int times = shardToJetty.get(s).size();
-// for (int i = 0; i < times; i++) {
-// try {
-// CloudJettyRunner cjetty = shardToJetty.get(s).get(i);
-// ZkNodeProps props = cjetty.info;
-// SolrClient client = cjetty.client.solrClient;
-// boolean active = Replica.State.getState(props.getStr(ZkStateReader.STATE_PROP)) == Replica.State.ACTIVE;
-// if (active) {
-// SolrQuery query = new SolrQuery("*:*");
-// query.set("distrib", false);
-// long results = client.query(query).getResults().getNumFound();
-// if (verbose) System.err.println(props + " : " + results);
-// if (verbose) System.err.println("shard:"
-// + props.getStr(ZkStateReader.SHARD_ID_PROP));
-// cnt += results;
-// break;
-// }
-// } catch (Exception e) {
-// ParWork.propagateInterrupt(e);
-// // if we have a problem, try the next one
-// if (i == times - 1) {
-// throw e;
-// }
-// }
-// }
-// }
-
-//controlDocs != cnt ||
- int cnt = -1;
- if (cloudClientDocs != controlDocs) {
- String msg = "document count mismatch. control=" + controlDocs + " sum(shards)="+ cnt + " cloudClient="+cloudClientDocs;
- log.error(msg);
-
- boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient, addFails, deleteFails);
- if (shouldFail) {
- fail(msg);
- }
- }
- }
-
- /**
- * Returns a non-null string if replicas within the same shard do not have a
- * consistent number of documents.
- * If expectFailure==false, the exact differences found will be logged since
- * this would be an unexpected failure.
- * verbose causes extra debugging into to be displayed, even if everything is
- * consistent.
- */
- protected String checkShardConsistency(String shard, boolean expectFailure, boolean verbose)
- throws Exception {
-
- List<JettySolrRunner> solrJetties = cluster.getJettysForShard(DEFAULT_COLLECTION, shard);
- if (solrJetties == null) {
- throw new RuntimeException("shard not found:" + shard);
- }
- long num = -1;
- long lastNum = -1;
- String failMessage = null;
- if (verbose) System.err.println("check const of " + shard);
- int cnt = 0;
- ZkStateReader zkStateReader = cloudClient.getZkStateReader();
-
- DocCollection coll = zkStateReader.getClusterState().getCollection(DEFAULT_COLLECTION);
-
- assertEquals(
- "The client count does not match up with the shard count for slice:"
- + shard,
- coll.getSlice(shard)
- .getReplicasMap().size(), solrJetties.size());
-
- Slice replicas = coll.getSlice(shard);
-
- Replica lastReplica = null;
- for (Replica replica : replicas) {
-
- if (verbose) System.err.println("client" + cnt++);
- if (verbose) System.err.println("Replica:" + replica);
- try (SolrClient client = getClient(replica.getCoreUrl())) {
- try {
- SolrParams query = params("q","*:*", "rows","0", "distrib","false", "tests","checkShardConsistency"); // "tests" is just a tag that won't do anything except be echoed in logs
- num = client.query(query).getResults().getNumFound();
- } catch (SolrException | SolrServerException e) {
- if (verbose) System.err.println("error contacting client: "
- + e.getMessage() + "\n");
- continue;
- }
-
- boolean live = false;
- String nodeName = replica.getNodeName();
- if (zkStateReader.isNodeLive(nodeName)) {
- live = true;
- }
- if (verbose) System.err.println(" live:" + live);
- if (verbose) System.err.println(" num:" + num + "\n");
-
- boolean active = replica.getState() == Replica.State.ACTIVE;
- if (active && live) {
- if (lastNum > -1 && lastNum != num && failMessage == null) {
- failMessage = shard + " is not consistent. Got " + lastNum + " from " + lastReplica.getCoreUrl() + " (previous client)" + " and got " + num + " from " + replica.getCoreUrl();
-
- if (!expectFailure || verbose) {
- System.err.println("######" + failMessage);
- SolrQuery query = new SolrQuery("*:*");
- query.set("distrib", false);
- query.set("fl", "id,_version_");
- query.set("rows", "100000");
- query.set("sort", "id asc");
- query.set("tests", "checkShardConsistency/showDiff");
-
- try (SolrClient lastClient = getClient(lastReplica.getCoreUrl())) {
- SolrDocumentList lst1 = lastClient.query(query).getResults();
- SolrDocumentList lst2 = client.query(query).getResults();
-
- CloudInspectUtil.showDiff(lst1, lst2, lastReplica.getCoreUrl(), replica.getCoreUrl());
- }
- }
-
- }
- lastNum = num;
- lastReplica = replica;
- }
- }
- }
- return failMessage;
-
- }
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
index fbb22e2..db44bf0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
@@ -112,7 +112,7 @@ public class FullSolrCloudDistribCmdsTest extends SolrCloudTestCase {
public static String createAndSetNewDefaultCollection() throws Exception {
final CloudHttp2SolrClient cloudClient = cluster.getSolrClient();
final String name = "test_collection_" + NAME_COUNTER.getAndIncrement();
- CollectionAdminRequest.createCollection(name, "_default", 3, 4).setMaxShardsPerNode(10)
+ CollectionAdminRequest.createCollection(name, "_default", 3, 3).setMaxShardsPerNode(10)
.process(cloudClient);
cloudClient.setDefaultCollection(name);
return name;
@@ -480,6 +480,8 @@ public class FullSolrCloudDistribCmdsTest extends SolrCloudTestCase {
long found = cloudClient.query(params("q", "*:*")).getResults().getNumFound();
assertEquals(numDocs + " found " + found, numDocs, found);
+
+ cluster.getSolrClient().getZkStateReader().checkShardConsistency(collectionName, false, true);
//checkShardConsistency(params("q","*:*", "rows", ""+(1 + numDocs),"_trace","addAll"));
CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
}
diff --git a/solr/solrj/src/java/org/apache/solr/cli/CliCommand.java b/solr/solrj/src/java/org/apache/solr/cli/CliCommand.java
index 0b60b49..ff1f941 100644
--- a/solr/solrj/src/java/org/apache/solr/cli/CliCommand.java
+++ b/solr/solrj/src/java/org/apache/solr/cli/CliCommand.java
@@ -18,6 +18,7 @@
package org.apache.solr.cli;
+import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.zookeeper.ZooKeeper;
@@ -31,6 +32,8 @@ public abstract class CliCommand {
protected ZooKeeper zk;
protected ZkStateReader zkStateReader;
+ protected CloudHttp2SolrClient solrClient;
+
protected PrintStream out;
protected PrintStream err;
private String cmdStr;
@@ -77,6 +80,10 @@ public abstract class CliCommand {
this.zkStateReader = zkStateReader;
}
+ public void setSolrClient(CloudHttp2SolrClient solrClient) {
+ this.solrClient = solrClient;
+ }
+
/**
* get the string used to call this command
* @return
diff --git a/solr/solrj/src/java/org/apache/solr/cli/ClusterCommand.java b/solr/solrj/src/java/org/apache/solr/cli/ClusterCheckCommand.java
similarity index 72%
copy from solr/solrj/src/java/org/apache/solr/cli/ClusterCommand.java
copy to solr/solrj/src/java/org/apache/solr/cli/ClusterCheckCommand.java
index 60795aa..750c724 100644
--- a/solr/solrj/src/java/org/apache/solr/cli/ClusterCommand.java
+++ b/solr/solrj/src/java/org/apache/solr/cli/ClusterCheckCommand.java
@@ -27,24 +27,24 @@ import org.apache.commons.cli.PosixParser;
/**
* ls command for cli
*/
-public class ClusterCommand extends CliCommand {
+public class ClusterCheckCommand extends CliCommand {
private static Options options = new Options();
private String[] args;
private CommandLine cl;
static {
- options.addOption("s", "state", false, "Outputs the Solr state in Zookeeper");
+ //options.addOption("s", "state", false, "Outputs the Solr state in Zookeeper");
}
- public ClusterCommand() {
- super("cluster", "");
+ public ClusterCheckCommand() {
+ super("cluster_check", "collection");
}
private void printHelp() {
HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("cluster [cmd]", options);
+ formatter.printHelp("cluster_state", options);
}
@Override
@@ -56,9 +56,9 @@ public class ClusterCommand extends CliCommand {
throw new CliParseException(ex);
}
args = cl.getArgs();
- if (args.length < 2) {
- throw new CliParseException(getUsageStr());
- }
+// if (args.length < 2) {
+// throw new CliParseException(getUsageStr());
+// }
return this;
}
@@ -68,17 +68,20 @@ public class ClusterCommand extends CliCommand {
if (args.length < 1) {
throw new MalformedCommandException(getUsageStr());
}
- System.out.println("cluster:" + args + " " + options);
- boolean withState = cl.hasOption("s");
- try {
- if (withState) {
- System.out.println("print the layout");
- zkStateReader.getZkClient().printLayoutToStream(out);
- }
- } catch (IllegalArgumentException ex) {
- throw new MalformedPathException(ex.getMessage());
- }
+ if (args.length == 2) {
+ try {
+ String collection = args[1];
+ zkStateReader.checkShardConsistency(collection, false, true);
+
+ } catch (IllegalArgumentException ex) {
+ throw new MalformedPathException(ex.getMessage());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+
return false;
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/cli/ClusterCommand.java b/solr/solrj/src/java/org/apache/solr/cli/ClusterStateCommand.java
similarity index 69%
rename from solr/solrj/src/java/org/apache/solr/cli/ClusterCommand.java
rename to solr/solrj/src/java/org/apache/solr/cli/ClusterStateCommand.java
index 60795aa..63c9bdd 100644
--- a/solr/solrj/src/java/org/apache/solr/cli/ClusterCommand.java
+++ b/solr/solrj/src/java/org/apache/solr/cli/ClusterStateCommand.java
@@ -27,24 +27,24 @@ import org.apache.commons.cli.PosixParser;
/**
* ls command for cli
*/
-public class ClusterCommand extends CliCommand {
+public class ClusterStateCommand extends CliCommand {
private static Options options = new Options();
private String[] args;
private CommandLine cl;
static {
- options.addOption("s", "state", false, "Outputs the Solr state in Zookeeper");
+ //options.addOption("s", "state", false, "Outputs the Solr state in Zookeeper");
}
- public ClusterCommand() {
- super("cluster", "");
+ public ClusterStateCommand() {
+ super("cluster_state", "[path]");
}
private void printHelp() {
HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("cluster [cmd]", options);
+ formatter.printHelp("cluster_state", options);
}
@Override
@@ -56,9 +56,9 @@ public class ClusterCommand extends CliCommand {
throw new CliParseException(ex);
}
args = cl.getArgs();
- if (args.length < 2) {
- throw new CliParseException(getUsageStr());
- }
+// if (args.length < 2) {
+// throw new CliParseException(getUsageStr());
+// }
return this;
}
@@ -68,17 +68,26 @@ public class ClusterCommand extends CliCommand {
if (args.length < 1) {
throw new MalformedCommandException(getUsageStr());
}
- System.out.println("cluster:" + args + " " + options);
- boolean withState = cl.hasOption("s");
- try {
- if (withState) {
- System.out.println("print the layout");
- zkStateReader.getZkClient().printLayoutToStream(out);
- }
- } catch (IllegalArgumentException ex) {
- throw new MalformedPathException(ex.getMessage());
- }
+ if (args.length == 2) {
+ try {
+
+ zkStateReader.getZkClient().printLayoutToStream(out, args[1]);
+
+ } catch (IllegalArgumentException ex) {
+ throw new MalformedPathException(ex.getMessage());
+ }
+ } else {
+ try {
+
+ zkStateReader.getZkClient().printLayoutToStream(out);
+
+ } catch (IllegalArgumentException ex) {
+ throw new MalformedPathException(ex.getMessage());
+ }
+ }
+
+
return false;
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/cli/CreateCollectionCommand.java b/solr/solrj/src/java/org/apache/solr/cli/CreateCollectionCommand.java
new file mode 100644
index 0000000..76960eb
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/cli/CreateCollectionCommand.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cli;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.Parser;
+import org.apache.commons.cli.PosixParser;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+
+/**
+ * create command for cli
+ */
+public class CreateCollectionCommand extends CliCommand {
+
+ private static Options options = new Options();
+ private String[] args;
+ private CommandLine cl;
+
+ static {
+ options.addOption(new Option("c", "config", true, "config set"));
+ // options.addOption(new Option("r", "router", true, "router"));
+ options.addOption(new Option("s", "num_shards", true, "shards"));
+ options.addOption(new Option("rf", "replication_factor", true, "replication factor"));
+ options.addOption(new Option("n", "nrt_replicas", true, "number of nrt replicas"));
+ options.addOption(new Option("p", "pull_replicas", true, "number of pull replicas"));
+ options.addOption(new Option("t", "tlog_replicas", true, "number of tlog replicas"));
+ }
+
+ public CreateCollectionCommand() {
+ super("create_collection", "[-c config] [-s num_shards] [[-rf replication_factor]|[[-n nrt_replicas] [-p pull_replicas] [-t tlog_replicas]] name");
+ }
+
+ @Override
+ public CliCommand parse(String[] cmdArgs) throws CliParseException {
+ Parser parser = new PosixParser();
+ try {
+ cl = parser.parse(options, cmdArgs);
+ } catch (ParseException ex) {
+ throw new CliParseException(ex);
+ }
+ args = cl.getArgs();
+ if (args.length < 2) {
+ throw new CliParseException(getUsageStr());
+ }
+ return this;
+ }
+
+ @Override
+ public boolean exec() throws CliException {
+ // System.out.println("create shards=" + cl.getOptionValue("s", "1") + " " + args[1]);
+ String name = args[1];
+
+ String config = cl.getOptionValue("c", null);
+ // String router = cl.getOptionValue("r", null);
+
+ int numShards = Integer.parseInt(cl.getOptionValue("s", "1"));
+
+ if (cl.hasOption("rf")) {
+ if (cl.hasOption("n") || cl.hasOption("p")|| cl.hasOption("t")) {
+ throw new MalformedCommandException("Cannot specify number of replica types with replication factory");
+ }
+ }
+
+ CollectionAdminRequest.Create create;
+ if (cl.hasOption("rf")) {
+ String replication = cl.getOptionValue("rf", "1");
+ int replicationInt = Integer.parseInt(replication);
+ create = CollectionAdminRequest.createCollection(name, config, numShards, replicationInt);
+ } else {
+ String nrtReplicas = cl.getOptionValue("n", "0");
+ String pullReplics = cl.getOptionValue("p", "0");
+ String tlogLogReplicas = cl.getOptionValue("t", "0");
+
+ int nrtReplicasInt = Integer.parseInt(nrtReplicas);
+ int pullReplicsInt = Integer.parseInt(pullReplics);
+ int tlogLogReplicasInt = Integer.parseInt(tlogLogReplicas);
+
+ create = CollectionAdminRequest.createCollection(name, config, numShards, nrtReplicasInt, tlogLogReplicasInt, pullReplicsInt);
+ }
+
+
+ try {
+ create.process(solrClient);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return true;
+ }
+
+}
diff --git a/solr/solrj/src/java/org/apache/solr/cli/DeleteCollectionCommand.java b/solr/solrj/src/java/org/apache/solr/cli/DeleteCollectionCommand.java
new file mode 100644
index 0000000..df5a68c
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/cli/DeleteCollectionCommand.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cli;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.Parser;
+import org.apache.commons.cli.PosixParser;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+
+/**
+ * create command for cli
+ */
+public class DeleteCollectionCommand extends CliCommand {
+
+ private static Options options = new Options();
+ private String[] args;
+ private CommandLine cl;
+
+ public DeleteCollectionCommand() {
+ super("delete_collection", "name");
+ }
+
+ @Override
+ public CliCommand parse(String[] cmdArgs) throws CliParseException {
+ Parser parser = new PosixParser();
+ try {
+ cl = parser.parse(options, cmdArgs);
+ } catch (ParseException ex) {
+ throw new CliParseException(ex);
+ }
+ args = cl.getArgs();
+ if (args.length < 2) {
+ throw new CliParseException(getUsageStr());
+ }
+ return this;
+ }
+
+ @Override
+ public boolean exec() throws CliException {
+ CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection(args[1]);
+ try {
+ delete.process(solrClient);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return true;
+ }
+
+}
diff --git a/solr/solrj/src/java/org/apache/solr/cli/Stellar.java b/solr/solrj/src/java/org/apache/solr/cli/Stellar.java
index eaf2848..8743dee 100644
--- a/solr/solrj/src/java/org/apache/solr/cli/Stellar.java
+++ b/solr/solrj/src/java/org/apache/solr/cli/Stellar.java
@@ -18,6 +18,7 @@
package org.apache.solr.cli;
+import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
@@ -70,7 +71,7 @@ public class Stellar {
protected ZooKeeper zk;
protected ZkStateReader zkStateReader;
protected String host = "";
-
+ protected CloudHttp2SolrClient solrClient;
public boolean getPrintWatches() {
return printWatches;
@@ -105,7 +106,12 @@ public class Stellar {
new GetAllChildrenNumberCommand().addToMap(commandMapCli);
new VersionCommand().addToMap(commandMapCli);
new AddWatchCommand().addToMap(commandMapCli);
- new ClusterCommand().addToMap(commandMapCli);
+
+ new CreateCollectionCommand().addToMap(commandMapCli);
+ new DeleteCollectionCommand().addToMap(commandMapCli);
+ new ClusterStateCommand().addToMap(commandMapCli);
+ new ClusterCheckCommand().addToMap(commandMapCli);
+
// add all to commandMap
for (Entry<String, CliCommand> entry : commandMapCli.entrySet()) {
@@ -293,6 +299,7 @@ public class Stellar {
zkStateReader = new ZkStateReader(host, Integer.parseInt(cl.getOption("timeout")), 10000);
zk = zkStateReader.getZkClient().getSolrZooKeeper();
+ solrClient = new CloudHttp2SolrClient.Builder(zkStateReader).build();
zkStateReader.createClusterStateWatchersAndUpdate();
}
@@ -427,6 +434,7 @@ public class Stellar {
if (cliCmd != null) {
cliCmd.setZk(zk);
cliCmd.setZkStateReader(zkStateReader);
+ cliCmd.setSolrClient(solrClient);
watch = cliCmd.parse(args).exec();
} else if (!commandMap.containsKey(cmd)) {
usage();
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseCloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseCloudSolrClient.java
index a0fdd4c..a3973db 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseCloudSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseCloudSolrClient.java
@@ -1136,6 +1136,8 @@ public abstract class BaseCloudSolrClient extends SolrClient {
final List<String> theUrlList = new ArrayList<>(); // we populate this as follows...
+ System.out.println("path:" + request.getPath());
+
if (request instanceof V2Request) {
if (!liveNodes.isEmpty()) {
List<String> liveNodesList = new ArrayList<>(liveNodes);
@@ -1145,6 +1147,7 @@ public abstract class BaseCloudSolrClient extends SolrClient {
}
} else if (ADMIN_PATHS.contains(request.getPath())) {
+ System.out.println("isadmin:" + request.getPath() + " livenodes:" + liveNodes);
for (String liveNode : liveNodes) {
theUrlList.add(Utils.getBaseUrlForNodeName(liveNode,
getClusterStateProvider().getClusterProperty(ZkStateReader.URL_SCHEME,"http")));
@@ -1215,7 +1218,7 @@ public abstract class BaseCloudSolrClient extends SolrClient {
"Could not find a healthy node to handle the request, collection names: " + collectionNames);
}
}
-
+ System.out.println("urllist:" + theUrlList);
LBSolrClient.Req req = new LBSolrClient.Req(request, theUrlList);
LBSolrClient.Rsp rsp = getLbClient().request(req);
return rsp.getResponse();
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBSolrClient.java
index 4bfced4..b4a9b58 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBSolrClient.java
@@ -667,9 +667,11 @@ public abstract class LBSolrClient extends SolrClient {
request.setBasePath(wrapper.baseUrl);
return getClient(wrapper.getBaseUrl()).request(request, collection);
} catch (SolrException e) {
+ e.printStackTrace();
// Server is alive but the request was malformed or invalid
throw e;
} catch (SolrServerException e) {
+ e.printStackTrace();
if (e.getRootCause() instanceof IOException) {
ex = e;
moveAliveToDead(wrapper);
@@ -679,6 +681,7 @@ public abstract class LBSolrClient extends SolrClient {
throw e;
}
} catch (Exception e) {
+ e.printStackTrace();
ParWork.propagateInterrupt(e);
throw new SolrServerException(e);
}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 82b8fbc..6bbe74f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -410,6 +410,10 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return new Create(collection, config, ImplicitDocRouter.NAME, null, checkNotNull("shards",shards), numNrtReplicas, numTlogReplicas, numPullReplicas);
}
+ public static Create createCollection(String collection, Integer numShards, Integer numNrtReplicas, Integer numTlogReplicas, Integer numPullReplicas) {
+ return new Create(collection, null, null, numShards, null, numNrtReplicas, numTlogReplicas, numPullReplicas);
+ }
+
/**
* Returns a SolrRequest for modifying a collection with the given properties
* @param collection the collection name
@@ -451,7 +455,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
private Create(String collection, String config, String routerName, Integer numShards, String shards, Integer numNrtReplicas, Integer numTlogReplicas, Integer numPullReplicas) {
super(CollectionAction.CREATE, SolrIdentifierValidator.validateCollectionName(collection));
// NOTE: there's very little we can assert about the args because nothing but "collection" is required by the server
- if ((null != shards) && (null != numShards)) {
+ if ((null != shards) && (null != numShards && numShards != 0)) {
throw new IllegalArgumentException("Can not specify both a numShards and a list of shards");
}
this.configName = config;
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/CloudInspectUtil.java b/solr/solrj/src/java/org/apache/solr/cloud/CloudInspectUtil.java
similarity index 94%
rename from solr/test-framework/src/java/org/apache/solr/cloud/CloudInspectUtil.java
rename to solr/solrj/src/java/org/apache/solr/cloud/CloudInspectUtil.java
index 1ea1a36..50408df 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/CloudInspectUtil.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/CloudInspectUtil.java
@@ -23,13 +23,13 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.ParWork;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.SolrParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -176,7 +176,7 @@ public class CloudInspectUtil {
public static boolean compareResults(SolrClient controlClient, SolrClient cloudClient, Set<String> addFails, Set<String> deleteFails)
throws SolrServerException, IOException {
- SolrParams q = SolrTestCaseJ4.params("q","*:*","rows","0", "tests","checkShardConsistency(vsControl)"); // add a tag to aid in debugging via logs
+ SolrParams q = ZkStateReader.params("q","*:*","rows","0", "tests","checkShardConsistency(vsControl)"); // add a tag to aid in debugging via logs
SolrDocumentList controlDocList = controlClient.query(q).getResults();
long controlDocs = controlDocList.getNumFound();
@@ -185,7 +185,7 @@ public class CloudInspectUtil {
long cloudClientDocs = cloudDocList.getNumFound();
// re-execute the query getting ids
- q = SolrTestCaseJ4.params("q", "*:*", "rows", "100000", "fl", "id", "tests", "checkShardConsistency(vsControl)/getIds"); // add a tag to aid in debugging via logs
+ q = ZkStateReader.params("q", "*:*", "rows", "100000", "fl", "id", "tests", "checkShardConsistency(vsControl)/getIds"); // add a tag to aid in debugging via logs
controlDocList = controlClient.query(q).getResults();
if (controlDocs != controlDocList.getNumFound()) {
log.error("Something changed! control now {}", controlDocList.getNumFound());
@@ -222,7 +222,7 @@ public class CloudInspectUtil {
if (foundId) {
// get versions for those ids that don't match
- q = SolrTestCaseJ4.params("q", ids.toString(), "rows", "100000", "fl", "id,_version_",
+ q = ZkStateReader.params("q", ids.toString(), "rows", "100000", "fl", "id,_version_",
"sort", "id asc", "tests",
"checkShardConsistency(vsControl)/getVers"); // add a tag to aid in
// debugging via logs
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java
index 24006d4..aea1ffa 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java
@@ -40,6 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.zookeeper.Watcher.Event.KeeperState.AuthFailed;
+import static org.apache.zookeeper.Watcher.Event.KeeperState.Closed;
import static org.apache.zookeeper.Watcher.Event.KeeperState.Disconnected;
import static org.apache.zookeeper.Watcher.Event.KeeperState.Expired;
@@ -88,7 +89,7 @@ public class ConnectionManager implements Watcher, Closeable {
}
public ZooKeeper getKeeper() {
- if (isClosed) {
+ if (keeper.getState() == ZooKeeper.States.CLOSED) {
throw new AlreadyClosedException(this + " SolrZkClient is not currently connected state=" + keeper.getState());
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
index 1f5a190..3a895cb 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
@@ -980,7 +980,7 @@ public class SolrZkClient implements Closeable {
// }
int lines = dataString.split("\\r\\n|\\r|\\n").length;
if ((stat != null && stat.getDataLength() < MAX_BYTES_FOR_ZK_LAYOUT_DATA_SHOW && lines < 4) || path.endsWith("state.json") || path
- .endsWith("security.json") || (path.endsWith("solrconfig.xml") && Boolean.getBoolean("solr.tests.printsolrconfig")) || path.endsWith("_statupdates")) {
+ .endsWith("security.json") || (path.endsWith("solrconfig.xml") && Boolean.getBoolean("solr.tests.printsolrconfig")) || path.endsWith("_statupdates") || path.contains("/terms/")) {
// if (path.endsWith(".xml")) {
// // this is the cluster state in xml format - lets pretty print
// dataString = prettyPrint(path, dataString);
@@ -1038,6 +1038,12 @@ public class SolrZkClient implements Closeable {
out.println(sb.toString());
}
+ public void printLayoutToStream(PrintStream out, String path) {
+ StringBuilder sb = new StringBuilder(512);
+ printLayout(path, 0, sb);
+ out.println(sb.toString());
+ }
+
public void printLayoutToFile(Path file) {
StringBuilder sb = new StringBuilder(512);
printLayout("/",0, sb);
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index cffd8ea..57eb20b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -50,14 +50,22 @@ import java.util.function.Predicate;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.apache.solr.cloud.CloudInspectUtil;
import org.apache.solr.common.AlreadyClosedException;
import org.apache.solr.common.Callable;
import org.apache.solr.common.ParWork;
import org.apache.solr.common.SolrCloseable;
+import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.CloseTracker;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.ObjectReleaseTracker;
@@ -1414,9 +1422,16 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
byte[] data = null;
IOUtils.closeQuietly(stateUpdateWatcher);
- data = getZkClient().getData(stateUpdatesPath, stateUpdateWatcher, null, true);
+
+ try {
+ data = getZkClient().getData(stateUpdatesPath, stateUpdateWatcher, null, true);
+ } catch (NoNodeException e) {
+ log.info("No node found for {}", stateUpdatesPath);
+ return;
+ }
if (data == null) {
+ log.info("No data found for {}", stateUpdatesPath);
return;
}
@@ -2737,6 +2752,199 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
};
}
+ /* Checks both shard replcia consistency and against the control shard.
+ * The test will be failed if differences are found.
+ */
+ protected void checkShardConsistency(String collection) throws Exception {
+ checkShardConsistency(collection, true, false);
+ }
+
+ /* Checks shard consistency and optionally checks against the control shard.
+ * The test will be failed if differences are found.
+ */
+ public void checkShardConsistency(String collection, boolean checkVsControl, boolean verbose)
+ throws Exception {
+ checkShardConsistency(collection, checkVsControl, verbose, null, null);
+ }
+
+ /* Checks shard consistency and optionally checks against the control shard.
+ * The test will be failed if differences are found.
+ */
+ protected void checkShardConsistency(String collection, boolean checkVsControl, boolean verbose, Set<String> addFails, Set<String> deleteFails)
+ throws Exception {
+
+ Set<String> theShards = getClusterState().getCollection(collection).getSlicesMap().keySet();
+ String failMessage = null;
+ for (String shard : theShards) {
+ String shardFailMessage = checkShardConsistency(collection, shard, false, verbose);
+ if (shardFailMessage != null && failMessage == null) {
+ failMessage = shardFailMessage;
+ }
+ }
+
+ if (failMessage != null) {
+ System.err.println(failMessage);
+ // MRM TODO: fail test if from tests
+
+ throw new AssertionError(failMessage);
+ }
+
+ if (!checkVsControl) return;
+
+ SolrParams q = params("q","*:*","rows","0", "tests","checkShardConsistency(vsControl)"); // add a tag to aid in debugging via logs
+
+// SolrDocumentList controlDocList = controlClient.query(q).getResults();
+// long controlDocs = controlDocList.getNumFound();
+//
+// SolrDocumentList cloudDocList = cloudClient.query(q).getResults();
+// long cloudClientDocs = cloudDocList.getNumFound();
+
+
+ // now check that the right # are on each shard
+ // theShards = shardToJetty.keySet();
+ // int cnt = 0;
+ // for (String s : theShards) {
+ // int times = shardToJetty.get(s).size();
+ // for (int i = 0; i < times; i++) {
+ // try {
+ // CloudJettyRunner cjetty = shardToJetty.get(s).get(i);
+ // ZkNodeProps props = cjetty.info;
+ // SolrClient client = cjetty.client.solrClient;
+ // boolean active = Replica.State.getState(props.getStr(ZkStateReader.STATE_PROP)) == Replica.State.ACTIVE;
+ // if (active) {
+ // SolrQuery query = new SolrQuery("*:*");
+ // query.set("distrib", false);
+ // long results = client.query(query).getResults().getNumFound();
+ // if (verbose) System.err.println(props + " : " + results);
+ // if (verbose) System.err.println("shard:"
+ // + props.getStr(ZkStateReader.SHARD_ID_PROP));
+ // cnt += results;
+ // break;
+ // }
+ // } catch (Exception e) {
+ // ParWork.propagateInterrupt(e);
+ // // if we have a problem, try the next one
+ // if (i == times - 1) {
+ // throw e;
+ // }
+ // }
+ // }
+ // }
+
+ //controlDocs != cnt ||
+ int cnt = -1;
+// if (cloudClientDocs != controlDocs) {
+// String msg = "document count mismatch. control=" + controlDocs + " sum(shards)="+ cnt + " cloudClient="+cloudClientDocs;
+// log.error(msg);
+//
+// boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient, addFails, deleteFails);
+// if (shouldFail) {
+// fail(msg);
+// }
+// }
+ }
+
+ /**
+ * Returns a non-null string if replicas within the same shard do not have a
+ * consistent number of documents.
+ * If expectFailure==false, the exact differences found will be logged since
+ * this would be an unexpected failure.
+ * verbose causes extra debugging into to be displayed, even if everything is
+ * consistent.
+ */
+ protected String checkShardConsistency(String collection, String shard, boolean expectFailure, boolean verbose)
+ throws Exception {
+
+
+ long num = -1;
+ long lastNum = -1;
+ String failMessage = null;
+ if (verbose) System.err.println("check const of " + shard);
+ int cnt = 0;
+
+ DocCollection coll = getClusterState().getCollection(collection);
+
+ Slice replicas = coll.getSlice(shard);
+
+ Replica lastReplica = null;
+ for (Replica replica : replicas) {
+
+ if (verbose) System.err.println("client" + cnt++);
+ if (verbose) System.err.println("Replica:" + replica);
+ try (SolrClient client = getHttpClient(replica.getCoreUrl())) {
+ try {
+ SolrParams query = params("q","*:*", "rows","0", "distrib","false", "tests","checkShardConsistency"); // "tests" is just a tag that won't do anything except be echoed in logs
+ num = client.query(query).getResults().getNumFound();
+ } catch (SolrException | SolrServerException e) {
+ if (verbose) System.err.println("error contacting client: "
+ + e.getMessage() + "\n");
+ continue;
+ }
+
+ boolean live = false;
+ String nodeName = replica.getNodeName();
+ if (isNodeLive(nodeName)) {
+ live = true;
+ }
+ if (verbose) System.err.println(" live:" + live);
+ if (verbose) System.err.println(" num:" + num + "\n");
+
+ boolean active = replica.getState() == Replica.State.ACTIVE;
+ if (active && live) {
+ if (lastNum > -1 && lastNum != num && failMessage == null) {
+ failMessage = shard + " is not consistent. Got " + lastNum + " from " + lastReplica.getCoreUrl() + " (previous client)" + " and got " + num + " from " + replica.getCoreUrl();
+
+ if (!expectFailure || verbose) {
+ System.err.println("######" + failMessage);
+ SolrQuery query = new SolrQuery("*:*");
+ query.set("distrib", false);
+ query.set("fl", "id,_version_");
+ query.set("rows", "100000");
+ query.set("sort", "id asc");
+ query.set("tests", "checkShardConsistency/showDiff");
+
+ try (SolrClient lastClient = getHttpClient(lastReplica.getCoreUrl())) {
+ SolrDocumentList lst1 = lastClient.query(query).getResults();
+ SolrDocumentList lst2 = client.query(query).getResults();
+
+ CloudInspectUtil.showDiff(lst1, lst2, lastReplica.getCoreUrl(), replica.getCoreUrl());
+ }
+ }
+
+ }
+ lastNum = num;
+ lastReplica = replica;
+ }
+ }
+ }
+ return failMessage;
+
+ }
+
+ /**
+ * Generates the correct SolrParams from an even list of strings.
+ * A string in an even position will represent the name of a parameter, while the following string
+ * at position (i+1) will be the assigned value.
+ *
+ * @param params an even list of strings
+ * @return the ModifiableSolrParams generated from the given list of strings.
+ */
+ public static ModifiableSolrParams params(String... params) {
+ if (params.length % 2 != 0) throw new RuntimeException("Params length should be even");
+ ModifiableSolrParams msp = new ModifiableSolrParams();
+ for (int i=0; i<params.length; i+=2) {
+ msp.add(params[i], params[i+1]);
+ }
+ return msp;
+ }
+
+ public Http2SolrClient getHttpClient(String baseUrl) {
+ Http2SolrClient client = new Http2SolrClient.Builder(baseUrl)
+ .idleTimeout(Integer.getInteger("socketTimeout", 30000))
+ .build();
+ return client;
+ }
+
private class CloudCollectionsListenerConsumer implements Consumer<CloudCollectionsListener> {
private final Set<String> oldCollections;
private final Set<String> newCollections;
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
index 69fec43..d602631 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
@@ -26,9 +26,7 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
@@ -49,7 +47,6 @@ import org.apache.solr.client.solrj.cloud.SocketProxy;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.embedded.SSLConfig;
-import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
@@ -608,27 +605,21 @@ public class MiniSolrCloudCluster {
ZkStateReader reader = solrClient.getZkStateReader();
reader.aliasesManager.applyModificationAndExportToZk(aliases -> Aliases.EMPTY);
- Map<String,Exception> errors = new HashMap<>();
- final Set<String> collections = reader.getClusterState().getCollectionsMap().keySet();
- collections.parallelStream().forEach(collection -> {
- try {
- CollectionAdminRequest.deleteCollection(collection).process(solrClient);
- } catch (BaseHttpSolrClient.RemoteSolrException e) {
- if (!e.getMessage().contains("Could not find") && !e.getMessage().contains("Error handling 'UNLOAD' action") && !e.getMessage().contains("Cannot unload non-existent core")) {
- errors.put(collection, e);
- }
- } catch (Exception e) {
- errors.put(collection, e);
- }
- });
- if (!errors.isEmpty()) {
- errors.forEach((k,v) -> {
- log.error("Failed to delete collection {}", k, v);
+ final Set<String> collections = reader.getClusterState().getCollectionsMap().keySet();
+ try (ParWork work = new ParWork(this, false, true)) {
+ collections.forEach(collection -> {
+ work.collect("", ()->{
+ try {
+ CollectionAdminRequest.deleteCollection(collection).process(solrClient);
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ });
});
- throw errors.values().iterator().next();
}
+
// may be deleted, but may not be gone yet - we only wait to not see it in ZK, not for core unloads
for (JettySolrRunner jetty : jettys) {
CoreContainer cc = jetty.getCoreContainer();
diff --git a/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml b/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml
index a207c3b..91acf82 100644
--- a/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml
+++ b/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml
@@ -20,14 +20,11 @@
<Appenders>
<Console name="STDERR_COLOR" target="SYSTEM_ERR">
- <LogstashLayout dateTimeFormatPattern="yyyy-MM-dd'T'HH:mm:ss.SSSZZZ"
- eventTemplateUri="classpath:LogstashJsonEventLayoutV1.json"
- prettyPrintEnabled="true"
- stackTraceEnabled="true"/>
+ <PatternLayout pattern="%style{%-4r}{yellow} %highlight{%maxLen{%-5p}{6}} %style{(%maxLen{%t}{8})}{yellow,bold} [%style{%X{node_name} %X{core}}{cyan}] %style{%c{1.}}{cyan} %highlight{%m %notEmpty{%ex}}\n"/>
</Console>
<File name="FILE" fileName="${sys:user.home}/solr-test.log" immediateFlush="false" append="false">
- <PatternLayout pattern="%style{%-4r}{yellow} %highlight{%maxLen{%-1p}{20}} %style{(%t)}{yellow,bold} [%style{%X{node_name} %X{core}}{cyan}] %style{%c{1.}}{cyan} %highlight{%m %notEmpty{%ex}}\n"/>
+ <PatternLayout pattern="%style{%-4r}{yellow} %highlight{%maxLen{%-5p}{6}} %style{(%t)}{yellow,bold} [%style{%X{node_name} %X{core}}{cyan}] %style{%c{1.}}{cyan} %highlight{%m %notEmpty{%ex}}\n"/>
</File>
</Appenders>
@@ -39,7 +36,12 @@
<AsyncLogger name="org.apache.hadoop" level="WARN"/>
<AsyncLogger name="org.apache.directory" level="WARN"/>
<AsyncLogger name="org.apache.solr.hadoop" level="INFO"/>
- <AsyncLogger name="org.eclipse.jetty" level="DEBUG"/>
+ <AsyncLogger name="org.eclipse.jetty" level="INFO"/>
+
+ <AsyncLogger name="org.apache.solr.core.SolrCore" level="DEBUG"/>
+ <AsyncLogger name="org.apache.solr.core.DefaultSolrCore" level="DEBUG"/>
+ <AsyncLogger name="org.apache.solr.core.SolrCoreState" level="DEBUG"/>
+
<AsyncLogger name="org.apache.solr.handler.admin.CollectionsHandler" level="INFO"/>
<AsyncLogger name="org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler" level="INFO"/>
<AsyncLogger name="org.apache.solr.cloud.api.collections.CreateCollectionCmd" level="INFO"/>
@@ -52,13 +54,16 @@
<AsyncLogger name="org.apache.solr.client.solrj.impl.LBSolrClient" level="INFO"/>
<AsyncLogger name="org.apache.solr.cloud.ZkController" level="INFO"/>
<AsyncLogger name="org.apache.solr.common.cloud.ZkMaintenanceUtils" level="INFO"/>
- <AsyncLogger name="org.apache.solr.update.processor.DistributedZkUpdateProcessor" level="DEBUG"/>
- <AsyncLogger name="org.apache.solr.update.processor.DistributedUpdateProcessor" level="DEBUG"/>
- <AsyncLogger name="org.apache.solr.update.SolrCmdDistributor" level="DEBUG"/>
- <AsyncLogger name="org.apache.solr.update.processor.LogUpdateProcessorFactory" level="DEBUG"/>
+ <AsyncLogger name="org.apache.solr.update.processor.DistributedZkUpdateProcessor" level="INFO"/>
+ <AsyncLogger name="org.apache.solr.update.processor.DistributedUpdateProcessor" level="INFO"/>
+ <AsyncLogger name="org.apache.solr.update.SolrCmdDistributor" level="INFO"/>
+ <AsyncLogger name="org.apache.solr.update.processor.LogUpdateProcessorFactory" level="INFO"/>
+
+ <AsyncLogger name="org.apache.solr.common.ParWork" level="INFO"/>
+
- <AsyncLogger name="com.google.inject.servlet" level="DEBUG"/>
- <AsyncLogger name="org.apache.solr.client.solrj.impl.Http2SolrClient" level="DEBUG"/>
+ <AsyncLogger name="com.google.inject.servlet" level="INFO"/>
+ <AsyncLogger name="org.apache.solr.client.solrj.impl.Http2SolrClient" level="INFO"/>
<AsyncRoot level="INFO">
<AppenderRef ref="STDERR_COLOR"/>