You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ds...@apache.org on 2023/07/12 18:04:39 UTC
[solr] branch branch_9_3 updated: SOLR-16507: SplitShardCmd shouldn't use NodeStateProvider (#1775)
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_9_3
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9_3 by this push:
new 1e7a35a6fce SOLR-16507: SplitShardCmd shouldn't use NodeStateProvider (#1775)
1e7a35a6fce is described below
commit 1e7a35a6fcebc0caf67ceb46f4bdb50a396813f7
Author: David Smiley <ds...@apache.org>
AuthorDate: Wed Jul 12 13:36:31 2023 -0400
SOLR-16507: SplitShardCmd shouldn't use NodeStateProvider (#1775)
Objective is to limit use of NodeStateProvider to where it's best.
Co-authored-by: vinayak hegde <vi...@gmail.com>
---
solr/CHANGES.txt | 2 +
.../solr/cloud/api/collections/SplitShardCmd.java | 59 ++++++++++------------
2 files changed, 28 insertions(+), 33 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e38b4771a21..806066a34b5 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -330,6 +330,8 @@ Other Changes
* SOLR-16715: Replace new HashMap(int) and new HashSet(int) with CollectionUtil.newHashMap / CollectionUtil.newHashSet (Kevin Risden)
+* SOLR-16507: Change SplitShardCmd to not use NodeStateProvider (Vinayak Hegde, David Smiley)
+
* SOLR-15703: replace all SolrException.log usage in Solr to just call log.error(...) directly (Kevin Risden)
* SOLR-16604: Use Solr Client Builders directly in unit tests instead of delegating to SolrTestCaseJ4. (Eric Pugh, David Smiley)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index cfbc21743e2..0060eb676af 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -28,6 +28,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.DE
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
@@ -41,17 +42,17 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
-import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.VersionedData;
-import org.apache.solr.client.solrj.impl.NodeValueFetcher;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.cloud.DistributedClusterStateUpdater;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker;
import org.apache.solr.cloud.overseer.OverseerAction;
-import org.apache.solr.cluster.placement.impl.ReplicaMetricImpl;
import org.apache.solr.common.LinkedHashMapWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
@@ -872,45 +873,37 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
Replica parentShardLeader,
SolrIndexSplitter.SplitMethod method,
SolrCloudManager cloudManager)
- throws SolrException {
+ throws SolrException, IOException {
+
// check that enough disk space is available on the parent leader node
// otherwise the actual index splitting will always fail
- NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();
- Map<String, Object> nodeValues =
- nodeStateProvider.getNodeValues(
- parentShardLeader.getNodeName(),
- Collections.singletonList(NodeValueFetcher.Tags.FREEDISK.tagName));
- Map<String, Map<String, List<Replica>>> infos =
- nodeStateProvider.getReplicaInfo(
- parentShardLeader.getNodeName(),
- Collections.singletonList(ReplicaMetricImpl.INDEX_SIZE_GB.getInternalName()));
- if (infos.get(collection) == null || infos.get(collection).get(shard) == null) {
+
+ String replicaName = Utils.parseMetricsReplicaName(collection, parentShardLeader.getCoreName());
+ String indexSizeMetricName =
+ "solr.core." + collection + "." + shard + "." + replicaName + ":INDEX.sizeInBytes";
+ String freeDiskSpaceMetricName = "solr.node:CONTAINER.fs.usableSpace";
+
+ ModifiableSolrParams params =
+ new ModifiableSolrParams()
+ .add("key", indexSizeMetricName)
+ .add("key", freeDiskSpaceMetricName);
+ SolrResponse rsp =
+ cloudManager.request(
+ new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/metrics", params));
+
+ Number size = (Number) rsp.getResponse().findRecursive("metrics", indexSizeMetricName);
+ if (size == null) {
log.warn("cannot verify information for parent shard leader");
return;
}
- // find the leader
- List<Replica> lst = infos.get(collection).get(shard);
- Double indexSize = null;
- for (Replica info : lst) {
- if (info.getCoreName().equals(parentShardLeader.getCoreName())) {
- Number size = (Number) info.get(ReplicaMetricImpl.INDEX_SIZE_GB.getInternalName());
- if (size == null) {
- log.warn("cannot verify information for parent shard leader");
- return;
- }
- indexSize = ReplicaMetricImpl.INDEX_SIZE_GB.convert(size);
- break;
- }
- }
- if (indexSize == null) {
- log.warn("missing replica information for parent shard leader");
- return;
- }
- Number freeSize = (Number) nodeValues.get(NodeValueFetcher.Tags.FREEDISK.tagName);
+ double indexSize = size.doubleValue();
+
+ Number freeSize = (Number) rsp.getResponse().findRecursive("metrics", freeDiskSpaceMetricName);
if (freeSize == null) {
log.warn("missing node disk space information for parent shard leader");
return;
}
+
// 100% more for REWRITE, 5% more for LINK
double neededSpace =
method == SolrIndexSplitter.SplitMethod.REWRITE ? 2.0 * indexSize : 1.05 * indexSize;