You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 10:06:58 UTC
[lucene] 11/17: SOLR-15019: Refactor to use NodeMetric /
ReplicaMetric. More cleanups.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-15019
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 7d3c1b974fbd7544cd07db746d1b4c3699c4ecdc
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Mon Dec 21 19:18:33 2020 +0100
SOLR-15019: Refactor to use NodeMetric / ReplicaMetric. More cleanups.
---
.../solr/cluster/placement/AttributeFetcher.java | 17 ++--
.../solr/cluster/placement/AttributeValues.java | 7 +-
.../{ReplicaMetric.java => MetricAttribute.java} | 38 ++-------
.../apache/solr/cluster/placement/NodeMetric.java | 95 ++++++++++++++++++++++
.../solr/cluster/placement/ReplicaMetric.java | 63 +-------------
.../solr/cluster/placement/ReplicaMetrics.java | 3 +-
.../placement/impl/AttributeFetcherImpl.java | 51 +++++-------
.../placement/impl/AttributeValuesImpl.java | 23 ++----
.../placement/impl/CollectionMetricsBuilder.java | 47 ++++++-----
.../cluster/placement/AttributeFetcherForTest.java | 2 +-
.../apache/solr/cluster/placement/Builders.java | 40 ++++-----
.../solr/cluster/placement/BuildersTest.java | 16 ++--
.../impl/PlacementPluginIntegrationTest.java | 29 ++++---
.../solrj/impl/SolrClientNodeStateProvider.java | 2 +-
14 files changed, 218 insertions(+), 215 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/AttributeFetcher.java b/solr/core/src/java/org/apache/solr/cluster/placement/AttributeFetcher.java
index af63baa..f7e7908 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/AttributeFetcher.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/AttributeFetcher.java
@@ -69,19 +69,20 @@ public interface AttributeFetcher {
AttributeFetcher requestNodeEnvironmentVariable(String name);
/**
- * Request a node metric from each node. To get the value use {@link AttributeValues#getMetric(Node, String, NodeMetricRegistry)}
- * @param metricName name of the metric (within the registry)
- * @param registry one of the node-level metric registries
+ * Request a node metric from each node. To get the value use {@link AttributeValues#getNodeMetric(Node, NodeMetric)}
+ * @param metric metric to retrieve (see {@link NodeMetric})
*/
- AttributeFetcher requestNodeMetric(String metricName, NodeMetricRegistry registry);
+ AttributeFetcher requestNodeMetric(NodeMetric<?> metric);
/**
- * Requests any metric from any metric registry on each node, using a fully-qualified metric key,
+ * Rfrom any metric registry on each node, using a fully-qualified metric key,
* for example <code>solr.jvm:system.properties:user.name</code>.
* To get the value use {@link AttributeValues#getNodeMetric(Node, String)}
* @param metricKey fully-qualified metric key
*/
- AttributeFetcher requestNodeMetric(String metricKey);
+ default AttributeFetcher requestNodeMetric(String metricKey) {
+ return requestNodeMetric(new NodeMetric<>(metricKey));
+ }
/**
* Request collection-level metrics. To get the values use {@link AttributeValues#getCollectionMetrics(String)}.
@@ -124,8 +125,4 @@ public interface AttributeFetcher {
*/
SOLR_JETTY
}
-
- enum DiskHardwareType {
- SSD, ROTATIONAL
- }
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/AttributeValues.java b/solr/core/src/java/org/apache/solr/cluster/placement/AttributeValues.java
index ca0d1b0..88400b8 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/AttributeValues.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/AttributeValues.java
@@ -28,11 +28,6 @@ public interface AttributeValues {
Optional<Integer> getCoresCount(Node node);
/**
- * For the given node: Hardware type of the disk partition where cores are stored
- */
- Optional<AttributeFetcher.DiskHardwareType> getDiskType(Node node);
-
- /**
* For the given node: Free disk size in Gigabytes of the partition on which cores are stored
*/
Optional<Double> getFreeDisk(Node node);
@@ -65,7 +60,7 @@ public interface AttributeValues {
/**
* For the given node: metric of specific name and registry
*/
- Optional<Object> getMetric(Node node, String metricName, AttributeFetcher.NodeMetricRegistry registry);
+ <T> Optional<T> getNodeMetric(Node node, NodeMetric<T> metric);
/**
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetric.java b/solr/core/src/java/org/apache/solr/cluster/placement/MetricAttribute.java
similarity index 65%
copy from solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetric.java
copy to solr/core/src/java/org/apache/solr/cluster/placement/MetricAttribute.java
index 0f01889..f0a8a15 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetric.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/MetricAttribute.java
@@ -24,9 +24,10 @@ import java.util.function.Function;
* internal metric name (as reported in <code>solr.core.[collection].[replica]</code> registry)
* and the desired format/unit conversion.
*/
-public class ReplicaMetric<T> {
+public class MetricAttribute<T> {
+
+ public static final double GB = 1024 * 1024 * 1024;
- private static final double GB = 1024 * 1024 * 1024;
@SuppressWarnings("unchecked")
private final Function<Object, T> IDENTITY_CONVERTER = v -> {
try {
@@ -36,36 +37,15 @@ public class ReplicaMetric<T> {
}
};
- public static final ReplicaMetric<Double> INDEX_SIZE_GB = new ReplicaMetric<>("sizeGB", "INDEX.sizeInBytes",
- v -> {
- double sizeInBytes;
- if (!(v instanceof Number)) {
- if (v == null) {
- return null;
- }
- try {
- sizeInBytes = Double.valueOf(String.valueOf(v)).doubleValue();
- } catch (Exception nfe) {
- return null;
- }
- } else {
- sizeInBytes = ((Number) v).doubleValue();
- }
- return sizeInBytes / GB;
- });
-
- public static final ReplicaMetric<Double> QUERY_RATE_1MIN = new ReplicaMetric<>("queryRate", "QUERY./select.requestTimes:1minRate");
- public static final ReplicaMetric<Double> UPDATE_RATE_1MIN = new ReplicaMetric<>("updateRate", "UPDATE./update.requestTimes:1minRate");
-
- private final String name;
- private final String internalName;
- private final Function<Object, T> converter;
+ protected final String name;
+ protected final String internalName;
+ protected final Function<Object, T> converter;
- public ReplicaMetric(String name, String internalName) {
+ public MetricAttribute(String name, String internalName) {
this(name, internalName, null);
}
- public ReplicaMetric(String name, String internalName, Function<Object, T> converter) {
+ public MetricAttribute(String name, String internalName, Function<Object, T> converter) {
Objects.requireNonNull(name);
Objects.requireNonNull(internalName);
this.name = name;
@@ -97,7 +77,7 @@ public class ReplicaMetric<T> {
if (o == null || getClass() != o.getClass()) {
return false;
}
- ReplicaMetric<?> that = (ReplicaMetric<?>) o;
+ MetricAttribute<?> that = (MetricAttribute<?>) o;
return name.equals(that.name) && internalName.equals(that.internalName) && converter.equals(that.converter);
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/NodeMetric.java b/solr/core/src/java/org/apache/solr/cluster/placement/NodeMetric.java
new file mode 100644
index 0000000..3aea6d2
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/NodeMetric.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cluster.placement;
+
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Node metric wrapper that defines a short symbolic name of the metric, the corresponding
+ * metric registry and the internal metric name, as well as the desired format/unit conversion.
+ */
+public class NodeMetric<T> extends MetricAttribute<T> {
+
+ /** System load average metric. */
+ public static final NodeMetric<Double> SYSLOAD_AVG =
+ new NodeMetric<>("sysLoadAvg", AttributeFetcher.NodeMetricRegistry.SOLR_JVM, "os.systemLoadAverage");
+
+ /** Available processors metric. */
+ public static final NodeMetric<Integer> AVAILABLE_PROCESSORS =
+ new NodeMetric<>("availableProcessors", AttributeFetcher.NodeMetricRegistry.SOLR_JVM, "os.availableProcessors");
+
+ private final AttributeFetcher.NodeMetricRegistry registry;
+
+ public NodeMetric(String name, AttributeFetcher.NodeMetricRegistry registry, String internalName) {
+ this(name, registry, internalName, null);
+ }
+
+ public NodeMetric(String name, AttributeFetcher.NodeMetricRegistry registry, String internalName, Function<Object, T> converter) {
+ super(name, internalName, converter);
+ Objects.requireNonNull(registry);
+ this.registry = registry;
+ }
+
+ public NodeMetric(String key) {
+ this(key, null);
+ }
+
+ public NodeMetric(String key, Function<Object, T> converter) {
+ super(key, key, converter);
+ this.registry = null;
+ }
+
+ public AttributeFetcher.NodeMetricRegistry getRegistry() {
+ return registry;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ NodeMetric<?> that = (NodeMetric<?>) o;
+ return registry == that.registry;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), registry);
+ }
+
+ @Override
+ public String toString() {
+ if (registry != null) {
+ return "NodeMetric{" +
+ "name='" + name + '\'' +
+ ", internalName='" + internalName + '\'' +
+ ", converter=" + converter +
+ ", registry=" + registry +
+ '}';
+ } else {
+ return "NodeMetric{key=" + internalName + "}";
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetric.java b/solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetric.java
index 0f01889..98bc5d4 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetric.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetric.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.cluster.placement;
-import java.util.Objects;
import java.util.function.Function;
/**
@@ -24,17 +23,7 @@ import java.util.function.Function;
* internal metric name (as reported in <code>solr.core.[collection].[replica]</code> registry)
* and the desired format/unit conversion.
*/
-public class ReplicaMetric<T> {
-
- private static final double GB = 1024 * 1024 * 1024;
- @SuppressWarnings("unchecked")
- private final Function<Object, T> IDENTITY_CONVERTER = v -> {
- try {
- return (T) v;
- } catch (ClassCastException cce) {
- return null;
- }
- };
+public class ReplicaMetric<T> extends MetricAttribute<T> {
public static final ReplicaMetric<Double> INDEX_SIZE_GB = new ReplicaMetric<>("sizeGB", "INDEX.sizeInBytes",
v -> {
@@ -57,57 +46,11 @@ public class ReplicaMetric<T> {
public static final ReplicaMetric<Double> QUERY_RATE_1MIN = new ReplicaMetric<>("queryRate", "QUERY./select.requestTimes:1minRate");
public static final ReplicaMetric<Double> UPDATE_RATE_1MIN = new ReplicaMetric<>("updateRate", "UPDATE./update.requestTimes:1minRate");
- private final String name;
- private final String internalName;
- private final Function<Object, T> converter;
-
public ReplicaMetric(String name, String internalName) {
- this(name, internalName, null);
+ super(name, internalName);
}
public ReplicaMetric(String name, String internalName, Function<Object, T> converter) {
- Objects.requireNonNull(name);
- Objects.requireNonNull(internalName);
- this.name = name;
- this.internalName = internalName;
- if (converter == null) {
- this.converter = IDENTITY_CONVERTER;
- } else {
- this.converter = converter;
- }
- }
-
- public String getName() {
- return name;
- }
-
- public String getInternalName() {
- return internalName;
- }
-
- public T convert(Object value) {
- return converter.apply(value);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ReplicaMetric<?> that = (ReplicaMetric<?>) o;
- return name.equals(that.name) && internalName.equals(that.internalName) && converter.equals(that.converter);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(name, internalName, converter);
- }
-
- @Override
- public String toString() {
- return name + "(" + internalName + ")";
+ super(name, internalName, converter);
}
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetrics.java b/solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetrics.java
index 0d13b11..bab16b2 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetrics.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/ReplicaMetrics.java
@@ -24,6 +24,5 @@ import java.util.Optional;
*/
public interface ReplicaMetrics {
- Double getReplicaSizeGB();
- Optional<Object> getReplicaMetric(String metricName);
+ <T> Optional<T> getReplicaMetric(ReplicaMetric<T> metric);
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
index bcd0eb6..bcba85c 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
@@ -24,6 +24,7 @@ import org.apache.solr.cluster.placement.AttributeFetcher;
import org.apache.solr.cluster.placement.AttributeValues;
import org.apache.solr.cluster.Node;
import org.apache.solr.cluster.placement.CollectionMetrics;
+import org.apache.solr.cluster.placement.NodeMetric;
import org.apache.solr.cluster.placement.ReplicaMetric;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Replica;
@@ -47,7 +48,7 @@ public class AttributeFetcherImpl implements AttributeFetcher {
boolean requestedNodeHeapUsage;
boolean requestedNodeSystemLoadAverage;
Set<String> requestedNodeSystemSnitchTags = new HashSet<>();
- Set<String> requestedNodeMetricSnitchTags = new HashSet<>();
+ Set<NodeMetric<?>> requestedNodeMetricSnitchTags = new HashSet<>();
Map<SolrCollection, Set<ReplicaMetric<?>>> requestedCollectionMetrics = new HashMap<>();
Set<Node> nodes = Collections.emptySet();
@@ -106,20 +107,22 @@ public class AttributeFetcherImpl implements AttributeFetcher {
}
@Override
- public AttributeFetcher requestNodeMetric(String metricName, NodeMetricRegistry registry) {
- requestedNodeMetricSnitchTags.add(getMetricSnitchTag(metricName, registry));
+ public AttributeFetcher requestNodeMetric(NodeMetric<?> metric) {
+ requestedNodeMetricSnitchTags.add(metric);
return this;
}
@Override
public AttributeFetcher requestNodeMetric(String metricKey) {
- requestedNodeMetricSnitchTags.add(getMetricKeySnitchTag(metricKey));
+ requestedNodeMetricSnitchTags.add(new NodeMetric<>(metricKey));
return this;
}
@Override
- public AttributeFetcher requestCollectionMetrics(SolrCollection solrCollection, Set<ReplicaMetric<?>> metricNames) {
- requestedCollectionMetrics.put(solrCollection, Set.copyOf(metricNames));
+ public AttributeFetcher requestCollectionMetrics(SolrCollection solrCollection, Set<ReplicaMetric<?>> metrics) {
+ if (!metrics.isEmpty()) {
+ requestedCollectionMetrics.put(solrCollection, Set.copyOf(metrics));
+ }
return this;
}
@@ -137,13 +140,12 @@ public class AttributeFetcherImpl implements AttributeFetcher {
// Maps in which attribute values will be added
Map<Node, Integer> nodeToCoreCount = new HashMap<>();
- Map<Node, DiskHardwareType> nodeToDiskType = new HashMap<>();
Map<Node, Double> nodeToFreeDisk = new HashMap<>();
Map<Node, Double> nodeToTotalDisk = new HashMap<>();
Map<Node, Double> nodeToHeapUsage = new HashMap<>();
Map<Node, Double> nodeToSystemLoadAverage = new HashMap<>();
Map<String, Map<Node, String>> systemSnitchToNodeToValue = new HashMap<>();
- Map<String, Map<Node, Object>> metricSnitchToNodeToValue = new HashMap<>();
+ Map<NodeMetric<?>, Map<Node, Object>> metricSnitchToNodeToValue = new HashMap<>();
Map<String, CollectionMetricsBuilder> collectionMetricsBuilders = new HashMap<>();
Map<Node, Set<String>> nodeToReplicaInternalTags = new HashMap<>();
Map<String, Set<ReplicaMetric<?>>> requestedCollectionNamesMetrics = requestedCollectionMetrics.entrySet().stream()
@@ -181,10 +183,11 @@ public class AttributeFetcherImpl implements AttributeFetcher {
systemSnitchToNodeToValue.put(sysPropSnitch, sysPropMap);
allSnitchTagsToInsertion.put(sysPropSnitch, (node, value) -> sysPropMap.put(node, (String) value));
}
- for (String metricSnitch : requestedNodeMetricSnitchTags) {
+ for (NodeMetric<?> metric : requestedNodeMetricSnitchTags) {
final Map<Node, Object> metricMap = new HashMap<>();
- metricSnitchToNodeToValue.put(metricSnitch, metricMap);
- allSnitchTagsToInsertion.put(metricSnitch, (node, value) -> metricMap.put(node, value));
+ metricSnitchToNodeToValue.put(metric, metricMap);
+ String metricSnitch = getMetricSnitchTag(metric);
+ allSnitchTagsToInsertion.put(metricSnitch, (node, value) -> metricMap.put(node, metric.convert(value)));
}
requestedCollectionMetrics.forEach((collection, tags) -> {
Set<String> collectionTags = tags.stream().map(tag -> tag.getInternalName()).collect(Collectors.toSet());
@@ -192,9 +195,6 @@ public class AttributeFetcherImpl implements AttributeFetcher {
shard.replicas().forEach(replica -> {
Set<String> perNodeInternalTags = nodeToReplicaInternalTags
.computeIfAbsent(replica.getNode(), n -> new HashSet<>());
- if (perNodeInternalTags.isEmpty()) {
- perNodeInternalTags.add(ReplicaMetric.INDEX_SIZE_GB.getInternalName());
- }
perNodeInternalTags.addAll(collectionTags);
}));
});
@@ -240,15 +240,8 @@ public class AttributeFetcherImpl implements AttributeFetcher {
if (requestedMetrics == null) {
throw new RuntimeException("impossible error");
}
- Double sizeGB = ReplicaMetric.INDEX_SIZE_GB.convert(replica.get(ReplicaMetric.INDEX_SIZE_GB.getInternalName()));
- if (sizeGB != null) {
- replicaMetricsBuilder.setSizeGB(sizeGB);
- }
requestedMetrics.forEach(metric -> {
- Object value = metric.convert(replica.get(metric.getInternalName()));
- if (value != null) {
- replicaMetricsBuilder.addMetric(metric.getName(), value);
- }
+ replicaMetricsBuilder.addMetric(metric, replica.get(metric.getInternalName()));
});
});
});
@@ -260,7 +253,6 @@ public class AttributeFetcherImpl implements AttributeFetcher {
collectionMetricsBuilders.forEach((name, builder) -> collectionMetrics.put(name, builder.build()));
return new AttributeValuesImpl(nodeToCoreCount,
- nodeToDiskType,
nodeToFreeDisk,
nodeToTotalDisk,
nodeToHeapUsage,
@@ -280,12 +272,13 @@ public class AttributeFetcherImpl implements AttributeFetcher {
}
}
- public static String getMetricSnitchTag(String metricName, NodeMetricRegistry registry) {
- return SolrClientNodeStateProvider.METRICS_PREFIX + SolrMetricManager.getRegistryName(getGroupFromMetricRegistry(registry), metricName);
- }
-
- public static String getMetricKeySnitchTag(String metricKey) {
- return SolrClientNodeStateProvider.METRICS_PREFIX + metricKey;
+ public static String getMetricSnitchTag(NodeMetric<?> metric) {
+ if (metric.getRegistry() != null) {
+ return SolrClientNodeStateProvider.METRICS_PREFIX +
+ SolrMetricManager.getRegistryName(getGroupFromMetricRegistry(metric.getRegistry())) + ":" + metric.getInternalName();
+ } else {
+ return SolrClientNodeStateProvider.METRICS_PREFIX + metric.getInternalName();
+ }
}
public static String getSystemPropertySnitchTag(String name) {
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeValuesImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeValuesImpl.java
index 898d272..f16fcf8 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeValuesImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeValuesImpl.java
@@ -17,17 +17,16 @@
package org.apache.solr.cluster.placement.impl;
-import org.apache.solr.cluster.placement.AttributeFetcher;
import org.apache.solr.cluster.placement.AttributeValues;
import org.apache.solr.cluster.Node;
import org.apache.solr.cluster.placement.CollectionMetrics;
+import org.apache.solr.cluster.placement.NodeMetric;
import java.util.Map;
import java.util.Optional;
public class AttributeValuesImpl implements AttributeValues {
final Map<Node, Integer> nodeToCoreCount;
- final Map<Node, AttributeFetcher.DiskHardwareType> nodeToDiskType;
final Map<Node, Double> nodeToFreeDisk;
final Map<Node, Double> nodeToTotalDisk;
final Map<Node, Double> nodeToHeapUsage;
@@ -35,21 +34,19 @@ public class AttributeValuesImpl implements AttributeValues {
// sysprop (or sysenv) name / node -> value
final Map<String, Map<Node, String>> systemSnitchToNodeToValue;
// metricName / node -> value
- final Map<String, Map<Node, Object>> metricSnitchToNodeToValue;
+ final Map<NodeMetric<?>, Map<Node, Object>> metricSnitchToNodeToValue;
// collection / shard / replica / metricName -> value
final Map<String, CollectionMetrics> collectionMetrics;
public AttributeValuesImpl(Map<Node, Integer> nodeToCoreCount,
- Map<Node, AttributeFetcher.DiskHardwareType> nodeToDiskType,
Map<Node, Double> nodeToFreeDisk,
Map<Node, Double> nodeToTotalDisk,
Map<Node, Double> nodeToHeapUsage,
Map<Node, Double> nodeToSystemLoadAverage,
Map<String, Map<Node, String>> systemSnitchToNodeToValue,
- Map<String, Map<Node, Object>> metricSnitchToNodeToValue,
+ Map<NodeMetric<?>, Map<Node, Object>> metricSnitchToNodeToValue,
Map<String, CollectionMetrics> collectionMetrics) {
this.nodeToCoreCount = nodeToCoreCount;
- this.nodeToDiskType = nodeToDiskType;
this.nodeToFreeDisk = nodeToFreeDisk;
this.nodeToTotalDisk = nodeToTotalDisk;
this.nodeToHeapUsage = nodeToHeapUsage;
@@ -65,11 +62,6 @@ public class AttributeValuesImpl implements AttributeValues {
}
@Override
- public Optional<AttributeFetcher.DiskHardwareType> getDiskType(Node node) {
- return Optional.ofNullable(nodeToDiskType.get(node));
- }
-
- @Override
public Optional<Double> getFreeDisk(Node node) {
return Optional.ofNullable(nodeToFreeDisk.get(node));
}
@@ -108,17 +100,18 @@ public class AttributeValuesImpl implements AttributeValues {
}
@Override
- public Optional<Object> getMetric(Node node, String metricName, AttributeFetcher.NodeMetricRegistry registry) {
- Map<Node, Object> nodeToValue = metricSnitchToNodeToValue.get(AttributeFetcherImpl.getMetricSnitchTag(metricName, registry));
+ @SuppressWarnings("unchecked")
+ public <T> Optional<T> getNodeMetric(Node node, NodeMetric<T> metric) {
+ Map<Node, Object> nodeToValue = metricSnitchToNodeToValue.get(metric);
if (nodeToValue == null) {
return Optional.empty();
}
- return Optional.ofNullable(nodeToValue.get(node));
+ return Optional.ofNullable((T) nodeToValue.get(node));
}
@Override
public Optional<Object> getNodeMetric(Node node, String metricKey) {
- Map<Node, Object> nodeToValue = metricSnitchToNodeToValue.get(AttributeFetcherImpl.getMetricKeySnitchTag(metricKey));
+ Map<Node, Object> nodeToValue = metricSnitchToNodeToValue.get(new NodeMetric<>(metricKey));
if (nodeToValue == null) {
return Optional.empty();
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java
index 8c1cb66..00bfdf2 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java
@@ -17,6 +17,7 @@
package org.apache.solr.cluster.placement.impl;
import org.apache.solr.cluster.placement.CollectionMetrics;
+import org.apache.solr.cluster.placement.ReplicaMetric;
import org.apache.solr.cluster.placement.ReplicaMetrics;
import org.apache.solr.cluster.placement.ShardMetrics;
@@ -44,31 +45,37 @@ public class CollectionMetricsBuilder {
public static class ShardMetricsBuilder {
final Map<String, ReplicaMetricsBuilder> replicaMetricsBuilders = new HashMap<>();
+ ReplicaMetricsBuilder leaderMetricsBuilder;
public Map<String, ReplicaMetricsBuilder> getReplicaMetricsBuilders() {
return replicaMetricsBuilders;
}
public ShardMetricsBuilder setLeaderMetrics(ReplicaMetricsBuilder replicaMetricsBuilder) {
- replicaMetricsBuilders.put(LEADER, replicaMetricsBuilder);
+ leaderMetricsBuilder = replicaMetricsBuilder;
return this;
}
- static final String LEADER = "__leader__";
-
public ShardMetrics build() {
final Map<String, ReplicaMetrics> metricsMap = new HashMap<>();
replicaMetricsBuilders.forEach((name, replicaBuilder) -> {
ReplicaMetrics metrics = replicaBuilder.build();
metricsMap.put(name, metrics);
+ // skip leader from map
if (replicaBuilder.leader) {
- metricsMap.put(LEADER, metrics);
+ if (leaderMetricsBuilder == null) {
+ leaderMetricsBuilder = replicaBuilder;
+ }
+ if (replicaBuilder != leaderMetricsBuilder) {
+ throw new RuntimeException("inconsistent data for leader metrics: found " + replicaBuilder + " but expected " + leaderMetricsBuilder);
+ }
}
});
+ final ReplicaMetrics finalLeaderMetrics = leaderMetricsBuilder != null ? leaderMetricsBuilder.build() : null;
return new ShardMetrics() {
@Override
public Optional<ReplicaMetrics> getLeaderMetrics() {
- return Optional.ofNullable(metricsMap.get(LEADER));
+ return Optional.ofNullable(finalLeaderMetrics);
}
@Override
@@ -80,35 +87,33 @@ public class CollectionMetricsBuilder {
}
public static class ReplicaMetricsBuilder {
- final Map<String, Object> metrics = new HashMap<>();
- Double sizeGB;
+ final Map<ReplicaMetric<?>, Object> metrics = new HashMap<>();
boolean leader;
- public ReplicaMetricsBuilder setSizeGB(double size) {
- this.sizeGB = size;
- return this;
- }
-
public ReplicaMetricsBuilder setLeader(boolean leader) {
this.leader = leader;
return this;
}
- public ReplicaMetricsBuilder addMetric(String metricName, Object value) {
- metrics.put(metricName, value);
+ /** Add unconverted (raw) values here, this method internally calls
+ * {@link ReplicaMetric#convert(Object)}.
+ * @param metric metric to add
+ * @param value raw (unconverted) metric value
+ */
+ public ReplicaMetricsBuilder addMetric(ReplicaMetric<?> metric, Object value) {
+ value = metric.convert(value);
+ if (value != null) {
+ metrics.put(metric, value);
+ }
return this;
}
public ReplicaMetrics build() {
return new ReplicaMetrics() {
@Override
- public Double getReplicaSizeGB() {
- return sizeGB;
- }
-
- @Override
- public Optional<Object> getReplicaMetric(String metricName) {
- return Optional.ofNullable(metrics.get(metricName));
+ @SuppressWarnings("unchecked")
+ public <T> Optional<T> getReplicaMetric(ReplicaMetric<T> metric) {
+ return Optional.ofNullable((T) metrics.get(metric));
}
};
}
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java
index 23a8e2f..004f8e2 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java
@@ -71,7 +71,7 @@ public class AttributeFetcherForTest implements AttributeFetcher {
}
@Override
- public AttributeFetcher requestNodeMetric(String metricName, NodeMetricRegistry registry) {
+ public AttributeFetcher requestNodeMetric(NodeMetric<?> metric) {
throw new UnsupportedOperationException("Not yet implemented...");
}
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java b/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java
index 06b5d61..e024798 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java
@@ -52,7 +52,6 @@ public class Builders {
NodeBuilder nodeBuilder = new NodeBuilder().setNodeName("node_" + n); // Default name, can be changed
nodeBuilder.setTotalDiskGB(10000.0);
nodeBuilder.setFreeDiskGB(5000.0);
- nodeBuilder.setDiskType(AttributeFetcher.DiskHardwareType.SSD);
nodeBuilders.add(nodeBuilder);
}
return this;
@@ -93,11 +92,10 @@ public class Builders {
public AttributeFetcher buildAttributeFetcher() {
Map<Node, Integer> nodeToCoreCount = new HashMap<>();
- Map<Node, AttributeFetcher.DiskHardwareType> nodeToDiskType = new HashMap<>();
Map<Node, Double> nodeToFreeDisk = new HashMap<>();
Map<Node, Double> nodeToTotalDisk = new HashMap<>();
Map<String, Map<Node, String>> sysprops = new HashMap<>();
- Map<String, Map<Node, Object>> metrics = new HashMap<>();
+ Map<NodeMetric<?>, Map<Node, Object>> metrics = new HashMap<>();
Map<String, CollectionMetrics> collectionMetrics = new HashMap<>();
// TODO And a few more missing and will be added...
@@ -107,10 +105,6 @@ public class Builders {
for (NodeBuilder nodeBuilder : nodeBuilders) {
Node node = nodeBuilder.build();
- if (nodeBuilder.getDiskType() != null) {
- nodeToDiskType.put(node, nodeBuilder.getDiskType());
- }
-
if (nodeBuilder.getCoreCount() != null) {
nodeToCoreCount.put(node, nodeBuilder.getCoreCount());
}
@@ -144,7 +138,7 @@ public class Builders {
}));
});
- AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, nodeToDiskType, nodeToFreeDisk,
+ AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, nodeToFreeDisk,
nodeToTotalDisk, Map.of(), Map.of(), sysprops, metrics, collectionMetrics);
return new AttributeFetcherForTest(attributeValues);
}
@@ -320,7 +314,7 @@ public class Builders {
CollectionMetricsBuilder.ReplicaMetricsBuilder replicaMetricsBuilder = new CollectionMetricsBuilder.ReplicaMetricsBuilder();
shardMetricsBuilder.getReplicaMetricsBuilders().put(replicaName, replicaMetricsBuilder);
if (initialSizeGBPerShard != null) {
- replicaMetricsBuilder.setSizeGB(initialSizeGBPerShard.get(shardNumber - 1));
+ replicaMetricsBuilder.addMetric(ReplicaMetric.INDEX_SIZE_GB, initialSizeGBPerShard.get(shardNumber - 1) * ReplicaMetric.GB);
}
if (leader == null && type != Replica.ReplicaType.PULL) {
leader = replicaBuilder;
@@ -420,6 +414,7 @@ public class Builders {
private Replica.ReplicaType replicaType;
private Replica.ReplicaState replicaState;
private NodeBuilder replicaNode;
+ private Map<ReplicaMetric<?>, Object> metrics;
public ReplicaBuilder setReplicaName(String replicaName) {
this.replicaName = replicaName;
@@ -450,6 +445,14 @@ public class Builders {
return this;
}
+ public ReplicaBuilder setReplicaMetric(ReplicaMetric<?> metric, Object value) {
+ if (metrics == null) {
+ metrics = new HashMap<>();
+ }
+ metrics.put(metric, metric.convert(value));
+ return this;
+ }
+
public Replica build(Shard shard) {
return new ClusterAbstractionsForTest.ReplicaImpl(replicaName, coreName, shard, replicaType, replicaState, replicaNode.build());
}
@@ -460,9 +463,8 @@ public class Builders {
private Integer coreCount = null;
private Double freeDiskGB = null;
private Double totalDiskGB = null;
- private AttributeFetcher.DiskHardwareType diskType;
private Map<String, String> sysprops = null;
- private Map<String, Double> metrics = null;
+ private Map<NodeMetric<?>, Object> metrics = null;
public NodeBuilder setNodeName(String nodeName) {
this.nodeName = nodeName;
@@ -493,17 +495,11 @@ public class Builders {
return this;
}
- public NodeBuilder setDiskType(AttributeFetcher.DiskHardwareType diskType) {
- this.diskType = diskType;
- return this;
- }
-
- public NodeBuilder setMetric(AttributeFetcher.NodeMetricRegistry registry, String key, Double value) {
+ public NodeBuilder setMetric(NodeMetric<?> metric, Object value) {
if (metrics == null) {
metrics = new HashMap<>();
}
- String name = AttributeFetcherImpl.getMetricSnitchTag(key, registry);
- metrics.put(name, value);
+ metrics.put(metric, metric.convert(value));
return this;
}
@@ -519,15 +515,11 @@ public class Builders {
return totalDiskGB;
}
- public AttributeFetcher.DiskHardwareType getDiskType() {
- return diskType;
- }
-
public Map<String, String> getSysprops() {
return sysprops;
}
- public Map<String, Double> getMetrics() {
+ public Map<NodeMetric<?>, Object> getMetrics() {
return metrics;
}
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/BuildersTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/BuildersTest.java
index 3d2d418..561c8ec 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/BuildersTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/BuildersTest.java
@@ -83,13 +83,11 @@ public class BuildersTest extends SolrTestCaseJ4 {
.requestNodeDiskType()
.requestNodeFreeDisk()
.requestNodeTotalDisk()
- .requestCollectionMetrics(collection, Set.of());
+ .requestCollectionMetrics(collection, Set.of(ReplicaMetric.INDEX_SIZE_GB));
AttributeValues attributeValues = attributeFetcher.fetchAttributes();
for (Node node : cluster.getLiveNodes()) {
Optional<Integer> coreCount = attributeValues.getCoresCount(node);
assertTrue("coreCount present", coreCount.isPresent());
- Optional<AttributeFetcher.DiskHardwareType> diskType = attributeValues.getDiskType(node);
- assertTrue("diskType present", diskType.isPresent());
Optional<Double> diskOpt = attributeValues.getFreeDisk(node);
assertTrue("freeDisk", diskOpt.isPresent());
diskOpt = attributeValues.getTotalDisk(node);
@@ -105,20 +103,24 @@ public class BuildersTest extends SolrTestCaseJ4 {
Optional<ReplicaMetrics> replicaMetricsOpt = shardMetrics.getLeaderMetrics();
assertTrue("leader metrics", replicaMetricsOpt.isPresent());
ReplicaMetrics leaderMetrics = replicaMetricsOpt.get();
+ Optional<Double> sizeOpt = leaderMetrics.getReplicaMetric(ReplicaMetric.INDEX_SIZE_GB);
+ assertTrue("missing size", sizeOpt.isPresent());
if (shardName.endsWith("1")) {
- assertEquals("size", Double.valueOf(10), leaderMetrics.getReplicaSizeGB());
+ assertEquals("size", 10, ((Number) sizeOpt.get()).intValue());
} else {
- assertEquals("size", Double.valueOf(20), leaderMetrics.getReplicaSizeGB());
+ assertEquals("size", 20, ((Number) sizeOpt.get()).intValue());
}
Shard shard = collection.getShard(shardName);
shard.iterator().forEachRemaining(r -> {
Optional<ReplicaMetrics> metricsOpt = shardMetrics.getReplicaMetrics(r.getReplicaName());
assertTrue("replica metrics", metricsOpt.isPresent());
ReplicaMetrics metrics = metricsOpt.get();
+ Optional<Double> replicaSizeOpt = metrics.getReplicaMetric(ReplicaMetric.INDEX_SIZE_GB);
+ assertTrue("missing size", replicaSizeOpt.isPresent());
if (shardName.endsWith("1")) {
- assertEquals("size", Double.valueOf(10), metrics.getReplicaSizeGB());
+ assertEquals("size", 10, ((Number) replicaSizeOpt.get()).intValue());
} else {
- assertEquals("size", Double.valueOf(20), metrics.getReplicaSizeGB());
+ assertEquals("size", 20, ((Number) replicaSizeOpt.get()).intValue());
}
});
}
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/impl/PlacementPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/impl/PlacementPluginIntegrationTest.java
index 88fb673..eb80b18 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/impl/PlacementPluginIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/impl/PlacementPluginIntegrationTest.java
@@ -30,6 +30,7 @@ import org.apache.solr.cluster.SolrCollection;
import org.apache.solr.cluster.placement.AttributeFetcher;
import org.apache.solr.cluster.placement.AttributeValues;
import org.apache.solr.cluster.placement.CollectionMetrics;
+import org.apache.solr.cluster.placement.NodeMetric;
import org.apache.solr.cluster.placement.PlacementPluginConfig;
import org.apache.solr.cluster.placement.PlacementPluginFactory;
import org.apache.solr.cluster.placement.ReplicaMetric;
@@ -240,19 +241,22 @@ public class PlacementPluginIntegrationTest extends SolrCloudTestCase {
Cluster cluster = new SimpleClusterAbstractionsImpl.ClusterImpl(cloudManager);
SolrCollection collection = cluster.getCollection(COLLECTION);
AttributeFetcher attributeFetcher = new AttributeFetcherImpl(cloudManager);
- String someMetricName = "solr.jvm:system.properties:user.name";
+ String someMetricKey = "solr.jvm:system.properties:user.name";
String sysprop = "user.name";
String sysenv = "PWD";
attributeFetcher
.fetchFrom(cluster.getLiveNodes())
.requestNodeHeapUsage()
- .requestNodeMetric(someMetricName)
+ .requestNodeSystemLoadAverage()
+ .requestNodeMetric(someMetricKey)
+ .requestNodeMetric(NodeMetric.SYSLOAD_AVG)
+ .requestNodeMetric(NodeMetric.AVAILABLE_PROCESSORS)
.requestNodeSystemProperty(sysprop)
.requestNodeEnvironmentVariable(sysenv)
.requestNodeTotalDisk()
.requestNodeFreeDisk()
.requestNodeCoresCount()
- .requestCollectionMetrics(collection, Set.of(ReplicaMetric.QUERY_RATE_1MIN, ReplicaMetric.UPDATE_RATE_1MIN));
+ .requestCollectionMetrics(collection, Set.of(ReplicaMetric.INDEX_SIZE_GB, ReplicaMetric.QUERY_RATE_1MIN, ReplicaMetric.UPDATE_RATE_1MIN));
AttributeValues attributeValues = attributeFetcher.fetchAttributes();
String userName = System.getProperty("user.name");
String pwd = System.getenv("PWD");
@@ -262,7 +266,10 @@ public class PlacementPluginIntegrationTest extends SolrCloudTestCase {
assertTrue("total disk", attributeValues.getTotalDisk(node).isPresent());
assertTrue("free disk", attributeValues.getFreeDisk(node).isPresent());
assertTrue("cores count", attributeValues.getCoresCount(node).isPresent());
- Optional<Object> userNameOpt = attributeValues.getNodeMetric(node, someMetricName);
+ assertTrue("systemLoadAverage 1", attributeValues.getSystemLoadAverage(node).isPresent());
+ assertTrue("systemLoadAverage 2", attributeValues.getNodeMetric(node, NodeMetric.SYSLOAD_AVG).isPresent());
+ assertTrue("availableProcessors", attributeValues.getNodeMetric(node, NodeMetric.AVAILABLE_PROCESSORS).isPresent());
+ Optional<Object> userNameOpt = attributeValues.getNodeMetric(node, someMetricKey);
assertTrue("user.name", userNameOpt.isPresent());
assertEquals("userName", userName, userNameOpt.get());
Optional<String> syspropOpt = attributeValues.getSystemProperty(node, sysprop);
@@ -281,12 +288,14 @@ public class PlacementPluginIntegrationTest extends SolrCloudTestCase {
Optional<ReplicaMetrics> replicaMetricsOpt = shardMetricsOpt.get().getReplicaMetrics(replica.getReplicaName());
assertTrue("replica metrics", replicaMetricsOpt.isPresent());
ReplicaMetrics replicaMetrics = replicaMetricsOpt.get();
- // this should always be present
- assertNotNull("size", replicaMetrics.getReplicaSizeGB());
- assertTrue("should be greater than 0 but was " + replicaMetrics.getReplicaSizeGB(),
- replicaMetrics.getReplicaSizeGB() > 0);
- assertNotNull("queryRate", replicaMetrics.getReplicaMetric(ReplicaMetric.QUERY_RATE_1MIN.getName()));
- assertNotNull("updateRate", replicaMetrics.getReplicaMetric(ReplicaMetric.UPDATE_RATE_1MIN.getName()));
+ Optional<Double> indexSizeOpt = replicaMetrics.getReplicaMetric(ReplicaMetric.INDEX_SIZE_GB);
+ assertTrue("indexSize", indexSizeOpt.isPresent());
+ assertTrue("wrong type, expected Double but was " + indexSizeOpt.get().getClass(), indexSizeOpt.get() instanceof Double);
+ assertTrue("indexSize should be > 0 but was " + indexSizeOpt.get(), indexSizeOpt.get() > 0);
+ assertTrue("indexSize should be < 0.01 but was " + indexSizeOpt.get(), indexSizeOpt.get() < 0.01);
+
+ assertNotNull("queryRate", replicaMetrics.getReplicaMetric(ReplicaMetric.QUERY_RATE_1MIN));
+ assertNotNull("updateRate", replicaMetrics.getReplicaMetric(ReplicaMetric.UPDATE_RATE_1MIN));
});
});
}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index a7e43c3..ccc6583 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -273,7 +273,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
}
if (requestedTags.contains(SYSLOADAVG)) {
Number n = (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/os.systemLoadAverage");
- if (n != null) ctx.getTags().put(SYSLOADAVG, n.doubleValue() * 100.0d);
+ if (n != null) ctx.getTags().put(SYSLOADAVG, n.doubleValue());
}
if (requestedTags.contains(HEAPUSAGE)) {
Number n = (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/memory.heap.usage");