You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/05/23 18:40:58 UTC
lucene-solr:jira/solr-11779: SOLR-11779: Misc cleanups. No need to
register db-s in SolrMetricManager.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11779 228ba6385 -> e53f4b83f
SOLR-11779: Misc cleanups. No need to register db-s in SolrMetricManager.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e53f4b83
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e53f4b83
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e53f4b83
Branch: refs/heads/jira/solr-11779
Commit: e53f4b83f2aeecfdba585baf6983d049372781f8
Parents: 228ba63
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed May 23 20:40:15 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed May 23 20:40:15 2018 +0200
----------------------------------------------------------------------
.../api/collections/DeleteCollectionCmd.java | 2 +-
.../solr/handler/admin/CoreAdminOperation.java | 2 +-
.../handler/admin/MetricsHistoryHandler.java | 141 ++++++++++---------
.../apache/solr/metrics/SolrMetricManager.java | 37 -----
.../cloud/MetricsHistoryIntegrationTest.java | 2 +-
.../src/resources/apispec/metrics.history.json | 11 +-
6 files changed, 80 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e53f4b83/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
index 18b1a0f..2a3a6cf 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
@@ -137,7 +137,7 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
MetricsHistoryHandler historyHandler = ocmh.overseer.getCoreContainer().getMetricsHistoryHandler();
if (historyHandler != null) {
String registry = SolrMetricManager.getRegistryName(SolrInfoBean.Group.collection, collection);
- historyHandler.getFactory().remove(registry);
+ historyHandler.removeHistory(registry);
}
}
} finally {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e53f4b83/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
index 99a0033..ec42a12 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
@@ -119,7 +119,7 @@ enum CoreAdminOperation implements CoreAdminOp {
cd.getShardId(),
replicaName);
}
- historyHandler.getFactory().remove(registry);
+ historyHandler.removeHistory(registry);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e53f4b83/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
index f5a60cf..e462ab2 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
@@ -37,6 +37,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.TimeZone;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -148,6 +149,8 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
private final Map<String, List<String>> counters = new HashMap<>();
private final Map<String, List<String>> gauges = new HashMap<>();
+ private final Map<String, RrdDb> knownDbs = new ConcurrentHashMap<>();
+
private ScheduledThreadPoolExecutor collectService;
private boolean logMissingCollection = true;
private boolean enable;
@@ -217,6 +220,12 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
return solrClient;
}
+ public void removeHistory(String registry) throws IOException {
+ registry = SolrMetricManager.overridableRegistryName(registry);
+ knownDbs.remove(registry);
+ factory.remove(registry);
+ }
+
@VisibleForTesting
public SolrRrdBackendFactory getFactory() {
return factory;
@@ -253,10 +262,6 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
}
private void collectMetrics() {
- if (metricManager == null) {
- // not inited yet
- return;
- }
log.debug("-- collectMetrics");
// check that .system exists
try {
@@ -324,20 +329,12 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
if (nl != null) {
for (Iterator<Map.Entry<String, Object>> it = nl.iterator(); it.hasNext(); ) {
Map.Entry<String, Object> entry = it.next();
- String reg = entry.getKey();
+ String registry = entry.getKey();
if (group != Group.core) { // add nodeName prefix
- reg = reg + "." + nodeName;
+ registry = registry + "." + nodeName;
}
- final String registry = reg;
- RrdDb db = metricManager.getOrCreateMetricHistory(registry, () -> {
- RrdDef def = createDef(registry, group);
- try {
- RrdDb newDb = new RrdDb(def, factory);
- return newDb;
- } catch (IOException e) {
- return null;
- }
- });
+
+ RrdDb db = getOrCreateDb(registry, group);
if (db == null) {
continue;
}
@@ -370,46 +367,6 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
}
}
- RrdDef createDef(String registry, Group group) {
- registry = SolrMetricManager.overridableRegistryName(registry);
-
- // base sampling period is collectPeriod - samples more frequent than
- // that will be dropped, samples less frequent will be interpolated
- RrdDef def = new RrdDef(URI_PREFIX + registry, collectPeriod);
- // set the start time early enough so that the first sample is always later
- // than the start of the archive
- def.setStartTime(TimeUnit.SECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS) - def.getStep());
-
- // add datasources
- List<Group> groups = new ArrayList<>();
- groups.add(group);
- if (group == Group.collection) {
- groups.add(Group.core);
- }
- for (Group g : groups) {
- // use NaN when more than 1 sample is missing
- counters.get(g.toString()).forEach(name ->
- def.addDatasource(name, DsType.COUNTER, collectPeriod * 2, Double.NaN, Double.NaN));
- gauges.get(g.toString()).forEach(name ->
- def.addDatasource(name, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN));
- }
- if (groups.contains(Group.node)) {
- // add nomNodes gauge
- def.addDatasource(NUM_NODES_KEY, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN);
- }
-
- // add archives
-
- // use AVERAGE consolidation,
- // use NaN when >50% samples are missing
- def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 240); // 4 hours
- def.addArchive(ConsolFun.AVERAGE, 0.5, 10, 288); // 48 hours
- def.addArchive(ConsolFun.AVERAGE, 0.5, 60, 336); // 2 weeks
- def.addArchive(ConsolFun.AVERAGE, 0.5, 240, 180); // 2 months
- def.addArchive(ConsolFun.AVERAGE, 0.5, 1440, 365); // 1 year
- return def;
- }
-
private void collectGlobalMetrics() {
if (!isOverseerLeader()) {
return;
@@ -522,15 +479,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
// now update the db-s
totals.forEach((group, perGroup) -> {
perGroup.forEach((reg, perReg) -> {
- RrdDb db = metricManager.getOrCreateMetricHistory(reg, () -> {
- RrdDef def = createDef(reg, group);
- try {
- RrdDb newDb = new RrdDb(def, factory);
- return newDb;
- } catch (IOException e) {
- return null;
- }
- });
+ RrdDb db = getOrCreateDb(reg, group);
if (db == null) {
return;
}
@@ -568,6 +517,59 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
});
}
+ private RrdDef createDef(String registry, Group group) {
+ registry = SolrMetricManager.overridableRegistryName(registry);
+
+ // base sampling period is collectPeriod - samples more frequent than
+ // that will be dropped, samples less frequent will be interpolated
+ RrdDef def = new RrdDef(URI_PREFIX + registry, collectPeriod);
+ // set the start time early enough so that the first sample is always later
+ // than the start of the archive
+ def.setStartTime(TimeUnit.SECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS) - def.getStep());
+
+ // add datasources
+ List<Group> groups = new ArrayList<>();
+ groups.add(group);
+ if (group == Group.collection) {
+ groups.add(Group.core);
+ }
+ for (Group g : groups) {
+ // use NaN when more than 1 sample is missing
+ counters.get(g.toString()).forEach(name ->
+ def.addDatasource(name, DsType.COUNTER, collectPeriod * 2, Double.NaN, Double.NaN));
+ gauges.get(g.toString()).forEach(name ->
+ def.addDatasource(name, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN));
+ }
+ if (groups.contains(Group.node)) {
+ // add nomNodes gauge
+ def.addDatasource(NUM_NODES_KEY, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN);
+ }
+
+ // add archives
+
+ // use AVERAGE consolidation,
+ // use NaN when >50% samples are missing
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 240); // 4 hours
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 10, 288); // 48 hours
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 60, 336); // 2 weeks
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 240, 180); // 2 months
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 1440, 365); // 1 year
+ return def;
+ }
+
+ private RrdDb getOrCreateDb(String registry, Group group) {
+ RrdDb db = knownDbs.computeIfAbsent(registry, r -> {
+ RrdDef def = createDef(r, group);
+ try {
+ RrdDb newDb = new RrdDb(def, factory);
+ return newDb;
+ } catch (IOException e) {
+ return null;
+ }
+ });
+ return db;
+ }
+
@Override
public void close() {
log.debug("Closing " + hashCode());
@@ -577,6 +579,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
if (factory != null) {
factory.close();
}
+ knownDbs.clear();
}
public enum Cmd {
@@ -646,7 +649,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
RrdDb db = new RrdDb(URI_PREFIX + name, true, factory);
res = new NamedList<>();
NamedList<Object> data = new NamedList<>();
- data.add("data", getData(db, dsNames, format, req.getParams()));
+ data.add("data", getDbData(db, dsNames, format, req.getParams()));
((NamedList)res).add(name, data);
db.close();
}
@@ -663,7 +666,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
RrdDb db = new RrdDb(URI_PREFIX + name, true, factory);
NamedList<Object> map = new NamedList<>();
NamedList<Object> status = new NamedList<>();
- status.add("status", reportStatus(db));
+ status.add("status", getDbStatus(db));
map.add(name, status);
db.close();
res = map;
@@ -687,7 +690,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
}
}
- private NamedList<Object> reportStatus(RrdDb db) throws IOException {
+ private NamedList<Object> getDbStatus(RrdDb db) throws IOException {
NamedList<Object> res = new SimpleOrderedMap<>();
res.add("lastModified", db.getLastUpdateTime());
RrdDef def = db.getRrdDef();
@@ -723,7 +726,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
return res;
}
- private NamedList<Object> getData(RrdDb db, String[] dsNames, Format format, SolrParams params) throws IOException {
+ private NamedList<Object> getDbData(RrdDb db, String[] dsNames, Format format, SolrParams params) throws IOException {
NamedList<Object> res = new SimpleOrderedMap<>();
if (dsNames == null || dsNames.length == 0) {
dsNames = db.getDsNames();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e53f4b83/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
index 5c3b49a..5fa3659 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
@@ -34,7 +34,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;
@@ -49,7 +48,6 @@ import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.MetricSet;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
-import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.MetricsConfig;
@@ -57,7 +55,6 @@ import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.core.SolrResourceLoader;
-import org.rrd4j.core.RrdDb;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,8 +97,6 @@ public class SolrMetricManager {
private final Map<String, Map<String, SolrMetricReporter>> reporters = new HashMap<>();
- private final ConcurrentHashMap<String, RrdDb> metricHistories = new ConcurrentHashMap<>();
-
private final Lock reportersLock = new ReentrantLock();
private final Lock swapLock = new ReentrantLock();
@@ -450,26 +445,6 @@ public class SolrMetricManager {
}
}
- public RrdDb getOrCreateMetricHistory(String registry, Supplier<RrdDb> supplier) {
- registry = overridableRegistryName(registry);
- final RrdDb existing = metricHistories.get(registry);
- if (existing == null) {
- final RrdDb created = supplier.get();
- if (created == null) {
- // maybe someone else succeeded
- return metricHistories.get(registry);
- }
- final RrdDb raced = metricHistories.putIfAbsent(registry, created);
- if (raced == null) {
- return created;
- } else {
- return raced;
- }
- } else {
- return existing;
- }
- }
-
/**
* Remove a named registry.
* @param registry name of the registry to remove
@@ -489,10 +464,6 @@ public class SolrMetricManager {
swapLock.unlock();
}
}
- RrdDb db = metricHistories.remove(registry);
- if (db != null) {
- IOUtils.closeQuietly(db);
- }
}
/**
@@ -519,20 +490,12 @@ public class SolrMetricManager {
}
MetricRegistry reg1 = registries.remove(registry1);
MetricRegistry reg2 = registries.remove(registry2);
- RrdDb db1 = metricHistories.remove(registry1);
- RrdDb db2 = metricHistories.remove(registry2);
if (reg2 != null) {
registries.put(registry1, reg2);
}
if (reg1 != null) {
registries.put(registry2, reg1);
}
- if (db2 != null) {
- metricHistories.put(registry1, db2);
- }
- if (db1 != null) {
- metricHistories.put(registry2, db1);
- }
} finally {
swapLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e53f4b83/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java
index c69b874..2012a1a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java
@@ -57,7 +57,7 @@ public class MetricsHistoryIntegrationTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
- boolean simulated = random().nextBoolean() && false;
+ boolean simulated = random().nextBoolean();
if (simulated) {
cloudManager = SimCloudManager.createCluster(1, TimeSource.get("simTime:50"));
solrClient = ((SimCloudManager)cloudManager).simGetSolrClient();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e53f4b83/solr/solrj/src/resources/apispec/metrics.history.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/resources/apispec/metrics.history.json b/solr/solrj/src/resources/apispec/metrics.history.json
index 49de7b9..975d775 100644
--- a/solr/solrj/src/resources/apispec/metrics.history.json
+++ b/solr/solrj/src/resources/apispec/metrics.history.json
@@ -9,15 +9,14 @@
"/cluster/metrics/history"
],
"params": {
- "collection": {
+ "action": {
"type": "string",
- "description": "Collection where metrics history is stored, '.system' by default.",
- "default": ".system"
+ "description": "one of supported actions",
+ "default": "list"
},
- "q": {
+ "name": {
"type": "string",
- "description": "Arbitrary query to limit the selected metrics.",
- "default": "*:*"
+ "description": "registry name"
}
}
}