You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/01/02 18:28:16 UTC
lucene-solr:jira/solr-9854: SOLR-9854 Add flush meter. Fix bugs,
add unit tests.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-9854 4ca11b3ed -> b9b69b150
SOLR-9854 Add flush meter. Fix bugs, add unit tests.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b9b69b15
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b9b69b15
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b9b69b15
Branch: refs/heads/jira/solr-9854
Commit: b9b69b150fb6ff068a13b285d0992f94b2832c25
Parents: 4ca11b3
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Mon Jan 2 19:27:47 2017 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Mon Jan 2 19:27:47 2017 +0100
----------------------------------------------------------------------
.../solr/core/MetricsDirectoryFactory.java | 6 +-
.../org/apache/solr/update/SolrIndexConfig.java | 7 +-
.../org/apache/solr/update/SolrIndexWriter.java | 77 ++++++++++++----
.../conf/solrconfig-indexmetrics.xml | 38 ++++++++
.../solr/update/SolrIndexMetricsTest.java | 94 ++++++++++++++++++++
5 files changed, 203 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b9b69b15/solr/core/src/java/org/apache/solr/core/MetricsDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/MetricsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/MetricsDirectoryFactory.java
index 7d2024a..62e82ac 100644
--- a/solr/core/src/java/org/apache/solr/core/MetricsDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/MetricsDirectoryFactory.java
@@ -253,7 +253,9 @@ public class MetricsDirectoryFactory extends DirectoryFactory implements SolrCor
- private static final String SEGMENTS = "segments_";
+ private static final String SEGMENTS = "segments";
+ private static final String SEGMENTS_PREFIX = "segments_";
+ private static final String PENDING_SEGMENTS_PREFIX = "pending_segments_";
private static final String TEMP = "temp";
private static final String OTHER = "other";
@@ -292,7 +294,7 @@ public class MetricsDirectoryFactory extends DirectoryFactory implements SolrCor
return null;
}
String lastName;
- if (name.startsWith(SEGMENTS)) {
+ if (name.startsWith(SEGMENTS_PREFIX) || name.startsWith(PENDING_SEGMENTS_PREFIX)) {
lastName = SEGMENTS;
} else {
int pos = name.lastIndexOf('.');
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b9b69b15/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
index 62876c9..d484e85 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
@@ -148,7 +148,12 @@ public class SolrIndexConfig implements MapSerializable {
writeLockTimeout=solrConfig.getInt(prefix+"/writeLockTimeout", def.writeLockTimeout);
lockType=solrConfig.get(prefix+"/lockType", def.lockType);
- metricsInfo = getPluginInfo(prefix + "/metrics", solrConfig, def.metricsInfo);
+ List<PluginInfo> infos = solrConfig.readPluginInfos(prefix + "/metrics", false, false);
+ if (infos.isEmpty()) {
+ metricsInfo = def.metricsInfo;
+ } else {
+ metricsInfo = infos.get(0);
+ }
mergeSchedulerInfo = getPluginInfo(prefix + "/mergeScheduler", solrConfig, def.mergeSchedulerInfo);
mergePolicyInfo = getPluginInfo(prefix + "/mergePolicy", solrConfig, def.mergePolicyInfo);
mergePolicyFactoryInfo = getPluginInfo(prefix + "/mergePolicyFactory", solrConfig, def.mergePolicyFactoryInfo);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b9b69b15/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
index c175dbe..626bc8e 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
@@ -71,20 +71,25 @@ public class SolrIndexWriter extends IndexWriter {
// metrics
private long majorMergeDocs = 512 * 1024;
- private Timer majorMerge;
- private Timer minorMerge;
- private Meter majorMergedDocs;
- private Meter majorDeletedDocs;
- private Counter mergeErrors;
- private boolean mergeDetails;
- private AtomicInteger runningMajorMerges = new AtomicInteger();
- private Gauge<Integer> runningMajorMergesGauge;
- private AtomicInteger runningMinorMerges = new AtomicInteger();
- private Gauge<Integer> runningMinorMergesGauge;
- private AtomicLong runningMajorMergesDocs = new AtomicLong();
- private Gauge<Long> runningMajorMergesDocsGauge;
- private AtomicLong runningMinorMergesDocs = new AtomicLong();
- private Gauge<Long> runningMinorMergesDocsGauge;
+ private final Timer majorMerge;
+ private final Timer minorMerge;
+ private final Meter majorMergedDocs;
+ private final Meter majorDeletedDocs;
+ private final Counter mergeErrors;
+ private final Meter flushMeter; // original counter is package-private in IndexWriter
+ private final boolean mergeDetails;
+ private final AtomicInteger runningMajorMerges = new AtomicInteger();
+ private final Gauge<Integer> runningMajorMergesGauge;
+ private final AtomicInteger runningMinorMerges = new AtomicInteger();
+ private final Gauge<Integer> runningMinorMergesGauge;
+ private final AtomicInteger runningMajorMergesSegments = new AtomicInteger();
+ private final Gauge<Integer> runningMajorMergesSegmentsGauge;
+ private final AtomicInteger runningMinorMergesSegments = new AtomicInteger();
+ private final Gauge<Integer> runningMinorMergesSegmentsGauge;
+ private final AtomicLong runningMajorMergesDocs = new AtomicLong();
+ private final Gauge<Long> runningMajorMergesDocsGauge;
+ private final AtomicLong runningMinorMergesDocs = new AtomicLong();
+ private final Gauge<Long> runningMinorMergesDocsGauge;
public static SolrIndexWriter create(SolrCore core, String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
@@ -110,6 +115,20 @@ public class SolrIndexWriter extends IndexWriter {
this.directory = d;
numOpens.incrementAndGet();
log.debug("Opened Writer " + name);
+ // no metrics
+ minorMerge = null;
+ majorMerge = null;
+ mergeErrors = null;
+ majorMergedDocs = null;
+ majorDeletedDocs = null;
+ runningMinorMergesGauge = null;
+ runningMinorMergesDocsGauge = null;
+ runningMinorMergesSegmentsGauge = null;
+ runningMajorMergesGauge = null;
+ runningMajorMergesDocsGauge = null;
+ runningMajorMergesSegmentsGauge = null;
+ flushMeter = null;
+ mergeDetails = false;
}
private SolrIndexWriter(SolrCore core, String name, String path, Directory directory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
@@ -132,10 +151,15 @@ public class SolrIndexWriter extends IndexWriter {
runningMinorMergesGauge = () -> runningMinorMerges.get();
runningMajorMergesDocsGauge = () -> runningMajorMergesDocs.get();
runningMinorMergesDocsGauge = () -> runningMinorMergesDocs.get();
+ runningMajorMergesSegmentsGauge = () -> runningMajorMergesSegments.get();
+ runningMinorMergesSegmentsGauge = () -> runningMinorMergesSegments.get();
metricManager.register(registry, runningMajorMergesGauge, true, "running", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
metricManager.register(registry, runningMinorMergesGauge, true, "running", SolrInfoMBean.Category.INDEX.toString(), "merge", "minor");
metricManager.register(registry, runningMajorMergesDocsGauge, true, "running.docs", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
metricManager.register(registry, runningMinorMergesDocsGauge, true, "running.docs", SolrInfoMBean.Category.INDEX.toString(), "merge", "minor");
+ metricManager.register(registry, runningMajorMergesSegmentsGauge, true, "running.segments", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
+ metricManager.register(registry, runningMinorMergesSegmentsGauge, true, "running.segments", SolrInfoMBean.Category.INDEX.toString(), "merge", "minor");
+ flushMeter = metricManager.meter(registry, "flush", SolrInfoMBean.Category.INDEX.toString());
if (config.metricsInfo != null && config.metricsInfo.initArgs != null) {
Object v = config.metricsInfo.initArgs.get("majorMergeDocs");
if (v != null) {
@@ -148,12 +172,20 @@ public class SolrIndexWriter extends IndexWriter {
Boolean Details = config.metricsInfo.initArgs.getBooleanArg("mergeDetails");
if (Details != null) {
mergeDetails = Details;
+ } else {
+ mergeDetails = false;
}
if (mergeDetails) {
majorMergedDocs = metricManager.meter(registry, "docs", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
- majorDeletedDocs = metricManager.meter(registry, "deletedDocs", SolrInfoMBean.Category.INDEX.toString(), "merge", "minor");
+ majorDeletedDocs = metricManager.meter(registry, "deletedDocs", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
+ } else {
+ majorMergedDocs = null;
+ majorDeletedDocs = null;
}
-
+ } else {
+ mergeDetails = false;
+ majorMergedDocs = null;
+ majorDeletedDocs = null;
}
}
@@ -180,10 +212,12 @@ public class SolrIndexWriter extends IndexWriter {
deletedDocs += info.getDelCount();
}
boolean major = totalNumDocs > majorMergeDocs;
+ int segmentsCount = merge.segments.size();
Timer.Context context;
if (major) {
runningMajorMerges.incrementAndGet();
runningMajorMergesDocs.addAndGet(totalNumDocs);
+ runningMajorMergesSegments.addAndGet(segmentsCount);
if (mergeDetails) {
majorMergedDocs.mark(totalNumDocs);
majorDeletedDocs.mark(deletedDocs);
@@ -192,6 +226,7 @@ public class SolrIndexWriter extends IndexWriter {
} else {
runningMinorMerges.incrementAndGet();
runningMinorMergesDocs.addAndGet(totalNumDocs);
+ runningMinorMergesSegments.addAndGet(segmentsCount);
context = minorMerge.time();
}
try {
@@ -204,13 +239,23 @@ public class SolrIndexWriter extends IndexWriter {
if (major) {
runningMajorMerges.decrementAndGet();
runningMajorMergesDocs.addAndGet(-totalNumDocs);
+ runningMajorMergesSegments.addAndGet(-segmentsCount);
} else {
runningMinorMerges.decrementAndGet();
runningMinorMergesDocs.addAndGet(-totalNumDocs);
+ runningMinorMergesSegments.addAndGet(-segmentsCount);
}
}
}
+ @Override
+ protected void doAfterFlush() throws IOException {
+ if (flushMeter != null) { // this is null when writer is used only for snapshot cleanup
+ flushMeter.mark();
+ }
+ super.doAfterFlush();
+ }
+
/**
* use DocumentBuilder now...
* private final void addField(Document doc, String name, String val) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b9b69b15/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexmetrics.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexmetrics.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexmetrics.xml
new file mode 100644
index 0000000..b0105fa
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexmetrics.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <indexConfig>
+ <metrics>
+ <bool name="directoryDetails">${solr.tests.directoryDetails:false}</bool>
+ <bool name="mergeDetails">${solr.tests.mergeDetails:false}</bool>
+ </metrics>
+ <!-- intentionally set very low values here to trigger multiple flushes and merges.
+ DO NOT USE THESE ABSURD VALUES IN PRODUCTION. -->
+ <mergeFactor>3</mergeFactor>
+ <maxBufferedDocs>100</maxBufferedDocs>
+
+ <mergePolicy class="org.apache.lucene.index.TieredMergePolicy"/>
+ </indexConfig>
+
+ <requestHandler name="standard" class="solr.StandardRequestHandler"></requestHandler>
+
+</config>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b9b69b15/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java
new file mode 100644
index 0000000..e17b1bd
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.update;
+
+import java.util.Map;
+
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test proper registration and collection of index and directory metrics.
+ */
+public class SolrIndexMetricsTest extends SolrTestCaseJ4 {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ System.setProperty("solr.tests.mergeDetails", "true");
+ System.setProperty("solr.tests.directoryDetails", "true");
+ initCore("solrconfig-indexmetrics.xml", "schema.xml");
+ }
+
+ @Test
+ public void testIndexMetrics() throws Exception {
+ SolrQueryRequest req = lrf.makeRequest();
+ UpdateHandler uh = req.getCore().getUpdateHandler();
+ AddUpdateCommand add = new AddUpdateCommand(req);
+ for (int i = 0; i < 1000; i++) {
+ add.clear();
+ add.solrDoc = new SolrInputDocument();
+ add.solrDoc.addField("id", "" + i);
+ add.solrDoc.addField("foo_s", "foo-" + i);
+ uh.addDoc(add);
+ }
+ uh.commit(new CommitUpdateCommand(req, false));
+ MetricRegistry registry = h.getCoreContainer().getMetricManager().registry(h.getCore().getCoreMetricManager().getRegistryName());
+ assertNotNull(registry);
+ // make sure all merges are finished
+ h.reload();
+
+ Map<String, Metric> metrics = registry.getMetrics();
+
+ assertTrue(metrics.entrySet().stream().filter(e -> e.getKey().startsWith("INDEX")).count() >= 12);
+ // this is variable, depending on the codec and the number of created files
+ assertTrue(metrics.entrySet().stream().filter(e -> e.getKey().startsWith("DIRECTORY")).count() > 50);
+
+ // check basic index meters
+ Timer timer = (Timer)metrics.get("INDEX.merge.minor");
+ assertEquals("minorMerge: " + timer.getCount(), 4, timer.getCount());
+ timer = (Timer)metrics.get("INDEX.merge.major");
+ assertEquals("majorMerge: " + timer.getCount(), 0, timer.getCount());
+ Meter meter = (Meter)metrics.get("INDEX.merge.major.docs");
+ assertEquals("majorMergeDocs: " + meter.getCount(), 0, meter.getCount());
+ meter = (Meter)metrics.get("INDEX.flush");
+ assertEquals("flush: " + meter.getCount(), 19, meter.getCount());
+
+ // check basic directory meters
+ meter = (Meter)metrics.get("DIRECTORY.total.reads");
+ assertTrue("totalReads", meter.getCount() > 0);
+ meter = (Meter)metrics.get("DIRECTORY.total.writes");
+ assertTrue("totalWrites", meter.getCount() > 0);
+ Histogram histogram = (Histogram)metrics.get("DIRECTORY.total.readSizes");
+ assertTrue("readSizes", histogram.getCount() > 0);
+ histogram = (Histogram)metrics.get("DIRECTORY.total.writeSizes");
+ assertTrue("writeSizes", histogram.getCount() > 0);
+ // check detailed meters
+ meter = (Meter)metrics.get("DIRECTORY.segments.writes");
+ assertTrue("segmentsWrites", meter.getCount() > 0);
+ histogram = (Histogram)metrics.get("DIRECTORY.segments.writeSizes");
+ assertTrue("segmentsWriteSizes", histogram.getCount() > 0);
+
+ }
+}