You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2016/12/19 12:52:07 UTC
[23/23] lucene-solr:feature/metrics: Merge branch 'master' into
feature/metrics
Merge branch 'master' into feature/metrics
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/67dc7604
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/67dc7604
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/67dc7604
Branch: refs/heads/feature/metrics
Commit: 67dc7604ef7340cf94ad286f3693977aec12f26f
Parents: f56da1d 321c6f0
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Mon Dec 19 13:51:25 2016 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Mon Dec 19 13:51:25 2016 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 17 +
.../lucene70/Lucene70DocValuesConsumer.java | 163 +++++--
.../lucene70/Lucene70DocValuesFormat.java | 5 +-
.../lucene70/Lucene70DocValuesProducer.java | 220 +++++++--
.../org/apache/lucene/index/LeafReader.java | 2 +-
.../apache/lucene/index/PrefixCodedTerms.java | 4 +-
.../org/apache/lucene/index/SegmentInfos.java | 7 +-
.../java/org/apache/lucene/util/LongValues.java | 9 +
.../apache/lucene/util/packed/DirectWriter.java | 8 +-
.../lucene70/TestLucene70DocValuesFormat.java | 152 +++++++
.../org/apache/lucene/facet/FacetQuery.java | 52 +++
.../apache/lucene/facet/MultiFacetQuery.java | 61 +++
.../org/apache/lucene/facet/TestFacetQuery.java | 103 +++++
.../apache/lucene/search/DocValuesStats.java | 409 +++++++++++++++++
.../lucene/search/DocValuesStatsCollector.java | 64 +++
.../search/TestDocValuesStatsCollector.java | 448 +++++++++++++++++++
.../apache/lucene/document/RangeFieldQuery.java | 1 +
.../lucene/search/DocValuesNumbersQuery.java | 26 +-
.../lucene/search/DocValuesTermsQuery.java | 49 +-
.../org/apache/lucene/search/LongHashSet.java | 156 +++++++
.../search/BaseRangeFieldQueryTestCase.java | 2 +-
.../apache/lucene/search/LongHashSetTests.java | 100 +++++
.../lucene/search/TestDocValuesTermsQuery.java | 1 +
.../org/apache/lucene/search/QueryUtils.java | 4 -
solr/CHANGES.txt | 14 +
.../solr/collection1/conf/solrconfig.xml | 5 -
solr/contrib/ltr/example/solrconfig.xml | 11 -
.../solr/solrcloud/conf/solrconfig.xml | 11 -
.../OverseerAutoReplicaFailoverThread.java | 9 +-
.../java/org/apache/solr/core/SolrConfig.java | 2 +-
.../src/java/org/apache/solr/core/SolrCore.java | 92 ++--
.../apache/solr/search/SolrFieldCacheMBean.java | 6 +-
.../apache/solr/servlet/SolrDispatchFilter.java | 15 +
.../apache/solr/servlet/SolrRequestParsers.java | 31 +-
.../solr/uninverting/UninvertingReader.java | 25 +-
.../solr/util/SolrFileCleaningTracker.java | 147 ++++++
.../solr/collection1/conf/solrconfig_perf.xml | 1 -
.../cloud/SegmentTerminateEarlyTestState.java | 12 +-
.../solr/cloud/TestMiniSolrCloudCluster.java | 50 ---
.../apache/solr/cloud/TestSegmentSorting.java | 133 ++++++
.../org/apache/solr/search/TestRealTimeGet.java | 2 +-
.../solr/search/TestSolrFieldCacheMBean.java | 3 +-
.../apache/solr/search/TestStressLucene.java | 2 +-
.../apache/solr/search/TestStressRecovery.java | 2 +-
.../apache/solr/search/TestStressReorder.java | 5 +-
.../solr/search/TestStressUserVersions.java | 4 +-
.../apache/solr/search/TestStressVersions.java | 2 +-
.../example-DIH/solr/db/conf/solrconfig.xml | 11 -
.../example-DIH/solr/mail/conf/solrconfig.xml | 11 -
.../example-DIH/solr/rss/conf/solrconfig.xml | 11 -
.../example-DIH/solr/solr/conf/solrconfig.xml | 11 -
.../example-DIH/solr/tika/conf/solrconfig.xml | 11 -
solr/example/files/conf/solrconfig.xml | 11 -
.../basic_configs/conf/solrconfig.xml | 11 -
.../conf/solrconfig.xml | 11 -
.../conf/solrconfig.xml | 11 -
.../solr/client/solrj/LargeVolumeTestBase.java | 3 +-
.../solr/client/solrj/SolrExampleTestBase.java | 1 -
.../java/org/apache/solr/SolrJettyTestBase.java | 2 -
.../java/org/apache/solr/SolrTestCaseJ4.java | 2 +-
60 files changed, 2371 insertions(+), 383 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67dc7604/solr/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67dc7604/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/core/SolrCore.java
index cad6423,b3facef..4a63686
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@@ -206,12 -200,6 +206,12 @@@ public final class SolrCore implements
private final ReentrantLock ruleExpiryLock;
private final ReentrantLock snapshotDelLock; // A lock instance to guard against concurrent deletions.
+ private final Timer newSearcherTimer;
+ private final Timer newSearcherWarmupTimer;
+ private final Counter newSearcherCounter;
- private final Counter newSearcherMaxErrorsCounter;
++ private final Counter newSearcherMaxReachedCounter;
+ private final Counter newSearcherOtherErrorsCounter;
+
public Date getStartTimeStamp() { return startTime; }
private final Map<Object, IndexFingerprint> perSegmentFingerprintCache = new MapMaker().weakKeys().makeMap();
@@@ -863,16 -838,6 +863,16 @@@
checkVersionFieldExistsInSchema(schema, coreDescriptor);
+ // Initialize the metrics manager
+ this.metricManager = initMetricManager(config);
+
+ // initialize searcher-related metrics
+ newSearcherCounter = SolrMetricManager.counter(metricManager.getRegistryName(), "newSearcher");
+ newSearcherTimer = SolrMetricManager.timer(metricManager.getRegistryName(), "newSearcherTime");
+ newSearcherWarmupTimer = SolrMetricManager.timer(metricManager.getRegistryName(), "newSearcherWarmup");
- newSearcherMaxErrorsCounter = SolrMetricManager.counter(metricManager.getRegistryName(), "newSearcherMaxErrors");
++ newSearcherMaxReachedCounter = SolrMetricManager.counter(metricManager.getRegistryName(), "newSearcherMaxReached");
+ newSearcherOtherErrorsCounter = SolrMetricManager.counter(metricManager.getRegistryName(), "newSearcherErrors");
+
// Initialize JMX
this.infoRegistry = initInfoRegistry(name, config);
infoRegistry.put("fieldCache", new SolrFieldCacheMBean());
@@@ -1943,53 -1886,57 +1943,59 @@@
// if it isn't necessary.
synchronized (searcherLock) {
- // see if we can return the current searcher
- if (_searcher!=null && !forceNew) {
- if (returnSearcher) {
- _searcher.incref();
- return _searcher;
- } else {
- return null;
+ for(;;) { // this loop is so w can retry in the event that we exceed maxWarmingSearchers
+ // see if we can return the current searcher
+ if (_searcher != null && !forceNew) {
+ if (returnSearcher) {
+ _searcher.incref();
+ return _searcher;
+ } else {
+ return null;
+ }
}
- }
- // check to see if we can wait for someone else's searcher to be set
- if (onDeckSearchers>0 && !forceNew && _searcher==null) {
- try {
- searcherLock.wait();
- } catch (InterruptedException e) {
- log.info(SolrException.toStr(e));
+ // check to see if we can wait for someone else's searcher to be set
+ if (onDeckSearchers > 0 && !forceNew && _searcher == null) {
+ try {
+ searcherLock.wait();
+ } catch (InterruptedException e) {
+ log.info(SolrException.toStr(e));
+ }
}
- }
- // check again: see if we can return right now
- if (_searcher!=null && !forceNew) {
- if (returnSearcher) {
- _searcher.incref();
- return _searcher;
- } else {
- return null;
+ // check again: see if we can return right now
+ if (_searcher != null && !forceNew) {
+ if (returnSearcher) {
+ _searcher.incref();
+ return _searcher;
+ } else {
+ return null;
+ }
+ }
+
+ // At this point, we know we need to open a new searcher...
+ // first: increment count to signal other threads that we are
+ // opening a new searcher.
+ onDeckSearchers++;
++ newSearcherCounter.inc();
+ if (onDeckSearchers < 1) {
+ // should never happen... just a sanity check
+ log.error(logid + "ERROR!!! onDeckSearchers is " + onDeckSearchers);
+ onDeckSearchers = 1; // reset
+ } else if (onDeckSearchers > maxWarmingSearchers) {
+ onDeckSearchers--;
++ newSearcherMaxReachedCounter.inc();
+ try {
+ searcherLock.wait();
+ } catch (InterruptedException e) {
+ log.info(SolrException.toStr(e));
+ }
+ continue; // go back to the top of the loop and retry
+ } else if (onDeckSearchers > 1) {
+ log.warn(logid + "PERFORMANCE WARNING: Overlapping onDeckSearchers=" + onDeckSearchers);
}
- }
- // At this point, we know we need to open a new searcher...
- // first: increment count to signal other threads that we are
- // opening a new searcher.
- onDeckSearchers++;
- newSearcherCounter.inc();
- if (onDeckSearchers < 1) {
- // should never happen... just a sanity check
- log.error(logid+"ERROR!!! onDeckSearchers is " + onDeckSearchers);
- onDeckSearchers=1; // reset
- } else if (onDeckSearchers > maxWarmingSearchers) {
- onDeckSearchers--;
- String msg="Error opening new searcher. exceeded limit of maxWarmingSearchers="+maxWarmingSearchers + ", try again later.";
- log.warn(logid+""+ msg);
- // HTTP 503==service unavailable, or 409==Conflict
- newSearcherMaxErrorsCounter.inc();
- throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,msg);
- } else if (onDeckSearchers > 1) {
- log.warn(logid+"PERFORMANCE WARNING: Overlapping onDeckSearchers=" + onDeckSearchers);
+ break; // I can now exit the loop and proceed to open a searcher
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67dc7604/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index cfd90cb,e8c4657..2852041
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@@ -47,12 -45,7 +47,13 @@@ import java.util.concurrent.atomic.Atom
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.codahale.metrics.jvm.BufferPoolMetricSet;
+import com.codahale.metrics.jvm.ClassLoadingGaugeSet;
+import com.codahale.metrics.jvm.FileDescriptorRatioGauge;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+ import org.apache.commons.io.FileCleaningTracker;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.commons.io.output.CloseShieldOutputStream;
import org.apache.commons.lang.StringUtils;