You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2012/10/29 23:13:04 UTC
svn commit: r1403555 - in /lucene/dev/trunk/solr: ./ core/
core/src/java/org/apache/solr/handler/ core/src/test/org/apache/solr/core/
test-framework/src/java/org/apache/solr/
Author: romseygeek
Date: Mon Oct 29 22:13:03 2012
New Revision: 1403555
URL: http://svn.apache.org/viewvc?rev=1403555&view=rev
Log:
SOLR-1972: Add extra query stats to RequestHandler
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/ivy.xml
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1403555&r1=1403554&r2=1403555&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Mon Oct 29 22:13:03 2012
@@ -55,6 +55,10 @@ New Features
* SOLR-3911: Make Directory and DirectoryFactory first class so that the majority
of Solr's features work with any custom implementations. (Mark Miller)
+* SOLR-1972: Add extra statistics to RequestHandlers - 5 & 15-minute reqs/sec
+ rolling averages; median, 75th, 95th, 99th, 99.9th percentile request times
+ (Alan Woodward)
+
Optimizations
----------------------
Modified: lucene/dev/trunk/solr/core/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/ivy.xml?rev=1403555&r1=1403554&r2=1403555&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/ivy.xml (original)
+++ lucene/dev/trunk/solr/core/ivy.xml Mon Oct 29 22:13:03 2012
@@ -28,6 +28,7 @@
<dependency org="org.easymock" name="easymock" rev="2.2" transitive="false"/>
<dependency org="com.spatial4j" name="spatial4j" rev="0.3" transitive="false"/>
<dependency org="javax.servlet" name="javax.servlet-api" rev="3.0.1" transitive="false"/>
+ <dependency org="com.yammer.metrics" name="metrics-core" rev="2.1.2" transitive="false"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java?rev=1403555&r1=1403554&r2=1403555&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java Mon Oct 29 22:13:03 2012
@@ -17,6 +17,11 @@
package org.apache.solr.handler;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.Timer;
+import com.yammer.metrics.core.TimerContext;
+import com.yammer.metrics.stats.Snapshot;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
@@ -30,26 +35,34 @@ import org.apache.solr.response.SolrQuer
import org.apache.solr.util.SolrPluginUtils;
import java.net.URL;
+import java.util.concurrent.atomic.AtomicLong;
/**
*
*/
public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfoMBean {
- // statistics
- // TODO: should we bother synchronizing these, or is an off-by-one error
- // acceptable every million requests or so?
- volatile long numRequests;
- volatile long numErrors;
- volatile long numTimeouts;
protected NamedList initArgs = null;
protected SolrParams defaults;
protected SolrParams appends;
protected SolrParams invariants;
- volatile long totalTime = 0;
- long handlerStart = System.currentTimeMillis();
protected boolean httpCaching = true;
+ // Statistics
+ private static final AtomicLong handlerNumber = new AtomicLong();
+ private final Counter numRequests;
+ private final Counter numErrors;
+ private final Counter numTimeouts;
+ private final Timer requestTimes;
+ long handlerStart = System.currentTimeMillis();
+
+ public RequestHandlerBase() {
+ String scope = new String("metrics-scope-" + handlerNumber.getAndIncrement());
+ numRequests = Metrics.newCounter(RequestHandlerBase.class, "numRequests", scope);
+ numErrors = Metrics.newCounter(RequestHandlerBase.class, "numErrors", scope);
+ numTimeouts = Metrics.newCounter(RequestHandlerBase.class, "numTimeouts", scope);
+ requestTimes = Metrics.newTimer(RequestHandlerBase.class, "requestTimes", scope);
+ }
/**
* Initializes the {@link org.apache.solr.request.SolrRequestHandler} by creating three {@link org.apache.solr.common.params.SolrParams} named.
@@ -93,7 +106,7 @@ public abstract class RequestHandlerBase
public void init(NamedList args) {
initArgs = args;
- // Copied from StandardRequestHandler
+ // Copied from StandardRequestHandler
if( args != null ) {
Object o = args.get("defaults");
if (o != null && o instanceof NamedList) {
@@ -113,6 +126,7 @@ public abstract class RequestHandlerBase
Object caching = initArgs.get("httpCaching");
httpCaching = caching != null ? Boolean.parseBoolean(caching.toString()) : true;
}
+
}
public NamedList getInitArgs() {
@@ -122,7 +136,8 @@ public abstract class RequestHandlerBase
public abstract void handleRequestBody( SolrQueryRequest req, SolrQueryResponse rsp ) throws Exception;
public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
- numRequests++;
+ numRequests.inc();
+ TimerContext timer = requestTimes.time();
try {
SolrPluginUtils.setDefaults(req,defaults,appends,invariants);
rsp.setHttpCaching(httpCaching);
@@ -133,7 +148,7 @@ public abstract class RequestHandlerBase
Object partialResults = header.get("partialResults");
boolean timedOut = partialResults == null ? false : (Boolean)partialResults;
if( timedOut ) {
- numTimeouts++;
+ numTimeouts.inc();
rsp.setHttpCaching(false);
}
}
@@ -154,11 +169,12 @@ public abstract class RequestHandlerBase
}
rsp.setException(e);
- numErrors++;
+ numErrors.inc();
+ }
+ finally {
+ timer.stop();
}
- totalTime += rsp.getEndTime() - req.getStartTime();
}
-
//////////////////////// SolrInfoMBeans methods //////////////////////
@@ -184,12 +200,20 @@ public abstract class RequestHandlerBase
public NamedList<Object> getStatistics() {
NamedList<Object> lst = new SimpleOrderedMap<Object>();
lst.add("handlerStart",handlerStart);
- lst.add("requests", numRequests);
- lst.add("errors", numErrors);
- lst.add("timeouts", numTimeouts);
- lst.add("totalTime",totalTime);
- lst.add("avgTimePerRequest", (float) totalTime / (float) this.numRequests);
- lst.add("avgRequestsPerSecond", (float) numRequests*1000 / (float)(System.currentTimeMillis()-handlerStart));
+ lst.add("requests", numRequests.count());
+ lst.add("errors", numErrors.count());
+ lst.add("timeouts", numTimeouts.count());
+ lst.add("totalTime",requestTimes.sum());
+ lst.add("avgRequestsPerSecond", requestTimes.meanRate());
+ lst.add("5minRateReqsPerSecond", requestTimes.fiveMinuteRate());
+ lst.add("15minRateReqsPerSecond", requestTimes.fifteenMinuteRate());
+ lst.add("avgTimePerRequest", requestTimes.mean());
+ Snapshot snapshot = requestTimes.getSnapshot();
+ lst.add("medianRequestTime", snapshot.getMedian());
+ lst.add("75thPcRequestTime", snapshot.get75thPercentile());
+ lst.add("95thPcRequestTime", snapshot.get95thPercentile());
+ lst.add("99thPcRequestTime", snapshot.get99thPercentile());
+ lst.add("999thPcRequestTime", snapshot.get999thPercentile());
return lst;
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java?rev=1403555&r1=1403554&r2=1403555&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java Mon Oct 29 22:13:03 2012
@@ -18,6 +18,7 @@
package org.apache.solr.core;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.StandardRequestHandler;
import org.apache.solr.request.SolrRequestHandler;
import org.junit.BeforeClass;
@@ -87,4 +88,23 @@ public class RequestHandlersTest extends
assertNull( core.getRequestHandler("/update/asdgadsgas" ) ); // prefix
}
+
+ @Test
+ public void testStatistics() {
+ SolrCore core = h.getCore();
+ SolrRequestHandler updateHandler = core.getRequestHandler("/update");
+ SolrRequestHandler termHandler = core.getRequestHandler("/terms");
+
+ assertU(adoc("id", "47",
+ "text", "line up and fly directly at the enemy death cannons, clogging them with wreckage!"));
+ assertU(commit());
+
+ NamedList updateStats = updateHandler.getStatistics();
+ NamedList termStats = termHandler.getStatistics();
+
+ Double updateTime = (Double) updateStats.get("totalTime");
+ Double termTime = (Double) termStats.get("totalTime");
+
+ assertFalse("RequestHandlers should not share statistics!", updateTime.equals(termTime));
+ }
}
Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java?rev=1403555&r1=1403554&r2=1403555&view=diff
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java (original)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java Mon Oct 29 22:13:03 2012
@@ -42,7 +42,8 @@ public class SolrIgnoredThreadsFilter im
return true;
}
- if (threadName.startsWith("facetExecutor-") ||
+ if (threadName.startsWith("facetExecutor-") ||
+ threadName.startsWith("metrics-") ||
threadName.startsWith("cmdDistribExecutor-") ||
threadName.startsWith("httpShardExecutor-")) {
return true;
Re: svn commit: r1403555 - in /lucene/dev/trunk/solr: ./ core/ core/src/java/org/apache/solr/handler/ core/src/test/org/apache/solr/core/ test-framework/src/java/org/apache/solr/
Posted by Alan Woodward <al...@romseysoftware.co.uk>.
Aargh, sorry about that, forgot to run ant precommit.
Apologies for the noise, everyone.
On 29 Oct 2012, at 23:02, Robert Muir wrote:
> the new ivy jar will want .sha1's and licenses files. (builds will fail)
>
> you can generate the .sha1s with ant jar-checksums. but the
> license/notice you have to put in licenses/ manually.
>
> On Mon, Oct 29, 2012 at 6:13 PM, <ro...@apache.org> wrote:
>> Author: romseygeek
>> Date: Mon Oct 29 22:13:03 2012
>> New Revision: 1403555
>>
>> URL: http://svn.apache.org/viewvc?rev=1403555&view=rev
>> Log:
>> SOLR-1972: Add extra query stats to RequestHandler
>>
>> Modified:
>> lucene/dev/trunk/solr/CHANGES.txt
>> lucene/dev/trunk/solr/core/ivy.xml
>> lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
>> lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
>> lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
>>
>> Modified: lucene/dev/trunk/solr/CHANGES.txt
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1403555&r1=1403554&r2=1403555&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/CHANGES.txt (original)
>> +++ lucene/dev/trunk/solr/CHANGES.txt Mon Oct 29 22:13:03 2012
>> @@ -55,6 +55,10 @@ New Features
>> * SOLR-3911: Make Directory and DirectoryFactory first class so that the majority
>> of Solr's features work with any custom implementations. (Mark Miller)
>>
>> +* SOLR-1972: Add extra statistics to RequestHandlers - 5 & 15-minute reqs/sec
>> + rolling averages; median, 75th, 95th, 99th, 99.9th percentile request times
>> + (Alan Woodward)
>> +
>> Optimizations
>> ----------------------
>>
>>
>> Modified: lucene/dev/trunk/solr/core/ivy.xml
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/ivy.xml?rev=1403555&r1=1403554&r2=1403555&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/ivy.xml (original)
>> +++ lucene/dev/trunk/solr/core/ivy.xml Mon Oct 29 22:13:03 2012
>> @@ -28,6 +28,7 @@
>> <dependency org="org.easymock" name="easymock" rev="2.2" transitive="false"/>
>> <dependency org="com.spatial4j" name="spatial4j" rev="0.3" transitive="false"/>
>> <dependency org="javax.servlet" name="javax.servlet-api" rev="3.0.1" transitive="false"/>
>> + <dependency org="com.yammer.metrics" name="metrics-core" rev="2.1.2" transitive="false"/>
>> <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
>> </dependencies>
>> </ivy-module>
>>
>> Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java?rev=1403555&r1=1403554&r2=1403555&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java (original)
>> +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java Mon Oct 29 22:13:03 2012
>> @@ -17,6 +17,11 @@
>>
>> package org.apache.solr.handler;
>>
>> +import com.yammer.metrics.Metrics;
>> +import com.yammer.metrics.core.Counter;
>> +import com.yammer.metrics.core.Timer;
>> +import com.yammer.metrics.core.TimerContext;
>> +import com.yammer.metrics.stats.Snapshot;
>> import org.apache.lucene.queryparser.classic.ParseException;
>> import org.apache.solr.common.SolrException;
>> import org.apache.solr.common.params.SolrParams;
>> @@ -30,26 +35,34 @@ import org.apache.solr.response.SolrQuer
>> import org.apache.solr.util.SolrPluginUtils;
>>
>> import java.net.URL;
>> +import java.util.concurrent.atomic.AtomicLong;
>>
>> /**
>> *
>> */
>> public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfoMBean {
>>
>> - // statistics
>> - // TODO: should we bother synchronizing these, or is an off-by-one error
>> - // acceptable every million requests or so?
>> - volatile long numRequests;
>> - volatile long numErrors;
>> - volatile long numTimeouts;
>> protected NamedList initArgs = null;
>> protected SolrParams defaults;
>> protected SolrParams appends;
>> protected SolrParams invariants;
>> - volatile long totalTime = 0;
>> - long handlerStart = System.currentTimeMillis();
>> protected boolean httpCaching = true;
>>
>> + // Statistics
>> + private static final AtomicLong handlerNumber = new AtomicLong();
>> + private final Counter numRequests;
>> + private final Counter numErrors;
>> + private final Counter numTimeouts;
>> + private final Timer requestTimes;
>> + long handlerStart = System.currentTimeMillis();
>> +
>> + public RequestHandlerBase() {
>> + String scope = new String("metrics-scope-" + handlerNumber.getAndIncrement());
>> + numRequests = Metrics.newCounter(RequestHandlerBase.class, "numRequests", scope);
>> + numErrors = Metrics.newCounter(RequestHandlerBase.class, "numErrors", scope);
>> + numTimeouts = Metrics.newCounter(RequestHandlerBase.class, "numTimeouts", scope);
>> + requestTimes = Metrics.newTimer(RequestHandlerBase.class, "requestTimes", scope);
>> + }
>>
>> /**
>> * Initializes the {@link org.apache.solr.request.SolrRequestHandler} by creating three {@link org.apache.solr.common.params.SolrParams} named.
>> @@ -93,7 +106,7 @@ public abstract class RequestHandlerBase
>> public void init(NamedList args) {
>> initArgs = args;
>>
>> - // Copied from StandardRequestHandler
>> + // Copied from StandardRequestHandler
>> if( args != null ) {
>> Object o = args.get("defaults");
>> if (o != null && o instanceof NamedList) {
>> @@ -113,6 +126,7 @@ public abstract class RequestHandlerBase
>> Object caching = initArgs.get("httpCaching");
>> httpCaching = caching != null ? Boolean.parseBoolean(caching.toString()) : true;
>> }
>> +
>> }
>>
>> public NamedList getInitArgs() {
>> @@ -122,7 +136,8 @@ public abstract class RequestHandlerBase
>> public abstract void handleRequestBody( SolrQueryRequest req, SolrQueryResponse rsp ) throws Exception;
>>
>> public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
>> - numRequests++;
>> + numRequests.inc();
>> + TimerContext timer = requestTimes.time();
>> try {
>> SolrPluginUtils.setDefaults(req,defaults,appends,invariants);
>> rsp.setHttpCaching(httpCaching);
>> @@ -133,7 +148,7 @@ public abstract class RequestHandlerBase
>> Object partialResults = header.get("partialResults");
>> boolean timedOut = partialResults == null ? false : (Boolean)partialResults;
>> if( timedOut ) {
>> - numTimeouts++;
>> + numTimeouts.inc();
>> rsp.setHttpCaching(false);
>> }
>> }
>> @@ -154,11 +169,12 @@ public abstract class RequestHandlerBase
>> }
>>
>> rsp.setException(e);
>> - numErrors++;
>> + numErrors.inc();
>> + }
>> + finally {
>> + timer.stop();
>> }
>> - totalTime += rsp.getEndTime() - req.getStartTime();
>> }
>> -
>>
>> //////////////////////// SolrInfoMBeans methods //////////////////////
>>
>> @@ -184,12 +200,20 @@ public abstract class RequestHandlerBase
>> public NamedList<Object> getStatistics() {
>> NamedList<Object> lst = new SimpleOrderedMap<Object>();
>> lst.add("handlerStart",handlerStart);
>> - lst.add("requests", numRequests);
>> - lst.add("errors", numErrors);
>> - lst.add("timeouts", numTimeouts);
>> - lst.add("totalTime",totalTime);
>> - lst.add("avgTimePerRequest", (float) totalTime / (float) this.numRequests);
>> - lst.add("avgRequestsPerSecond", (float) numRequests*1000 / (float)(System.currentTimeMillis()-handlerStart));
>> + lst.add("requests", numRequests.count());
>> + lst.add("errors", numErrors.count());
>> + lst.add("timeouts", numTimeouts.count());
>> + lst.add("totalTime",requestTimes.sum());
>> + lst.add("avgRequestsPerSecond", requestTimes.meanRate());
>> + lst.add("5minRateReqsPerSecond", requestTimes.fiveMinuteRate());
>> + lst.add("15minRateReqsPerSecond", requestTimes.fifteenMinuteRate());
>> + lst.add("avgTimePerRequest", requestTimes.mean());
>> + Snapshot snapshot = requestTimes.getSnapshot();
>> + lst.add("medianRequestTime", snapshot.getMedian());
>> + lst.add("75thPcRequestTime", snapshot.get75thPercentile());
>> + lst.add("95thPcRequestTime", snapshot.get95thPercentile());
>> + lst.add("99thPcRequestTime", snapshot.get99thPercentile());
>> + lst.add("999thPcRequestTime", snapshot.get999thPercentile());
>> return lst;
>> }
>>
>>
>> Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java?rev=1403555&r1=1403554&r2=1403555&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java (original)
>> +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java Mon Oct 29 22:13:03 2012
>> @@ -18,6 +18,7 @@
>> package org.apache.solr.core;
>>
>> import org.apache.solr.SolrTestCaseJ4;
>> +import org.apache.solr.common.util.NamedList;
>> import org.apache.solr.handler.StandardRequestHandler;
>> import org.apache.solr.request.SolrRequestHandler;
>> import org.junit.BeforeClass;
>> @@ -87,4 +88,23 @@ public class RequestHandlersTest extends
>>
>> assertNull( core.getRequestHandler("/update/asdgadsgas" ) ); // prefix
>> }
>> +
>> + @Test
>> + public void testStatistics() {
>> + SolrCore core = h.getCore();
>> + SolrRequestHandler updateHandler = core.getRequestHandler("/update");
>> + SolrRequestHandler termHandler = core.getRequestHandler("/terms");
>> +
>> + assertU(adoc("id", "47",
>> + "text", "line up and fly directly at the enemy death cannons, clogging them with wreckage!"));
>> + assertU(commit());
>> +
>> + NamedList updateStats = updateHandler.getStatistics();
>> + NamedList termStats = termHandler.getStatistics();
>> +
>> + Double updateTime = (Double) updateStats.get("totalTime");
>> + Double termTime = (Double) termStats.get("totalTime");
>> +
>> + assertFalse("RequestHandlers should not share statistics!", updateTime.equals(termTime));
>> + }
>> }
>>
>> Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java?rev=1403555&r1=1403554&r2=1403555&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java (original)
>> +++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java Mon Oct 29 22:13:03 2012
>> @@ -42,7 +42,8 @@ public class SolrIgnoredThreadsFilter im
>> return true;
>> }
>>
>> - if (threadName.startsWith("facetExecutor-") ||
>> + if (threadName.startsWith("facetExecutor-") ||
>> + threadName.startsWith("metrics-") ||
>> threadName.startsWith("cmdDistribExecutor-") ||
>> threadName.startsWith("httpShardExecutor-")) {
>> return true;
>>
>>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1403555 - in /lucene/dev/trunk/solr: ./ core/
core/src/java/org/apache/solr/handler/ core/src/test/org/apache/solr/core/ test-framework/src/java/org/apache/solr/
Posted by Robert Muir <rc...@gmail.com>.
the new ivy jar will want .sha1's and licenses files. (builds will fail)
you can generate the .sha1s with ant jar-checksums. but the
license/notice you have to put in licenses/ manually.
On Mon, Oct 29, 2012 at 6:13 PM, <ro...@apache.org> wrote:
> Author: romseygeek
> Date: Mon Oct 29 22:13:03 2012
> New Revision: 1403555
>
> URL: http://svn.apache.org/viewvc?rev=1403555&view=rev
> Log:
> SOLR-1972: Add extra query stats to RequestHandler
>
> Modified:
> lucene/dev/trunk/solr/CHANGES.txt
> lucene/dev/trunk/solr/core/ivy.xml
> lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
> lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
> lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
>
> Modified: lucene/dev/trunk/solr/CHANGES.txt
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1403555&r1=1403554&r2=1403555&view=diff
> ==============================================================================
> --- lucene/dev/trunk/solr/CHANGES.txt (original)
> +++ lucene/dev/trunk/solr/CHANGES.txt Mon Oct 29 22:13:03 2012
> @@ -55,6 +55,10 @@ New Features
> * SOLR-3911: Make Directory and DirectoryFactory first class so that the majority
> of Solr's features work with any custom implementations. (Mark Miller)
>
> +* SOLR-1972: Add extra statistics to RequestHandlers - 5 & 15-minute reqs/sec
> + rolling averages; median, 75th, 95th, 99th, 99.9th percentile request times
> + (Alan Woodward)
> +
> Optimizations
> ----------------------
>
>
> Modified: lucene/dev/trunk/solr/core/ivy.xml
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/ivy.xml?rev=1403555&r1=1403554&r2=1403555&view=diff
> ==============================================================================
> --- lucene/dev/trunk/solr/core/ivy.xml (original)
> +++ lucene/dev/trunk/solr/core/ivy.xml Mon Oct 29 22:13:03 2012
> @@ -28,6 +28,7 @@
> <dependency org="org.easymock" name="easymock" rev="2.2" transitive="false"/>
> <dependency org="com.spatial4j" name="spatial4j" rev="0.3" transitive="false"/>
> <dependency org="javax.servlet" name="javax.servlet-api" rev="3.0.1" transitive="false"/>
> + <dependency org="com.yammer.metrics" name="metrics-core" rev="2.1.2" transitive="false"/>
> <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
> </dependencies>
> </ivy-module>
>
> Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java?rev=1403555&r1=1403554&r2=1403555&view=diff
> ==============================================================================
> --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java (original)
> +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java Mon Oct 29 22:13:03 2012
> @@ -17,6 +17,11 @@
>
> package org.apache.solr.handler;
>
> +import com.yammer.metrics.Metrics;
> +import com.yammer.metrics.core.Counter;
> +import com.yammer.metrics.core.Timer;
> +import com.yammer.metrics.core.TimerContext;
> +import com.yammer.metrics.stats.Snapshot;
> import org.apache.lucene.queryparser.classic.ParseException;
> import org.apache.solr.common.SolrException;
> import org.apache.solr.common.params.SolrParams;
> @@ -30,26 +35,34 @@ import org.apache.solr.response.SolrQuer
> import org.apache.solr.util.SolrPluginUtils;
>
> import java.net.URL;
> +import java.util.concurrent.atomic.AtomicLong;
>
> /**
> *
> */
> public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfoMBean {
>
> - // statistics
> - // TODO: should we bother synchronizing these, or is an off-by-one error
> - // acceptable every million requests or so?
> - volatile long numRequests;
> - volatile long numErrors;
> - volatile long numTimeouts;
> protected NamedList initArgs = null;
> protected SolrParams defaults;
> protected SolrParams appends;
> protected SolrParams invariants;
> - volatile long totalTime = 0;
> - long handlerStart = System.currentTimeMillis();
> protected boolean httpCaching = true;
>
> + // Statistics
> + private static final AtomicLong handlerNumber = new AtomicLong();
> + private final Counter numRequests;
> + private final Counter numErrors;
> + private final Counter numTimeouts;
> + private final Timer requestTimes;
> + long handlerStart = System.currentTimeMillis();
> +
> + public RequestHandlerBase() {
> + String scope = new String("metrics-scope-" + handlerNumber.getAndIncrement());
> + numRequests = Metrics.newCounter(RequestHandlerBase.class, "numRequests", scope);
> + numErrors = Metrics.newCounter(RequestHandlerBase.class, "numErrors", scope);
> + numTimeouts = Metrics.newCounter(RequestHandlerBase.class, "numTimeouts", scope);
> + requestTimes = Metrics.newTimer(RequestHandlerBase.class, "requestTimes", scope);
> + }
>
> /**
> * Initializes the {@link org.apache.solr.request.SolrRequestHandler} by creating three {@link org.apache.solr.common.params.SolrParams} named.
> @@ -93,7 +106,7 @@ public abstract class RequestHandlerBase
> public void init(NamedList args) {
> initArgs = args;
>
> - // Copied from StandardRequestHandler
> + // Copied from StandardRequestHandler
> if( args != null ) {
> Object o = args.get("defaults");
> if (o != null && o instanceof NamedList) {
> @@ -113,6 +126,7 @@ public abstract class RequestHandlerBase
> Object caching = initArgs.get("httpCaching");
> httpCaching = caching != null ? Boolean.parseBoolean(caching.toString()) : true;
> }
> +
> }
>
> public NamedList getInitArgs() {
> @@ -122,7 +136,8 @@ public abstract class RequestHandlerBase
> public abstract void handleRequestBody( SolrQueryRequest req, SolrQueryResponse rsp ) throws Exception;
>
> public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
> - numRequests++;
> + numRequests.inc();
> + TimerContext timer = requestTimes.time();
> try {
> SolrPluginUtils.setDefaults(req,defaults,appends,invariants);
> rsp.setHttpCaching(httpCaching);
> @@ -133,7 +148,7 @@ public abstract class RequestHandlerBase
> Object partialResults = header.get("partialResults");
> boolean timedOut = partialResults == null ? false : (Boolean)partialResults;
> if( timedOut ) {
> - numTimeouts++;
> + numTimeouts.inc();
> rsp.setHttpCaching(false);
> }
> }
> @@ -154,11 +169,12 @@ public abstract class RequestHandlerBase
> }
>
> rsp.setException(e);
> - numErrors++;
> + numErrors.inc();
> + }
> + finally {
> + timer.stop();
> }
> - totalTime += rsp.getEndTime() - req.getStartTime();
> }
> -
>
> //////////////////////// SolrInfoMBeans methods //////////////////////
>
> @@ -184,12 +200,20 @@ public abstract class RequestHandlerBase
> public NamedList<Object> getStatistics() {
> NamedList<Object> lst = new SimpleOrderedMap<Object>();
> lst.add("handlerStart",handlerStart);
> - lst.add("requests", numRequests);
> - lst.add("errors", numErrors);
> - lst.add("timeouts", numTimeouts);
> - lst.add("totalTime",totalTime);
> - lst.add("avgTimePerRequest", (float) totalTime / (float) this.numRequests);
> - lst.add("avgRequestsPerSecond", (float) numRequests*1000 / (float)(System.currentTimeMillis()-handlerStart));
> + lst.add("requests", numRequests.count());
> + lst.add("errors", numErrors.count());
> + lst.add("timeouts", numTimeouts.count());
> + lst.add("totalTime",requestTimes.sum());
> + lst.add("avgRequestsPerSecond", requestTimes.meanRate());
> + lst.add("5minRateReqsPerSecond", requestTimes.fiveMinuteRate());
> + lst.add("15minRateReqsPerSecond", requestTimes.fifteenMinuteRate());
> + lst.add("avgTimePerRequest", requestTimes.mean());
> + Snapshot snapshot = requestTimes.getSnapshot();
> + lst.add("medianRequestTime", snapshot.getMedian());
> + lst.add("75thPcRequestTime", snapshot.get75thPercentile());
> + lst.add("95thPcRequestTime", snapshot.get95thPercentile());
> + lst.add("99thPcRequestTime", snapshot.get99thPercentile());
> + lst.add("999thPcRequestTime", snapshot.get999thPercentile());
> return lst;
> }
>
>
> Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java?rev=1403555&r1=1403554&r2=1403555&view=diff
> ==============================================================================
> --- lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java (original)
> +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java Mon Oct 29 22:13:03 2012
> @@ -18,6 +18,7 @@
> package org.apache.solr.core;
>
> import org.apache.solr.SolrTestCaseJ4;
> +import org.apache.solr.common.util.NamedList;
> import org.apache.solr.handler.StandardRequestHandler;
> import org.apache.solr.request.SolrRequestHandler;
> import org.junit.BeforeClass;
> @@ -87,4 +88,23 @@ public class RequestHandlersTest extends
>
> assertNull( core.getRequestHandler("/update/asdgadsgas" ) ); // prefix
> }
> +
> + @Test
> + public void testStatistics() {
> + SolrCore core = h.getCore();
> + SolrRequestHandler updateHandler = core.getRequestHandler("/update");
> + SolrRequestHandler termHandler = core.getRequestHandler("/terms");
> +
> + assertU(adoc("id", "47",
> + "text", "line up and fly directly at the enemy death cannons, clogging them with wreckage!"));
> + assertU(commit());
> +
> + NamedList updateStats = updateHandler.getStatistics();
> + NamedList termStats = termHandler.getStatistics();
> +
> + Double updateTime = (Double) updateStats.get("totalTime");
> + Double termTime = (Double) termStats.get("totalTime");
> +
> + assertFalse("RequestHandlers should not share statistics!", updateTime.equals(termTime));
> + }
> }
>
> Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java?rev=1403555&r1=1403554&r2=1403555&view=diff
> ==============================================================================
> --- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java (original)
> +++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java Mon Oct 29 22:13:03 2012
> @@ -42,7 +42,8 @@ public class SolrIgnoredThreadsFilter im
> return true;
> }
>
> - if (threadName.startsWith("facetExecutor-") ||
> + if (threadName.startsWith("facetExecutor-") ||
> + threadName.startsWith("metrics-") ||
> threadName.startsWith("cmdDistribExecutor-") ||
> threadName.startsWith("httpShardExecutor-")) {
> return true;
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org