You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2012/12/14 05:59:35 UTC
svn commit: r1421670 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/codecs/ lucene/core/
lucene/core/src/test/org/apache/l...
Author: yonik
Date: Fri Dec 14 04:59:33 2012
New Revision: 1421670
URL: http://svn.apache.org/viewvc?rev=1421670&view=rev
Log:
SOLR-2592: test expected number of requests to servers
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/site/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/testlogging.properties (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java?rev=1421670&r1=1421669&r2=1421670&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java Fri Dec 14 04:59:33 2012
@@ -19,13 +19,17 @@ package org.apache.solr.client.solrj.emb
import java.io.IOException;
import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
+import java.util.LinkedList;
import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
import javax.servlet.DispatcherType;
import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -53,6 +57,7 @@ public class JettySolrRunner {
Server server;
FilterHolder dispatchFilter;
+ FilterHolder debugFilter;
String context;
@@ -73,6 +78,53 @@ public class JettySolrRunner {
private boolean stopAtShutdown;
+ public static class DebugFilter implements Filter {
+ public int requestsToKeep = 10;
+ private AtomicLong nRequests = new AtomicLong();
+
+ public long getTotalRequests() {
+ return nRequests.get();
+
+ }
+
+ // TODO: keep track of certain number of last requests
+ private LinkedList<HttpServletRequest> requests = new LinkedList<HttpServletRequest>();
+
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ }
+
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+ nRequests.incrementAndGet();
+
+ /***
+ HttpServletRequest req = (HttpServletRequest)servletRequest;
+ HttpServletResponse resp = (HttpServletResponse)servletResponse;
+
+ String path = req.getServletPath();
+ if( req.getPathInfo() != null ) {
+ // this lets you handle /update/commit when /update is a servlet
+ path += req.getPathInfo();
+ }
+ System.out.println("###################### FILTER request " + servletRequest);
+ System.out.println("\t\tgetServletPath="+req.getServletPath());
+ System.out.println("\t\tgetPathInfo="+req.getPathInfo());
+ ***/
+
+ filterChain.doFilter(servletRequest, servletResponse);
+ }
+
+ @Override
+ public void destroy() {
+ }
+ }
+
+
+
+
+
public JettySolrRunner(String solrHome, String context, int port) {
this.init(solrHome, context, port, true);
}
@@ -174,6 +226,7 @@ public class JettySolrRunner {
schemaFilename);
// SolrDispatchFilter filter = new SolrDispatchFilter();
// FilterHolder fh = new FilterHolder(filter);
+ debugFilter = root.addFilter(DebugFilter.class, "*", EnumSet.of(DispatcherType.REQUEST) );
dispatchFilter = root.addFilter(SolrDispatchFilter.class, "*", EnumSet.of(DispatcherType.REQUEST) );
if (solrConfigFilename != null) System.clearProperty("solrconfig");
if (schemaFilename != null) System.clearProperty("schema");
@@ -295,6 +348,10 @@ public class JettySolrRunner {
return lastPort;
}
+ public DebugFilter getDebugFilter() {
+ return (DebugFilter)debugFilter.getFilter();
+ }
+
// --------------------------------------------------------------
// --------------------------------------------------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java?rev=1421670&r1=1421669&r2=1421670&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java Fri Dec 14 04:59:33 2012
@@ -45,6 +45,13 @@ import java.util.Set;
public class ShardRoutingTest extends AbstractFullDistribZkTestBase {
+
+ String bucket1 = "shard1"; // shard1: top bits:10 80000000:bfffffff
+ String bucket2 = "shard2"; // shard2: top bits:11 c0000000:ffffffff
+ String bucket3 = "shard3"; // shard3: top bits:00 00000000:3fffffff
+ String bucket4 = "shard4"; // shard4: top bits:01 40000000:7fffffff
+
+
@BeforeClass
public static void beforeShardHashingTest() throws Exception {
// TODO: we use an fs based dir because something
@@ -109,6 +116,7 @@ public class ShardRoutingTest extends Ab
waitForRecoveriesToFinish(false);
doHashingTest();
+ doTestNumRequests();
testFinished = true;
} finally {
@@ -127,10 +135,6 @@ public class ShardRoutingTest extends Ab
// for now, we know how ranges will be distributed to shards.
// may have to look it up in clusterstate if that assumption changes.
- String bucket1 = "shard1"; // shard1: top bits:10 80000000:bfffffff
- String bucket2 = "shard2"; // shard2: top bits:11 c0000000:ffffffff
- String bucket3 = "shard3"; // shard3: top bits:00 00000000:3fffffff
- String bucket4 = "shard4"; // shard4: top bits:01 40000000:7fffffff
doAddDoc("b!doc1");
doAddDoc("c!doc2");
@@ -196,10 +200,73 @@ public class ShardRoutingTest extends Ab
doAddDoc("e!doc4");
commit();
+ }
+
+
+ public void doTestNumRequests() throws Exception {
+ List<CloudJettyRunner> runners = shardToJetty.get(bucket1);
+ CloudJettyRunner leader = shardToLeaderJetty.get(bucket1);
+ CloudJettyRunner replica = null;
+ for (CloudJettyRunner r : runners) {
+ if (r != leader) replica = r;
+ }
+
+ long nStart = getNumRequests();
+ leader.client.solrClient.add( sdoc("id","b!doc1") );
+ long nEnd = getNumRequests();
+ assertEquals(2, nEnd - nStart); // one request to leader, which makes another to a replica
+
+
+ nStart = getNumRequests();
+ replica.client.solrClient.add( sdoc("id","b!doc1") );
+ nEnd = getNumRequests();
+ assertEquals(3, nEnd - nStart); // orig request + replica forwards to leader, which forward back to replica.
+
+ nStart = getNumRequests();
+ replica.client.solrClient.add( sdoc("id","b!doc1") );
+ nEnd = getNumRequests();
+ assertEquals(3, nEnd - nStart); // orig request + replica forwards to leader, which forward back to replica.
+
+ CloudJettyRunner leader2 = shardToLeaderJetty.get(bucket2);
+
+
+ nStart = getNumRequests();
+ replica.client.solrClient.query( params("q","*:*", "shards",bucket1) );
+ nEnd = getNumRequests();
+ assertEquals(1, nEnd - nStart); // short circuit should prevent distrib search
+
+ nStart = getNumRequests();
+ replica.client.solrClient.query( params("q","*:*", "shard.keys","b!") );
+ nEnd = getNumRequests();
+ assertEquals(1, nEnd - nStart); // short circuit should prevent distrib search
+
+ nStart = getNumRequests();
+ leader2.client.solrClient.query( params("q","*:*", "shard.keys","b!") );
+ nEnd = getNumRequests();
+ assertEquals(3, nEnd - nStart); // original + 2 phase distrib search. we could improve this!
+
+ nStart = getNumRequests();
+ leader2.client.solrClient.query( params("q","*:*") );
+ nEnd = getNumRequests();
+ assertEquals(9, nEnd - nStart); // original + 2 phase distrib search * 4 shards.
+
+ nStart = getNumRequests();
+ leader2.client.solrClient.query( params("q","*:*", "shard.keys","b!,d!") );
+ nEnd = getNumRequests();
+ assertEquals(5, nEnd - nStart); // original + 2 phase distrib search * 2 shards.
+ }
+
+ long getNumRequests() {
+ long n = controlJetty.getDebugFilter().getTotalRequests();
+ for (JettySolrRunner jetty : jettys) {
+ n += jetty.getDebugFilter().getTotalRequests();
+ }
+ return n;
}
+
void doAddDoc(String id) throws Exception {
index("id",id);
// todo - target diff servers and use cloud clients as well as non-cloud clients