You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2012/12/14 05:59:35 UTC

svn commit: r1421670 - in /lucene/dev/branches/branch_4x: ./ dev-tools/ lucene/ lucene/analysis/ lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/ lucene/benchmark/ lucene/codecs/ lucene/core/ lucene/core/src/test/org/apache/l...

Author: yonik
Date: Fri Dec 14 04:59:33 2012
New Revision: 1421670

URL: http://svn.apache.org/viewvc?rev=1421670&view=rev
Log:
SOLR-2592: test expected number of requests to servers

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/dev-tools/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/BUILD.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/LICENSE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/MIGRATE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/README.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/   (props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java   (props changed)
    lucene/dev/branches/branch_4x/lucene/backwards/   (props changed)
    lucene/dev/branches/branch_4x/lucene/benchmark/   (props changed)
    lucene/dev/branches/branch_4x/lucene/build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/codecs/   (props changed)
    lucene/dev/branches/branch_4x/lucene/common-build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip   (props changed)
    lucene/dev/branches/branch_4x/lucene/demo/   (props changed)
    lucene/dev/branches/branch_4x/lucene/facet/   (props changed)
    lucene/dev/branches/branch_4x/lucene/grouping/   (props changed)
    lucene/dev/branches/branch_4x/lucene/highlighter/   (props changed)
    lucene/dev/branches/branch_4x/lucene/ivy-settings.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/join/   (props changed)
    lucene/dev/branches/branch_4x/lucene/licenses/   (props changed)
    lucene/dev/branches/branch_4x/lucene/memory/   (props changed)
    lucene/dev/branches/branch_4x/lucene/misc/   (props changed)
    lucene/dev/branches/branch_4x/lucene/module-build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/queries/   (props changed)
    lucene/dev/branches/branch_4x/lucene/queryparser/   (props changed)
    lucene/dev/branches/branch_4x/lucene/sandbox/   (props changed)
    lucene/dev/branches/branch_4x/lucene/site/   (props changed)
    lucene/dev/branches/branch_4x/lucene/spatial/   (props changed)
    lucene/dev/branches/branch_4x/lucene/suggest/   (props changed)
    lucene/dev/branches/branch_4x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/lucene/tools/   (props changed)
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/LICENSE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/README.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/build.xml   (props changed)
    lucene/dev/branches/branch_4x/solr/cloud-dev/   (props changed)
    lucene/dev/branches/branch_4x/solr/common-build.xml   (props changed)
    lucene/dev/branches/branch_4x/solr/contrib/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java
    lucene/dev/branches/branch_4x/solr/example/   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/scripts/   (props changed)
    lucene/dev/branches/branch_4x/solr/site/   (props changed)
    lucene/dev/branches/branch_4x/solr/solrj/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/solr/testlogging.properties   (props changed)
    lucene/dev/branches/branch_4x/solr/webapp/   (props changed)

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java?rev=1421670&r1=1421669&r2=1421670&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java Fri Dec 14 04:59:33 2012
@@ -19,13 +19,17 @@ package org.apache.solr.client.solrj.emb
 
 import java.io.IOException;
 import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
+import java.util.LinkedList;
 import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
 
 import javax.servlet.DispatcherType;
 import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -53,6 +57,7 @@ public class JettySolrRunner {
   Server server;
 
   FilterHolder dispatchFilter;
+  FilterHolder debugFilter;
 
   String context;
 
@@ -73,6 +78,53 @@ public class JettySolrRunner {
 
   private boolean stopAtShutdown;
 
+  public static class DebugFilter implements Filter {
+    public int requestsToKeep = 10;
+    private AtomicLong nRequests = new AtomicLong();
+
+    public long getTotalRequests() {
+      return nRequests.get();
+
+    }
+
+    // TODO: keep track of certain number of last requests
+    private LinkedList<HttpServletRequest> requests = new LinkedList<HttpServletRequest>();
+
+
+    @Override
+    public void init(FilterConfig filterConfig) throws ServletException {
+    }
+
+    @Override
+    public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+      nRequests.incrementAndGet();
+
+      /***
+      HttpServletRequest req = (HttpServletRequest)servletRequest;
+      HttpServletResponse resp = (HttpServletResponse)servletResponse;
+
+      String path = req.getServletPath();
+      if( req.getPathInfo() != null ) {
+        // this lets you handle /update/commit when /update is a servlet
+        path += req.getPathInfo();
+      }
+      System.out.println("###################### FILTER request " + servletRequest);
+      System.out.println("\t\tgetServletPath="+req.getServletPath());
+      System.out.println("\t\tgetPathInfo="+req.getPathInfo());
+      ***/
+
+      filterChain.doFilter(servletRequest, servletResponse);
+    }
+
+    @Override
+    public void destroy() {
+    }
+  }
+
+
+
+
+
   public JettySolrRunner(String solrHome, String context, int port) {
     this.init(solrHome, context, port, true);
   }
@@ -174,6 +226,7 @@ public class JettySolrRunner {
             schemaFilename);
 //        SolrDispatchFilter filter = new SolrDispatchFilter();
 //        FilterHolder fh = new FilterHolder(filter);
+        debugFilter = root.addFilter(DebugFilter.class, "*", EnumSet.of(DispatcherType.REQUEST) );
         dispatchFilter = root.addFilter(SolrDispatchFilter.class, "*", EnumSet.of(DispatcherType.REQUEST) );
         if (solrConfigFilename != null) System.clearProperty("solrconfig");
         if (schemaFilename != null) System.clearProperty("schema");
@@ -295,6 +348,10 @@ public class JettySolrRunner {
     return lastPort;
   }
 
+  public DebugFilter getDebugFilter() {
+    return (DebugFilter)debugFilter.getFilter();
+  }
+
   // --------------------------------------------------------------
   // --------------------------------------------------------------
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java?rev=1421670&r1=1421669&r2=1421670&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java Fri Dec 14 04:59:33 2012
@@ -45,6 +45,13 @@ import java.util.Set;
 
 
 public class ShardRoutingTest extends AbstractFullDistribZkTestBase {
+
+  String bucket1 = "shard1";      // shard1: top bits:10  80000000:bfffffff
+  String bucket2 = "shard2";      // shard2: top bits:11  c0000000:ffffffff
+  String bucket3 = "shard3";      // shard3: top bits:00  00000000:3fffffff
+  String bucket4 = "shard4";      // shard4: top bits:01  40000000:7fffffff
+
+
   @BeforeClass
   public static void beforeShardHashingTest() throws Exception {
     // TODO: we use an fs based dir because something
@@ -109,6 +116,7 @@ public class ShardRoutingTest extends Ab
       waitForRecoveriesToFinish(false);
 
       doHashingTest();
+      doTestNumRequests();
 
       testFinished = true;
     } finally {
@@ -127,10 +135,6 @@ public class ShardRoutingTest extends Ab
     // for now,  we know how ranges will be distributed to shards.
     // may have to look it up in clusterstate if that assumption changes.
 
-    String bucket1 = "shard1";      // shard1: top bits:10  80000000:bfffffff
-    String bucket2 = "shard2";      // shard2: top bits:11  c0000000:ffffffff
-    String bucket3 = "shard3";      // shard3: top bits:00  00000000:3fffffff
-    String bucket4 = "shard4";      // shard4: top bits:01  40000000:7fffffff
 
     doAddDoc("b!doc1");
     doAddDoc("c!doc2");
@@ -196,10 +200,73 @@ public class ShardRoutingTest extends Ab
     doAddDoc("e!doc4");
 
     commit();
+  }
+
+
+  public void doTestNumRequests() throws Exception {
 
+    List<CloudJettyRunner> runners = shardToJetty.get(bucket1);
+    CloudJettyRunner leader = shardToLeaderJetty.get(bucket1);
+    CloudJettyRunner replica =  null;
+    for (CloudJettyRunner r : runners) {
+      if (r != leader) replica = r;
+    }
+
+    long nStart = getNumRequests();
+    leader.client.solrClient.add( sdoc("id","b!doc1") );
+    long nEnd = getNumRequests();
+    assertEquals(2, nEnd - nStart);   // one request to leader, which makes another to a replica
+
+
+    nStart = getNumRequests();
+    replica.client.solrClient.add( sdoc("id","b!doc1") );
+    nEnd = getNumRequests();
+    assertEquals(3, nEnd - nStart);   // orig request + replica forwards to leader, which forward back to replica.
+
+    nStart = getNumRequests();
+    replica.client.solrClient.add( sdoc("id","b!doc1") );
+    nEnd = getNumRequests();
+    assertEquals(3, nEnd - nStart);   // orig request + replica forwards to leader, which forward back to replica.
+
+    CloudJettyRunner leader2 = shardToLeaderJetty.get(bucket2);
+
+
+    nStart = getNumRequests();
+    replica.client.solrClient.query( params("q","*:*", "shards",bucket1) );
+    nEnd = getNumRequests();
+    assertEquals(1, nEnd - nStart);   // short circuit should prevent distrib search
+
+    nStart = getNumRequests();
+    replica.client.solrClient.query( params("q","*:*", "shard.keys","b!") );
+    nEnd = getNumRequests();
+    assertEquals(1, nEnd - nStart);   // short circuit should prevent distrib search
+
+    nStart = getNumRequests();
+    leader2.client.solrClient.query( params("q","*:*", "shard.keys","b!") );
+    nEnd = getNumRequests();
+    assertEquals(3, nEnd - nStart);   // original + 2 phase distrib search.  we could improve this!
+
+    nStart = getNumRequests();
+    leader2.client.solrClient.query( params("q","*:*") );
+    nEnd = getNumRequests();
+    assertEquals(9, nEnd - nStart);   // original + 2 phase distrib search * 4 shards.
+
+    nStart = getNumRequests();
+    leader2.client.solrClient.query( params("q","*:*", "shard.keys","b!,d!") );
+    nEnd = getNumRequests();
+    assertEquals(5, nEnd - nStart);   // original + 2 phase distrib search * 2 shards.
+  }
 
+
+  long getNumRequests() {
+    long n = controlJetty.getDebugFilter().getTotalRequests();
+    for (JettySolrRunner jetty : jettys) {
+      n += jetty.getDebugFilter().getTotalRequests();
+    }
+    return n;
   }
 
+
   void doAddDoc(String id) throws Exception {
     index("id",id);
     // todo - target diff servers and use cloud clients as well as non-cloud clients