Too many SocketTimeoutException from solr server

[高甜甜 <ga...@sina.com>]

 Hi,

I using solr as my indexing server, but i get some error now. I deploy solr to tomcat alone, i use solrj to communication with solr server.

I get many socketTimeoutException when i use solrj to execute a query. The exception occurs very often when the query and update operation executed at the same time.

Can you help me to solve the problem? The problem occurs in production environment, so i have to solve the problem as soon as possible.

This is the error:

Caused by: org.apache.solr.client.solrj.SolrServerException: Error executing query
    at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:96)
    at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:109)
    at com.nikoyo.dm6p.application.util.SolrHttpClient.query(SolrHttpClient.java:206)
    at com.nikoyo.dm6p.application.query.service.SolrSearchImpl.getResult(SolrSearchImpl.java:90)
    at com.nikoyo.dm6p.application.query.service.SolrSearchImpl.search(SolrSearchImpl.java:134)
    at com.nikoyo.dm6p.web.server.webservice.cmhk.CmhkServiceImpl.findSingleDocType(CmhkServiceImpl.java:1982)
    ... 45 more
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.commons.httpclient.ConnectTimeoutException: The host did not accept the connection within timeout of 1000 ms
    at org.apache.solr.client.solrj.impl.CommonsHttpSolrServer.request(CommonsHttpSolrServer.java:391)
    at org.apache.solr.client.solrj.impl.CommonsHttpSolrServer.request(CommonsHttpSolrServer.java:183)
    at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
    ... 50 more
Caused by: org.apache.commons.httpclient.ConnectTimeoutException: The host did not accept the connection within timeout of 1000 ms
    at org.apache.commons.httpclient.protocol.ReflectionSocketFactory.createSocket(ReflectionSocketFactory.java:155)
    at org.apache.commons.httpclient.protocol.DefaultProtocolSocketFactory.createSocket(DefaultProtocolSocketFactory.java:125)
    at org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:707)
    at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387)
    at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:171)
    at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:397)
    at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:323)
    at org.apache.solr.client.solrj.impl.CommonsHttpSolrServer.request(CommonsHttpSolrServer.java:335)
    ... 52 more
Caused by: java.net.SocketTimeoutException: connect timed out
    at java.net.PlainSocketImpl.socketConnect(Native Method)
    at java.net.PlainSocketImpl.doConnect(PlainSocketImpl.java:333)
    at java.net.PlainSocketImpl.connectToAddress(PlainSocketImpl.java:195)
    at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:182)
    at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:366)
    at java.net.Socket.connect(Socket.java:519)
    at sun.reflect.GeneratedMethodAccessor102.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:585)
    at org.apache.commons.httpclient.protocol.ReflectionSocketFactory.createSocket(ReflectionSocketFactory.java:140)
    ... 59 more

My solrj client code:

private SolrServer getSolrServer() throws MalformedURLException {
    if (commonsHttpSolrServer == null) {
           commonsHttpSolrServer = new CommonsHttpSolrServer("http://localhost:8983/solr", new HttpClient(new SimpleHttpConnectionManager(true)));
        commonsHttpSolrServer.setSoTimeout(10000);
        commonsHttpSolrServer.setConnectionTimeout(1000);
        commonsHttpSolrServer.setDefaultMaxConnectionsPerHost(500);
        commonsHttpSolrServer.setMaxTotalConnections(1000);
        commonsHttpSolrServer.setFollowRedirects(false);
        commonsHttpSolrServer.setAllowCompression(true);
        commonsHttpSolrServer.setMaxRetries(1);
   }
   return commonsHttpSolrServer;
}

public void add(SolrInputDocument doc) {
    try {
        SolrServer solrServer = getSolrServer();
        solrServer.add(doc);
    } catch (Exception e) {
        //...
    }
}

public SolrDocumentList query(SolrQuery query) throws MalformedURLException, SolrServerException {
    SolrServer solrServer = getSolrServer();
    QueryResponse queryResponse = solrServer.query(query);

    return queryResponse.getResults();
}


This is my tomcat config:

<Connector port="8983" maxHttpHeaderSize="8192"
           maxThreads="150" minSpareThreads="25" maxSpareThreads="75"
           enableLookups="false" redirectPort="8443" acceptCount="100"
           connectionTimeout="30000" disableUploadTimeout="true" URIEncoding="UTF-8"/>


This is my solr config:

<?xml version="1.0" encoding="UTF-8" ?>
<config>
  <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>

  <dataDir>${solr.data.dir:./solr/data}</dataDir>

  <indexDefaults>
    <useCompoundFile>false</useCompoundFile>

    <mergeFactor>10</mergeFactor>
    <ramBufferSizeMB>32</ramBufferSizeMB>
    <maxMergeDocs>2147483647</maxMergeDocs>
    <maxFieldLength>10000</maxFieldLength>
    <writeLockTimeout>1000</writeLockTimeout>
    <commitLockTimeout>10000</commitLockTimeout>
    
    <lockType>single</lockType>
  </indexDefaults>

  <mainIndex>
    <useCompoundFile>false</useCompoundFile>
    <ramBufferSizeMB>32</ramBufferSizeMB>
    <mergeFactor>10</mergeFactor>
    <maxMergeDocs>2147483647</maxMergeDocs>
    <maxFieldLength>10000</maxFieldLength>
    <unlockOnStartup>false</unlockOnStartup>
  </mainIndex>
  
  <jmx />

  <updateHandler class="solr.DirectUpdateHandler2">
    <autoCommit> 
      <maxDocs>10000</maxDocs>
      <maxTime>1000</maxTime> 
    </autoCommit>
  </updateHandler>
  
  <query>
    <maxBooleanClauses>1024</maxBooleanClauses>
    <filterCache
      class="solr.LRUCache"
      size="512"
      initialSize="512"
      autowarmCount="128"/>

    <queryResultCache
      class="solr.LRUCache"
      size="512"
      initialSize="512"
      autowarmCount="32"/>

    <documentCache
      class="solr.LRUCache"
      size="512"
      initialSize="512"
      autowarmCount="0"/>

    <enableLazyFieldLoading>true</enableLazyFieldLoading>

    <queryResultWindowSize>50</queryResultWindowSize>
    
    <queryResultMaxDocsCached>200</queryResultMaxDocsCached>

    <HashDocSet maxSize="3000" loadFactor="0.75"/>

    <listener event="newSearcher" class="solr.QuerySenderListener">
      <arr name="queries">
        <lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>
        <lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>
        <lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>
      </arr>
    </listener>

    <listener event="firstSearcher" class="solr.QuerySenderListener">
      <arr name="queries">
        <lst> <str name="q">fast_warm</str> <str name="start">0</str> <str name="rows">10</str> </lst>
        <lst><str name="q">static firstSearcher warming query from solrconfig.xml</str></lst>
      </arr>
    </listener>

    <useColdSearcher>false</useColdSearcher>

    <maxWarmingSearchers>20</maxWarmingSearchers>

  </query>

  <requestDispatcher handleSelect="true" >
    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
        
    <httpCaching lastModifiedFrom="openTime"
                 etagSeed="Solr">
    </httpCaching>
  </requestDispatcher>
  
  <requestHandler name="standard" class="solr.SearchHandler" default="true">
     <lst name="defaults">
       <str name="echoParams">explicit</str>
     </lst>
  </requestHandler>

  <requestHandler name="dismax" class="solr.SearchHandler" >
    <lst name="defaults">
     <str name="defType">dismax</str>
     <str name="echoParams">explicit</str>
     <float name="tie">0.01</float>
     <str name="qf">
        text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
     </str>
     <str name="pf">
        text^0.2 features^1.1 name^1.5 manu^1.4 manu_exact^1.9
     </str>
     <str name="bf">
        ord(popularity)^0.5 recip(rord(price),1,1000,1000)^0.3
     </str>
     <str name="fl">
        id,name,price,score
     </str>
     <str name="mm">
        2<-1 5<-2 6<90%
     </str>
     <int name="ps">100</int>
     <str name="q.alt">*:*</str>   
     <str name="hl.fl">text features name</str>
     <str name="f.name.hl.fragsize">0</str>
     <str name="f.name.hl.alternateField">name</str>
     <str name="f.text.hl.fragmenter">regex</str> <!-- defined below -->
    </lst>
  </requestHandler>

  <requestHandler name="partitioned" class="solr.SearchHandler" >
    <lst name="defaults">
     <str name="defType">dismax</str>
     <str name="echoParams">explicit</str>
     <str name="qf">text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0</str>
     <str name="mm">2<-1 5<-2 6<90%</str>
     <str name="bq">incubationdate_dt:[* TO NOW/DAY-1MONTH]^2.2</str>
    </lst>
    <lst name="appends">
      <str name="fq">inStock:true</str>
    </lst>
    <lst name="invariants">
      <str name="facet.field">cat</str>
      <str name="facet.field">manu_exact</str>
      <str name="facet.query">price:[* TO 500]</str>
      <str name="facet.query">price:[500 TO *]</str>
    </lst>
  </requestHandler>
  
  <searchComponent name="spellcheck" class="solr.SpellCheckComponent">

    <str name="queryAnalyzerFieldType">textSpell</str>

    <lst name="spellchecker">
      <str name="name">default</str>
      <str name="field">spell</str>
      <str name="spellcheckIndexDir">./spellchecker1</str>

    </lst>
    <lst name="spellchecker">
      <str name="name">jarowinkler</str>
      <str name="field">spell</str>
      <str name="distanceMeasure">org.apache.lucene.search.spell.JaroWinklerDistance</str>
      <str name="spellcheckIndexDir">./spellchecker2</str>

    </lst>

    <lst name="spellchecker">
      <str name="classname">solr.FileBasedSpellChecker</str>
      <str name="name">file</str>
      <str name="sourceLocation">spellings.txt</str>
      <str name="characterEncoding">UTF-8</str>
      <str name="spellcheckIndexDir">./spellcheckerFile</str>
    </lst>
  </searchComponent>

  <requestHandler name="/spellCheckCompRH" class="solr.SearchHandler">
    <lst name="defaults">
      <str name="spellcheck.onlyMorePopular">false</str>
      <str name="spellcheck.extendedResults">false</str>
      <str name="spellcheck.count">1</str>
    </lst>
    <arr name="last-components">
      <str>spellcheck</str>
    </arr>
  </requestHandler>
  
  <searchComponent name="elevator" class="solr.QueryElevationComponent" >
    <str name="queryFieldType">string</str>
    <str name="config-file">elevate.xml</str>
  </searchComponent>
 
  <requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy">
    <lst name="defaults">
      <str name="echoParams">explicit</str>
    </lst>
    <arr name="last-components">
      <str>elevator</str>
    </arr>
  </requestHandler>
  
  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" />

  <requestHandler name="/analysis" class="solr.AnalysisRequestHandler" />
  
  <requestHandler name="/update/csv" class="solr.CSVRequestHandler" startup="lazy" />

  <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />
  
  <requestHandler name="/admin/ping" class="PingRequestHandler">
    <lst name="defaults">
      <str name="qt">standard</str>
      <str name="q">solrpingquery</str>
      <str name="echoParams">all</str>
    </lst>
  </requestHandler>
    
  <requestHandler name="/debug/dump" class="solr.DumpRequestHandler" >
    <lst name="defaults">
     <str name="echoParams">explicit</str> <!-- for all params (including the default etc) use: 'all' -->
     <str name="echoHandler">true</str>
    </lst>
  </requestHandler>
  
  <highlighting>
   <fragmenter name="gap" class="org.apache.solr.highlight.GapFragmenter" default="true">
    <lst name="defaults">
     <int name="hl.fragsize">100</int>
    </lst>
   </fragmenter>

   <fragmenter name="regex" class="org.apache.solr.highlight.RegexFragmenter">
    <lst name="defaults">
      <int name="hl.fragsize">70</int>
      <float name="hl.regex.slop">0.5</float> 
      <str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str>
    </lst>
   </fragmenter>
   
   <formatter name="html" class="org.apache.solr.highlight.HtmlFormatter" default="true">
    <lst name="defaults">
     <str name="hl.simple.pre"><![CDATA[<em>]]></str>
     <str name="hl.simple.post"><![CDATA[</em>]]></str>
    </lst>
   </formatter>
  </highlighting>
  
  <queryResponseWriter name="xslt" class="org.apache.solr.request.XSLTResponseWriter">
    <int name="xsltCacheLifetimeSeconds">5</int>
  </queryResponseWriter> 

  <admin>
    <defaultQuery>solr</defaultQuery>
  </admin>

</config>