You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/12/19 18:48:57 UTC
svn commit: r1552377 [11/15] - in /lucene/dev/branches/lucene5339: ./
dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/benchmark/src/ dev-tools/idea/lucene/demo/
dev-tools/idea/lucene/facet/ dev-tools/idea/solr/con...
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java Thu Dec 19 17:48:47 2013
@@ -198,8 +198,8 @@ public abstract class ValueSourceParser
ValueSource source = fp.parseValueSource();
float min = fp.parseFloat();
float max = fp.parseFloat();
- float target = fp.parseFloat();
- Float def = fp.hasMoreArguments() ? fp.parseFloat() : null;
+ ValueSource target = fp.parseValueSource();
+ ValueSource def = fp.hasMoreArguments() ? fp.parseValueSource() : null;
return new RangeMapFloatFunction(source, min, max, target, def);
}
});
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java Thu Dec 19 17:48:47 2013
@@ -92,7 +92,9 @@ public class SearchGroupShardResponsePro
if (srsp.getSolrResponse() != null) {
nl.add("time", srsp.getSolrResponse().getElapsedTime());
}
-
+ if (srsp.getShardAddress() != null) {
+ nl.add("shardAddress", srsp.getShardAddress());
+ }
shardInfo.add(srsp.getShard(), nl);
}
if (rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false) && srsp.getException() != null) {
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java Thu Dec 19 17:48:47 2013
@@ -108,7 +108,9 @@ public class TopGroupsShardResponseProce
if (srsp.getSolrResponse() != null) {
individualShardInfo.add("time", srsp.getSolrResponse().getElapsedTime());
}
-
+ if (srsp.getShardAddress() != null) {
+ individualShardInfo.add("shardAddress", srsp.getShardAddress());
+ }
shardInfo.add(srsp.getShard(), individualShardInfo);
}
if (rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false) && srsp.getException() != null) {
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java Thu Dec 19 17:48:47 2013
@@ -52,6 +52,7 @@ import org.apache.solr.response.QueryRes
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
import org.apache.solr.servlet.cache.Method;
+import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.util.FastWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,6 +65,7 @@ import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -92,6 +94,9 @@ import java.util.WeakHashMap;
*/
public class SolrDispatchFilter implements Filter
{
+ private static final String CONNECTION_HEADER = "Connection";
+ private static final String TRANSFER_ENCODING_HEADER = "Transfer-Encoding";
+
final Logger log;
protected volatile CoreContainer cores;
@@ -309,8 +314,12 @@ public class SolrDispatchFilter implemen
// if we couldn't find it locally, look on other nodes
if (core == null && idx > 0) {
String coreUrl = getRemotCoreUrl(cores, corename, origCorename);
- if (coreUrl != null) {
- path = path.substring( idx );
+ // don't proxy for internal update requests
+ SolrParams queryParams = SolrRequestParsers.parseQueryString(req.getQueryString());
+ if (coreUrl != null
+ && queryParams
+ .get(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM) == null) {
+ path = path.substring(idx);
remoteQuery(coreUrl + path, req, solrReq, resp);
return;
} else {
@@ -493,10 +502,14 @@ public class SolrDispatchFilter implemen
con.setRequestMethod(req.getMethod());
con.setUseCaches(false);
- con.setDoOutput(true);
+ boolean isPostOrPutRequest = "POST".equals(req.getMethod()) || "PUT".equals(req.getMethod());
+
+ if (isPostOrPutRequest) {
+ con.setDoOutput(true);
+ }
con.setDoInput(true);
- for (Enumeration e = req.getHeaderNames(); e.hasMoreElements();) {
- String headerName = e.nextElement().toString();
+ for (Enumeration<String> e = req.getHeaderNames(); e.hasMoreElements();) {
+ String headerName = e.nextElement();
con.setRequestProperty(headerName, req.getHeader(headerName));
}
try {
@@ -504,7 +517,7 @@ public class SolrDispatchFilter implemen
InputStream is;
OutputStream os;
- if ("POST".equals(req.getMethod())) {
+ if (isPostOrPutRequest) {
is = req.getInputStream();
os = con.getOutputStream(); // side effect: method is switched to POST
try {
@@ -518,11 +531,18 @@ public class SolrDispatchFilter implemen
resp.setStatus(con.getResponseCode());
- for (Iterator i = con.getHeaderFields().entrySet().iterator(); i
- .hasNext();) {
- Map.Entry mapEntry = (Map.Entry) i.next();
- if (mapEntry.getKey() != null) resp.setHeader(mapEntry.getKey()
- .toString(), ((List) mapEntry.getValue()).get(0).toString());
+ for (Iterator<Entry<String,List<String>>> i = con.getHeaderFields().entrySet().iterator(); i.hasNext();) {
+ Map.Entry<String,List<String>> mapEntry = i.next();
+ String header = mapEntry.getKey();
+
+ // We pull out these two headers below because they can cause chunked
+ // encoding issues with Tomcat and certain clients
+ if (header != null && !header.equals(TRANSFER_ENCODING_HEADER)
+ && !header.equals(CONNECTION_HEADER)) {
+ for (String value : mapEntry.getValue()) {
+ resp.addHeader(mapEntry.getKey(), value);
+ }
+ }
}
resp.setCharacterEncoding(con.getContentEncoding());
@@ -552,13 +572,14 @@ public class SolrDispatchFilter implemen
ClusterState clusterState = cores.getZkController().getClusterState();
Collection<Slice> slices = clusterState.getActiveSlices(collectionName);
boolean byCoreName = false;
+
if (slices == null) {
+ slices = new ArrayList<Slice>();
// look by core name
byCoreName = true;
- Set<String> collections = clusterState.getCollections();
- for (String collection : collections) {
- slices = new ArrayList<Slice>();
- slices.addAll(clusterState.getActiveSlices(collection));
+ slices = getSlicesForCollections(clusterState, slices, true);
+ if (slices == null || slices.size() == 0) {
+ slices = getSlicesForCollections(clusterState, slices, false);
}
}
@@ -566,6 +587,21 @@ public class SolrDispatchFilter implemen
return null;
}
+ String coreUrl = getCoreUrl(cores, collectionName, origCorename, clusterState,
+ slices, byCoreName, true);
+
+ if (coreUrl == null) {
+ coreUrl = getCoreUrl(cores, collectionName, origCorename, clusterState,
+ slices, byCoreName, false);
+ }
+
+ return coreUrl;
+ }
+
+ private String getCoreUrl(CoreContainer cores, String collectionName,
+ String origCorename, ClusterState clusterState, Collection<Slice> slices,
+ boolean byCoreName, boolean activeReplicas) {
+ String coreUrl;
Set<String> liveNodes = clusterState.getLiveNodes();
Iterator<Slice> it = slices.iterator();
while (it.hasNext()) {
@@ -573,8 +609,9 @@ public class SolrDispatchFilter implemen
Map<String,Replica> sliceShards = slice.getReplicasMap();
for (ZkNodeProps nodeProps : sliceShards.values()) {
ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
- if (liveNodes.contains(coreNodeProps.getNodeName())
- && coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) {
+ if (!activeReplicas || (liveNodes.contains(coreNodeProps.getNodeName())
+ && coreNodeProps.getState().equals(ZkStateReader.ACTIVE))) {
+
if (byCoreName && !collectionName.equals(coreNodeProps.getCoreName())) {
// if it's by core name, make sure they match
continue;
@@ -583,7 +620,7 @@ public class SolrDispatchFilter implemen
// don't count a local core
continue;
}
- String coreUrl;
+
if (origCorename != null) {
coreUrl = coreNodeProps.getBaseUrl() + "/" + origCorename;
} else {
@@ -599,6 +636,19 @@ public class SolrDispatchFilter implemen
}
return null;
}
+
+ private Collection<Slice> getSlicesForCollections(ClusterState clusterState,
+ Collection<Slice> slices, boolean activeSlices) {
+ Set<String> collections = clusterState.getCollections();
+ for (String collection : collections) {
+ if (activeSlices) {
+ slices.addAll(clusterState.getActiveSlices(collection));
+ } else {
+ slices.addAll(clusterState.getSlices(collection));
+ }
+ }
+ return slices;
+ }
private SolrCore getCoreByCollection(CoreContainer cores, String corename, String path) {
String collection = corename;
@@ -708,6 +758,7 @@ public class SolrDispatchFilter implemen
ServletRequest request,
HttpServletResponse response,
Throwable ex) throws IOException {
+ SolrCore localCore = null;
try {
SolrQueryResponse solrResp = new SolrQueryResponse();
if(ex instanceof Exception) {
@@ -717,7 +768,9 @@ public class SolrDispatchFilter implemen
solrResp.setException(new RuntimeException(ex));
}
if(core==null) {
- core = cores.getCore(""); // default core
+ localCore = cores.getCore(""); // default core
+ } else {
+ localCore = core;
}
if(req==null) {
final SolrParams solrParams;
@@ -737,6 +790,10 @@ public class SolrDispatchFilter implemen
SimpleOrderedMap info = new SimpleOrderedMap();
int code = ResponseUtils.getErrorInfo(ex, info, log);
response.sendError( code, info.toString() );
+ } finally {
+ if (core == null && localCore != null) {
+ localCore.close();
+ }
}
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java Thu Dec 19 17:48:47 2013
@@ -584,7 +584,7 @@ public class SolrRequestParsers
if (!isFormData(req)) {
throw new SolrException( ErrorCode.BAD_REQUEST, "Not application/x-www-form-urlencoded content: "+req.getContentType() );
}
-
+
final Map<String,String[]> map = new HashMap<String, String[]>();
// also add possible URL parameters and include into the map (parsed using UTF-8):
@@ -600,7 +600,7 @@ public class SolrRequestParsers
throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content length (" +
totalLength + " bytes) exceeds upload limit of " + uploadLimitKB + " KB");
}
-
+
// get query String from request body, using the charset given in content-type:
final String cs = ContentStreamBase.getCharsetFromContentType(req.getContentType());
final Charset charset = (cs == null) ? IOUtils.CHARSET_UTF_8 : Charset.forName(cs);
@@ -680,7 +680,10 @@ public class SolrRequestParsers
if (ServletFileUpload.isMultipartContent(req)) {
return multipart.parseParamsAndFillStreams(req, streams);
}
- return raw.parseParamsAndFillStreams(req, streams);
+ if (req.getContentType() != null) {
+ return raw.parseParamsAndFillStreams(req, streams);
+ }
+ throw new SolrException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Must specify a Content-Type header with POST requests");
}
throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported method: " + method + " for request " + req);
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java Thu Dec 19 17:48:47 2013
@@ -20,11 +20,25 @@ package org.apache.solr.spelling.suggest
import org.apache.lucene.search.suggest.Lookup;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.spelling.suggest.jaspell.JaspellLookupFactory;
/**
* Suggester factory for creating {@link Lookup} instances.
*/
public abstract class LookupFactory {
+
+ /** Default lookup implementation to use for SolrSuggester */
+ public static String DEFAULT_FILE_BASED_DICT = JaspellLookupFactory.class.getName();
+
+ /**
+ * Create a Lookup using config options in <code>params</code> and
+ * current <code>core</code>
+ */
public abstract Lookup create(NamedList params, SolrCore core);
+
+ /**
+ * <p>Returns the filename in which the in-memory data structure is stored </p>
+ * <b>NOTE:</b> not all {@link Lookup} implementations store in-memory data structures
+ * */
public abstract String storeFileName();
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java Thu Dec 19 17:48:47 2013
@@ -69,6 +69,9 @@ public class AnalyzingInfixLookupFactory
throw new IllegalArgumentException("Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory");
}
FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString());
+ if (ft == null) {
+ throw new IllegalArgumentException("Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema");
+ }
Analyzer indexAnalyzer = ft.getAnalyzer();
Analyzer queryAnalyzer = ft.getQueryAnalyzer();
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingLookupFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingLookupFactory.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingLookupFactory.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingLookupFactory.java Thu Dec 19 17:48:47 2013
@@ -83,6 +83,10 @@ public class AnalyzingLookupFactory exte
throw new IllegalArgumentException("Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory");
}
FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString());
+ if (ft == null) {
+ throw new IllegalArgumentException("Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema");
+ }
+
Analyzer indexAnalyzer = ft.getAnalyzer();
Analyzer queryAnalyzer = ft.getQueryAnalyzer();
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java Thu Dec 19 17:48:47 2013
@@ -78,6 +78,9 @@ public class FuzzyLookupFactory extends
}
// retrieve index and query analyzers for the field
FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString());
+ if (ft == null) {
+ throw new IllegalArgumentException("Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema");
+ }
Analyzer indexAnalyzer = ft.getAnalyzer();
Analyzer queryAnalyzer = ft.getQueryAnalyzer();
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java Thu Dec 19 17:48:47 2013
@@ -54,7 +54,6 @@ public class HdfsDirectory extends BaseD
public HdfsDirectory(Path hdfsDirPath, Configuration configuration)
throws IOException {
- assert hdfsDirPath.toString().startsWith("hdfs:/") : hdfsDirPath.toString();
setLockFactory(NoLockFactory.getNoLockFactory());
this.hdfsDirPath = hdfsDirPath;
this.configuration = configuration;
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java Thu Dec 19 17:48:47 2013
@@ -21,14 +21,18 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.LockReleaseFailedException;
import org.apache.solr.util.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HdfsLockFactory extends LockFactory {
+ public static Logger log = LoggerFactory.getLogger(HdfsLockFactory.class);
private Path lockPath;
private Configuration configuration;
@@ -98,9 +102,14 @@ public class HdfsLockFactory extends Loc
FileSystem fs = null;
try {
fs = FileSystem.newInstance(lockPath.toUri(), conf);
-
+ if (!fs.exists(lockPath)) {
+ fs.mkdirs(lockPath);
+ }
file = fs.create(new Path(lockPath, lockName), false);
- } catch (IOException e) {
+ } catch (FileAlreadyExistsException e) {
+ return false;
+ }catch (IOException e) {
+ log.error("Error creating lock file", e);
return false;
} finally {
IOUtils.closeQuietly(file);
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java Thu Dec 19 17:48:47 2013
@@ -109,14 +109,16 @@ public class AddUpdateCommand extends Up
}
public String getPrintableId() {
- IndexSchema schema = req.getSchema();
- SchemaField sf = schema.getUniqueKeyField();
- if (solrDoc != null && sf != null) {
- SolrInputField field = solrDoc.getField(sf.getName());
- if (field != null) {
- return field.getFirstValue().toString();
- }
- }
+ if (req != null) {
+ IndexSchema schema = req.getSchema();
+ SchemaField sf = schema.getUniqueKeyField();
+ if (solrDoc != null && sf != null) {
+ SolrInputField field = solrDoc.getField(sf.getName());
+ if (field != null) {
+ return field.getFirstValue().toString();
+ }
+ }
+ }
return "(null)";
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Thu Dec 19 17:48:47 2013
@@ -842,8 +842,10 @@ public class DirectUpdateHandler2 extend
lst.add("cumulative_deletesById", deleteByIdCommandsCumulative.get());
lst.add("cumulative_deletesByQuery", deleteByQueryCommandsCumulative.get());
lst.add("cumulative_errors", numErrorsCumulative.get());
- lst.add("transaction_logs_total_size", ulog.getTotalLogsSize());
- lst.add("transaction_logs_total_number", ulog.getTotalLogsNumber());
+ if (this.ulog != null) {
+ lst.add("transaction_logs_total_size", ulog.getTotalLogsSize());
+ lst.add("transaction_logs_total_number", ulog.getTotalLogsNumber());
+ }
return lst;
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/PeerSync.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/PeerSync.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/PeerSync.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/PeerSync.java Thu Dec 19 17:48:47 2013
@@ -17,6 +17,9 @@
package org.apache.solr.update;
+import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
+import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+
import java.io.IOException;
import java.net.ConnectException;
import java.net.SocketException;
@@ -37,7 +40,6 @@ import org.apache.solr.cloud.ZkControlle
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
@@ -48,16 +50,11 @@ import org.apache.solr.handler.component
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.update.processor.DistributedUpdateProcessorFactory;
-import org.apache.solr.update.processor.RunUpdateProcessorFactory;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
-
/** @lucene.experimental */
public class PeerSync {
public static Logger log = LoggerFactory.getLogger(PeerSync.class);
@@ -82,16 +79,7 @@ public class PeerSync {
private long ourHighThreshold; // 80th percentile
private boolean cantReachIsSuccess;
private boolean getNoVersionsIsSuccess;
- private static final HttpClient client;
- static {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 20);
- params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000);
- params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 30000);
- params.set(HttpClientUtil.PROP_SO_TIMEOUT, 30000);
- params.set(HttpClientUtil.PROP_USE_RETRY, false);
- client = HttpClientUtil.createClient(params);
- }
+ private final HttpClient client;
// comparator that sorts by absolute value, putting highest first
private static Comparator<Long> absComparator = new Comparator<Long>() {
@@ -141,7 +129,7 @@ public class PeerSync {
this.maxUpdates = nUpdates;
this.cantReachIsSuccess = cantReachIsSuccess;
this.getNoVersionsIsSuccess = getNoVersionsIsSuccess;
-
+ this.client = core.getCoreDescriptor().getCoreContainer().getUpdateShardHandler().getHttpClient();
uhandler = core.getUpdateHandler();
ulog = uhandler.getUpdateLog();
@@ -317,7 +305,8 @@ public class PeerSync {
}
if (cantReachIsSuccess && sreq.purpose == 1 && srsp.getException() instanceof SolrException && ((SolrException) srsp.getException()).code() == 404) {
- log.warn(msg() + " got a 404 from " + srsp.getShardAddress() + ", counting as success");
+ log.warn(msg() + " got a 404 from " + srsp.getShardAddress() + ", counting as success. " +
+ "Perhaps /get is not registered?");
return true;
}
// TODO: at least log???
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java Thu Dec 19 17:48:47 2013
@@ -18,13 +18,14 @@ package org.apache.solr.update;
*/
import java.io.IOException;
+import java.net.ConnectException;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.ExecutorService;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
@@ -39,11 +40,14 @@ import org.slf4j.LoggerFactory;
public class SolrCmdDistributor {
- private static final int MAX_RETRIES_ON_FORWARD = 15;
+ private static final int MAX_RETRIES_ON_FORWARD = 25;
public static Logger log = LoggerFactory.getLogger(SolrCmdDistributor.class);
private StreamingSolrServers servers;
+ private int retryPause = 500;
+ private int maxRetriesOnForward = MAX_RETRIES_ON_FORWARD;
+
private List<Error> allErrors = new ArrayList<Error>();
private List<Error> errors = new ArrayList<Error>();
@@ -51,8 +55,14 @@ public class SolrCmdDistributor {
public boolean abortCheck();
}
- public SolrCmdDistributor(ExecutorService updateExecutor) {
- servers = new StreamingSolrServers(updateExecutor);
+ public SolrCmdDistributor(UpdateShardHandler updateShardHandler) {
+ servers = new StreamingSolrServers(updateShardHandler);
+ }
+
+ public SolrCmdDistributor(StreamingSolrServers servers, int maxRetriesOnForward, int retryPause) {
+ this.servers = servers;
+ this.maxRetriesOnForward = maxRetriesOnForward;
+ this.retryPause = retryPause;
}
public void finish() {
@@ -69,59 +79,72 @@ public class SolrCmdDistributor {
List<Error> errors = new ArrayList<Error>(this.errors);
errors.addAll(servers.getErrors());
+ List<Error> resubmitList = new ArrayList<Error>();
- boolean blockUntilFinishedAgain = false;
for (Error err : errors) {
- String oldNodeUrl = err.req.node.getUrl();
-
- // if there is a retry url, we want to retry...
- boolean isRetry = err.req.node.checkRetry();
- boolean doRetry = false;
- int rspCode = err.statusCode;
-
- if (testing_errorHook != null) Diagnostics.call(testing_errorHook, err.e);
-
- // this can happen in certain situations such as shutdown
- if (isRetry) {
- if (rspCode == 404 || rspCode == 403 || rspCode == 503
- || rspCode == 500) {
- doRetry = true;
- }
+ try {
+ String oldNodeUrl = err.req.node.getUrl();
+
+ // if there is a retry url, we want to retry...
+ boolean isRetry = err.req.node.checkRetry();
+
+ boolean doRetry = false;
+ int rspCode = err.statusCode;
- // if its an ioexception, lets try again
- if (err.e instanceof IOException) {
- doRetry = true;
- } else if (err.e instanceof SolrServerException) {
- if (((SolrServerException) err.e).getRootCause() instanceof IOException) {
+ if (testing_errorHook != null) Diagnostics.call(testing_errorHook,
+ err.e);
+
+ // this can happen in certain situations such as shutdown
+ if (isRetry) {
+ if (rspCode == 404 || rspCode == 403 || rspCode == 503) {
doRetry = true;
}
- }
- if (err.req.retries < MAX_RETRIES_ON_FORWARD && doRetry) {
- err.req.retries++;
- SolrException.log(SolrCmdDistributor.log, "forwarding update to "
- + oldNodeUrl + " failed - retrying ... retries: " + err.req.retries);
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- log.warn(null, e);
+ // if its a connect exception, lets try again
+ if (err.e instanceof SolrServerException) {
+ if (((SolrServerException) err.e).getRootCause() instanceof ConnectException) {
+ doRetry = true;
+ }
}
- submit(err.req);
- blockUntilFinishedAgain = true;
+ if (err.e instanceof ConnectException) {
+ doRetry = true;
+ }
+
+ if (err.req.retries < maxRetriesOnForward && doRetry) {
+ err.req.retries++;
+
+ SolrException.log(SolrCmdDistributor.log, "forwarding update to "
+ + oldNodeUrl + " failed - retrying ... retries: "
+ + err.req.retries + " " + err.req.cmdString + " params:"
+ + err.req.uReq.getParams() + " rsp:" + rspCode, err.e);
+ try {
+ Thread.sleep(retryPause);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ log.warn(null, e);
+ }
+
+ resubmitList.add(err);
+ } else {
+ allErrors.add(err);
+ }
} else {
allErrors.add(err);
}
- } else {
- allErrors.add(err);
+ } catch (Exception e) {
+ // continue on
+ log.error("Unexpected Error while doing request retries", e);
}
}
servers.clearErrors();
this.errors.clear();
+ for (Error err : resubmitList) {
+ submit(err.req);
+ }
- if (blockUntilFinishedAgain) {
+ if (resubmitList.size() > 0) {
servers.blockUntilFinished();
doRetriesIfNeeded();
}
@@ -142,7 +165,7 @@ public class SolrCmdDistributor {
uReq.deleteByQuery(cmd.query);
}
- submit(new Req(node, uReq, sync));
+ submit(new Req(cmd.toString(), node, uReq, sync));
}
}
@@ -156,7 +179,7 @@ public class SolrCmdDistributor {
UpdateRequest uReq = new UpdateRequest();
uReq.setParams(params);
uReq.add(cmd.solrDoc, cmd.commitWithin, cmd.overwrite);
- submit(new Req(node, uReq, synchronous));
+ submit(new Req(cmd.toString(), node, uReq, synchronous));
}
}
@@ -173,10 +196,10 @@ public class SolrCmdDistributor {
addCommit(uReq, cmd);
- log.debug("Distrib commit to:" + nodes + " params:" + params);
+ log.debug("Distrib commit to: {} params: {}", nodes, params);
for (Node node : nodes) {
- submit(new Req(node, uReq, false));
+ submit(new Req(cmd.toString(), node, uReq, false));
}
}
@@ -204,7 +227,11 @@ public class SolrCmdDistributor {
return;
}
-
+ if (log.isDebugEnabled()) {
+ log.debug("sending update to "
+ + req.node.getUrl() + " retry:"
+ + req.retries + " " + req.cmdString + " params:" + req.uReq.getParams());
+ }
try {
SolrServer solrServer = servers.getSolrServer(req);
NamedList<Object> rsp = solrServer.request(req.uReq);
@@ -225,11 +252,13 @@ public class SolrCmdDistributor {
public UpdateRequest uReq;
public int retries;
public boolean synchronous;
+ public String cmdString;
- public Req(Node node, UpdateRequest uReq, boolean synchronous) {
+ public Req(String cmdString, Node node, UpdateRequest uReq, boolean synchronous) {
this.node = node;
this.uReq = uReq;
this.synchronous = synchronous;
+ this.cmdString = cmdString;
}
}
@@ -243,7 +272,7 @@ public class SolrCmdDistributor {
public static class Error {
public Exception e;
- public int statusCode;
+ public int statusCode = -1;
public Req req;
}
@@ -351,10 +380,14 @@ public class SolrCmdDistributor {
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
+ } catch (Exception e) {
+ // we retry with same info
+ log.warn(null, e);
+ return true;
}
-
+
this.nodeProps = leaderProps;
-
+
return true;
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Thu Dec 19 17:48:47 2013
@@ -215,7 +215,7 @@ public class SolrIndexSplitter {
}
public static String getRouteKey(String idString) {
- int idx = idString.indexOf(CompositeIdRouter.separator);
+ int idx = idString.indexOf(CompositeIdRouter.SEPARATOR);
if (idx <= 0) return null;
String part1 = idString.substring(0, idx);
int commaIdx = part1.indexOf(CompositeIdRouter.bitsSeparator);
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/StreamingSolrServers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/StreamingSolrServers.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/StreamingSolrServers.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/StreamingSolrServers.java Thu Dec 19 17:48:47 2013
@@ -20,8 +20,10 @@ package org.apache.solr.update;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ExecutorService;
import org.apache.http.client.HttpClient;
@@ -33,28 +35,30 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.update.SolrCmdDistributor.Error;
+import org.apache.solr.update.processor.DistributedUpdateProcessor;
+import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class StreamingSolrServers {
public static Logger log = LoggerFactory.getLogger(StreamingSolrServers.class);
- private static HttpClient httpClient;
- static {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 128);
- params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 32);
- params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, false);
- httpClient = HttpClientUtil.createClient(params);
- }
+ private HttpClient httpClient;
private Map<String,ConcurrentUpdateSolrServer> solrServers = new HashMap<String,ConcurrentUpdateSolrServer>();
private List<Error> errors = Collections.synchronizedList(new ArrayList<Error>());
private ExecutorService updateExecutor;
- public StreamingSolrServers(ExecutorService updateExecutor) {
- this.updateExecutor = updateExecutor;
+ public StreamingSolrServers(UpdateShardHandler updateShardHandler) {
+ this.updateExecutor = updateShardHandler.getUpdateExecutor();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, false);
+ params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 30000);
+ params.set(HttpClientUtil.PROP_USE_RETRY, false);
+
+ httpClient = updateShardHandler.getHttpClient();
}
public List<Error> getErrors() {
@@ -69,7 +73,7 @@ public class StreamingSolrServers {
String url = getFullUrl(req.node.getUrl());
ConcurrentUpdateSolrServer server = solrServers.get(url);
if (server == null) {
- server = new ConcurrentUpdateSolrServer(url, httpClient, 100, 1, updateExecutor) {
+ server = new ConcurrentUpdateSolrServer(url, httpClient, 100, 1, updateExecutor, true) {
@Override
public void handleError(Throwable ex) {
log.error("error", ex);
@@ -85,6 +89,10 @@ public class StreamingSolrServers {
server.setParser(new BinaryResponseParser());
server.setRequestWriter(new BinaryRequestWriter());
server.setPollQueueTime(0);
+ Set<String> queryParams = new HashSet<String>(2);
+ queryParams.add(DistributedUpdateProcessor.DISTRIB_FROM);
+ queryParams.add(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM);
+ server.setQueryParams(queryParams);
solrServers.put(url, server);
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/TransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/TransactionLog.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/TransactionLog.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/TransactionLog.java Thu Dec 19 17:48:47 2013
@@ -564,7 +564,10 @@ public class TransactionLog {
}
public long getLogSize() {
- return tlogFile.length();
+ if (tlogFile != null) {
+ return tlogFile.length();
+ }
+ return 0;
}
/** Returns a reader that can be used while a log is still in use.
@@ -579,7 +582,6 @@ public class TransactionLog {
return new FSReverseReader();
}
-
public class LogReader {
private ChannelFastInputStream fis;
private LogCodec codec = new LogCodec(resolver);
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java Thu Dec 19 17:48:47 2013
@@ -17,16 +17,18 @@ package org.apache.solr.update;
* limitations under the License.
*/
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import org.apache.http.client.HttpClient;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
-import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.apache.solr.core.ConfigSolr;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,36 +36,48 @@ public class UpdateShardHandler {
private static Logger log = LoggerFactory.getLogger(UpdateShardHandler.class);
- private ThreadPoolExecutor cmdDistribExecutor = new ThreadPoolExecutor(0,
- Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
- new DefaultSolrThreadFactory("cmdDistribExecutor"));
+ private ExecutorService updateExecutor = Executors.newCachedThreadPool(
+ new SolrjNamedThreadFactory("updateExecutor"));
+
+ private PoolingClientConnectionManager clientConnectionManager;
private final HttpClient client;
- public UpdateShardHandler(int distribUpdateConnTimeout, int distribUpdateSoTimeout) {
+ public UpdateShardHandler(ConfigSolr cfg) {
+
+ clientConnectionManager = new PoolingClientConnectionManager();
+ clientConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnections());
+ clientConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
+
+
ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 500);
- params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 16);
- params.set(HttpClientUtil.PROP_SO_TIMEOUT, distribUpdateSoTimeout);
- params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, distribUpdateConnTimeout);
- client = HttpClientUtil.createClient(params);
+ params.set(HttpClientUtil.PROP_SO_TIMEOUT, cfg.getDistributedSocketTimeout());
+ params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, cfg.getDistributedConnectionTimeout());
+ params.set(HttpClientUtil.PROP_USE_RETRY, false);
+ client = HttpClientUtil.createClient(params, clientConnectionManager);
}
public HttpClient getHttpClient() {
return client;
}
+
+ public ClientConnectionManager getConnectionManager() {
+ return clientConnectionManager;
+ }
- public ThreadPoolExecutor getCmdDistribExecutor() {
- return cmdDistribExecutor;
+ public ExecutorService getUpdateExecutor() {
+ return updateExecutor;
}
public void close() {
try {
- ExecutorUtil.shutdownNowAndAwaitTermination(cmdDistribExecutor);
+ ExecutorUtil.shutdownAndAwaitTermination(updateExecutor);
} catch (Throwable e) {
SolrException.log(log, e);
+ } finally {
+ clientConnectionManager.shutdown();
}
- client.getConnectionManager().shutdown();
}
+
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Thu Dec 19 17:48:47 2013
@@ -17,10 +17,24 @@ package org.apache.solr.update.processor
* limitations under the License.
*/
-import org.apache.http.client.HttpClient;
+import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestRecovery;
import org.apache.solr.cloud.CloudDescriptor;
@@ -51,10 +65,7 @@ import org.apache.solr.common.params.Upd
import org.apache.solr.common.util.Hash;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.apache.solr.handler.component.RealTimeGetComponent;
-import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
@@ -78,25 +89,13 @@ import org.apache.zookeeper.KeeperExcept
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-
// NOT mt-safe... create a new processor for each add thread
// TODO: we really should not wait for distrib after local? unless a certain replication factor is asked for
public class DistributedUpdateProcessor extends UpdateRequestProcessor {
+ public static final String DISTRIB_FROM_SHARD = "distrib.from.shard";
+ public static final String DISTRIB_FROM_COLLECTION = "distrib.from.collection";
+ public static final String DISTRIB_FROM_PARENT = "distrib.from.parent";
+ public static final String DISTRIB_FROM = "distrib.from";
private static final String TEST_DISTRIB_SKIP_SERVERS = "test.distrib.skip.servers";
public final static Logger log = LoggerFactory.getLogger(DistributedUpdateProcessor.class);
@@ -122,17 +121,6 @@ public class DistributedUpdateProcessor
}
}
}
-
- private final HttpClient client;
- {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000);
- params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 20);
- params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 15000);
- params.set(HttpClientUtil.PROP_SO_TIMEOUT, 60000);
- params.set(HttpClientUtil.PROP_USE_RETRY, false);
- client = HttpClientUtil.createClient(params);
- }
public static final String COMMIT_END_POINT = "commit_end_point";
public static final String LOG_REPLAY = "log_replay";
@@ -197,7 +185,7 @@ public class DistributedUpdateProcessor
this.zkEnabled = coreDesc.getCoreContainer().isZooKeeperAware();
zkController = req.getCore().getCoreDescriptor().getCoreContainer().getZkController();
if (zkEnabled) {
- cmdDistrib = new SolrCmdDistributor(coreDesc.getCoreContainer().getUpdateExecutor());
+ cmdDistrib = new SolrCmdDistributor(coreDesc.getCoreContainer().getUpdateShardHandler());
}
//this.rsp = reqInfo != null ? reqInfo.getRsp() : null;
@@ -268,7 +256,7 @@ public class DistributedUpdateProcessor
// if request is coming from another collection then we want it to be sent to all replicas
// even if it's phase is FROMLEADER
- String fromCollection = updateCommand.getReq().getParams().get("distrib.from.collection");
+ String fromCollection = updateCommand.getReq().getParams().get(DISTRIB_FROM_COLLECTION);
if (DistribPhase.FROMLEADER == phase && !isSubShardLeader && fromCollection == null) {
// we are coming from the leader, just go local - add no urls
@@ -408,13 +396,12 @@ public class DistributedUpdateProcessor
for (DocRouter.Range range : ranges) {
if (range.includes(hash)) {
if (nodes == null) nodes = new ArrayList<Node>();
- Collection<Slice> activeSlices = cstate.getActiveSlices(rule.getTargetCollectionName());
+ DocCollection targetColl = cstate.getCollection(rule.getTargetCollectionName());
+ Collection<Slice> activeSlices = targetColl.getRouter().getSearchSlicesSingle(id, null, targetColl);
if (activeSlices == null || activeSlices.isEmpty()) {
throw new SolrException(ErrorCode.SERVER_ERROR,
- "No active slices found for target collection: " + rule.getTargetCollectionName());
+ "No active slices serving " + id + " found for target collection: " + rule.getTargetCollectionName());
}
- // it doesn't matter where we forward it so just choose the first one
- // todo this can be optimized
Replica targetLeader = cstate.getLeader(rule.getTargetCollectionName(), activeSlices.iterator().next().getName());
nodes.add(new StdNode(new ZkCoreNodeProps(targetLeader)));
break;
@@ -459,13 +446,13 @@ public class DistributedUpdateProcessor
boolean isReplayOrPeersync = (updateCommand.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
if (isReplayOrPeersync) return;
- String from = req.getParams().get("distrib.from");
+ String from = req.getParams().get(DISTRIB_FROM);
ClusterState clusterState = zkController.getClusterState();
CloudDescriptor cloudDescriptor = req.getCore().getCoreDescriptor().getCloudDescriptor();
Slice mySlice = clusterState.getSlice(collection, cloudDescriptor.getShardId());
boolean localIsLeader = cloudDescriptor.isLeader();
if (DistribPhase.FROMLEADER == phase && localIsLeader && from != null) { // from will be null on log replay
- String fromShard = req.getParams().get("distrib.from.parent");
+ String fromShard = req.getParams().get(DISTRIB_FROM_PARENT);
if (fromShard != null) {
if (Slice.ACTIVE.equals(mySlice.getState())) {
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE,
@@ -480,7 +467,7 @@ public class DistributedUpdateProcessor
"Request says it is coming from parent shard leader but parent hash range is not superset of my range");
}
} else {
- String fromCollection = req.getParams().get("distrib.from.collection"); // is it because of a routing rule?
+ String fromCollection = req.getParams().get(DISTRIB_FROM_COLLECTION); // is it because of a routing rule?
if (fromCollection == null) {
log.error("Request says it is coming from leader, but we are the leader: " + req.getParamString());
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Request says it is coming from leader, but we are the leader");
@@ -559,9 +546,9 @@ public class DistributedUpdateProcessor
if (subShardLeaders != null && !subShardLeaders.isEmpty()) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
- params.set("distrib.from", ZkCoreNodeProps.getCoreUrl(
+ params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
zkController.getBaseUrl(), req.getCore().getName()));
- params.set("distrib.from.parent", req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId());
+ params.set(DISTRIB_FROM_PARENT, req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId());
for (Node subShardLeader : subShardLeaders) {
cmdDistrib.distribAdd(cmd, Collections.singletonList(subShardLeader), params, true);
}
@@ -570,10 +557,10 @@ public class DistributedUpdateProcessor
if (nodesByRoutingRules != null && !nodesByRoutingRules.isEmpty()) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
- params.set("distrib.from", ZkCoreNodeProps.getCoreUrl(
+ params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
zkController.getBaseUrl(), req.getCore().getName()));
- params.set("distrib.from.collection", req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName());
- params.set("distrib.from.shard", req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId());
+ params.set(DISTRIB_FROM_COLLECTION, req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName());
+ params.set(DISTRIB_FROM_SHARD, req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId());
for (Node nodesByRoutingRule : nodesByRoutingRules) {
cmdDistrib.distribAdd(cmd, Collections.singletonList(nodesByRoutingRule), params, true);
}
@@ -588,13 +575,9 @@ public class DistributedUpdateProcessor
(isLeader || isSubShardLeader ?
DistribPhase.FROMLEADER.toString() :
DistribPhase.TOLEADER.toString()));
- if (isLeader || isSubShardLeader) {
- params.set("distrib.from", ZkCoreNodeProps.getCoreUrl(
- zkController.getBaseUrl(), req.getCore().getName()));
- }
-
- params.set("distrib.from", ZkCoreNodeProps.getCoreUrl(
+ params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
zkController.getBaseUrl(), req.getCore().getName()));
+
cmdDistrib.distribAdd(cmd, nodes, params);
}
@@ -681,7 +664,7 @@ public class DistributedUpdateProcessor
}
}
};
- ExecutorService executor = req.getCore().getCoreDescriptor().getCoreContainer().getUpdateExecutor();
+ ExecutorService executor = req.getCore().getCoreDescriptor().getCoreContainer().getUpdateShardHandler().getUpdateExecutor();
executor.execute(thread);
}
@@ -745,7 +728,7 @@ public class DistributedUpdateProcessor
boolean isReplayOrPeersync = (cmd.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
boolean leaderLogic = isLeader && !isReplayOrPeersync;
- boolean forwardedFromCollection = cmd.getReq().getParams().get("distrib.from.collection") != null;
+ boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null;
VersionBucket bucket = vinfo.bucket(bucketHash);
@@ -990,9 +973,9 @@ public class DistributedUpdateProcessor
if (subShardLeaders != null && !subShardLeaders.isEmpty()) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
- params.set("distrib.from", ZkCoreNodeProps.getCoreUrl(
+ params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
zkController.getBaseUrl(), req.getCore().getName()));
- params.set("distrib.from.parent", cloudDesc.getShardId());
+ params.set(DISTRIB_FROM_PARENT, cloudDesc.getShardId());
cmdDistrib.distribDelete(cmd, subShardLeaders, params, true);
}
@@ -1000,10 +983,10 @@ public class DistributedUpdateProcessor
if (nodesByRoutingRules != null && !nodesByRoutingRules.isEmpty()) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
- params.set("distrib.from", ZkCoreNodeProps.getCoreUrl(
+ params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
zkController.getBaseUrl(), req.getCore().getName()));
- params.set("distrib.from.collection", req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName());
- params.set("distrib.from.shard", req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId());
+ params.set(DISTRIB_FROM_COLLECTION, req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName());
+ params.set(DISTRIB_FROM_SHARD, req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId());
for (Node nodesByRoutingRule : nodesByRoutingRules) {
cmdDistrib.distribDelete(cmd, Collections.singletonList(nodesByRoutingRule), params, true);
}
@@ -1016,13 +999,11 @@ public class DistributedUpdateProcessor
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(DISTRIB_UPDATE_PARAM,
- (isLeader || isSubShardLeader ?
- DistribPhase.FROMLEADER.toString() :
- DistribPhase.TOLEADER.toString()));
- if (isLeader || isSubShardLeader) {
- params.set("distrib.from", ZkCoreNodeProps.getCoreUrl(
- zkController.getBaseUrl(), req.getCore().getName()));
- }
+ (isLeader || isSubShardLeader ? DistribPhase.FROMLEADER.toString()
+ : DistribPhase.TOLEADER.toString()));
+ params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
+ zkController.getBaseUrl(), req.getCore().getName()));
+
cmdDistrib.distribDelete(cmd, nodes, params);
}
@@ -1083,6 +1064,8 @@ public class DistributedUpdateProcessor
ModifiableSolrParams outParams = new ModifiableSolrParams(filterParams(req.getParams()));
outParams.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
+ outParams.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
+ zkController.getBaseUrl(), req.getCore().getName()));
SolrParams params = req.getParams();
String route = params.get(ShardParams._ROUTE_);
@@ -1198,7 +1181,7 @@ public class DistributedUpdateProcessor
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(VERSION_FIELD, Long.toString(cmd.getVersion()));
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
- params.set("update.from", ZkCoreNodeProps.getCoreUrl(
+ params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
zkController.getBaseUrl(), req.getCore().getName()));
boolean someReplicas = false;
@@ -1235,10 +1218,10 @@ public class DistributedUpdateProcessor
if (nodesByRoutingRules != null && !nodesByRoutingRules.isEmpty()) {
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
- params.set("distrib.from", ZkCoreNodeProps.getCoreUrl(
+ params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
zkController.getBaseUrl(), req.getCore().getName()));
- params.set("distrib.from.collection", req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName());
- params.set("distrib.from.shard", req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId());
+ params.set(DISTRIB_FROM_COLLECTION, req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName());
+ params.set(DISTRIB_FROM_SHARD, req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId());
cmdDistrib.distribDelete(cmd, nodesByRoutingRules, params, true);
}
if (replicas != null) {
@@ -1333,7 +1316,7 @@ public class DistributedUpdateProcessor
boolean isReplayOrPeersync = (cmd.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
boolean leaderLogic = isLeader && !isReplayOrPeersync;
- boolean forwardedFromCollection = cmd.getReq().getParams().get("distrib.from.collection") != null;
+ boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null;
if (!leaderLogic && versionOnUpdate==0) {
throw new SolrException(ErrorCode.BAD_REQUEST, "missing _version_ on update from leader");
@@ -1438,7 +1421,9 @@ public class DistributedUpdateProcessor
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
if (!req.getParams().getBool(COMMIT_END_POINT, false)) {
params.set(COMMIT_END_POINT, true);
-
+ params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
+ params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
+ zkController.getBaseUrl(), req.getCore().getName()));
if (nodes != null) {
cmdDistrib.distribCommit(cmd, nodes, params);
finish();
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java Thu Dec 19 17:48:47 2013
@@ -112,7 +112,9 @@ public class SolrLogLayout extends Layou
public String _format(LoggingEvent event) {
String message = (String) event.getMessage();
-
+ if (message == null) {
+ message = "";
+ }
StringBuilder sb = new StringBuilder(message.length() + 80);
long now = event.timeStamp;
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java Thu Dec 19 17:48:47 2013
@@ -430,7 +430,7 @@ public class SolrPluginUtils {
// we can use the Lucene sort ability.
Sort sort = null;
if (commands.size() >= 2) {
- sort = QueryParsing.parseSort(commands.get(1), req);
+ sort = QueryParsing.parseSortSpec(commands.get(1), req).getSort();
}
DocList results = req.getSearcher().getDocList(query,(DocSet)null, sort, start, limit);
@@ -825,7 +825,7 @@ public class SolrPluginUtils {
SolrException sortE = null;
Sort ss = null;
try {
- ss = QueryParsing.parseSort(sort, req);
+ ss = QueryParsing.parseSortSpec(sort, req).getSort();
} catch (SolrException e) {
sortE = e;
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/test-files/log4j.properties
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/test-files/log4j.properties?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/test-files/log4j.properties (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/test-files/log4j.properties Thu Dec 19 17:48:47 2013
@@ -8,3 +8,20 @@ log4j.appender.CONSOLE.layout.Conversion
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.apache.hadoop=WARN
+log4j.logger.org.apache.solr.hadoop=INFO
+
+#log4j.logger.org.apache.solr.update.processor.LogUpdateProcessor=DEBUG
+#log4j.logger.org.apache.solr.update.processor.DistributedUpdateProcessor=DEBUG
+#log4j.logger.org.apache.solr.update.PeerSync=DEBUG
+#log4j.logger.org.apache.solr.core.CoreContainer=DEBUG
+#log4j.logger.org.apache.solr.cloud.RecoveryStrategy=DEBUG
+#log4j.logger.org.apache.solr.cloud.SyncStrategy=DEBUG
+#log4j.logger.org.apache.solr.handler.admin.CoreAdminHandler=DEBUG
+#log4j.logger.org.apache.solr.cloud.ZkController=DEBUG
+#log4j.logger.org.apache.solr.update.DefaultSolrCoreState=DEBUG
+#log4j.logger.org.apache.solr.common.cloud.ConnectionManager=DEBUG
+#log4j.logger.org.apache.solr.update.UpdateLog=DEBUG
+#log4j.logger.org.apache.solr.cloud.ChaosMonkey=DEBUG
+#log4j.logger.org.apache.solr.update.TransactionLog=DEBUG
+#log4j.logger.org.apache.solr.handler.ReplicationHandler=DEBUG
+#log4j.logger.org.apache.solr.handler.SnapPuller=DEBUG
\ No newline at end of file
Modified: lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml Thu Dec 19 17:48:47 2013
@@ -53,6 +53,7 @@
<fields>
<field name="id" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
<field name="text" type="text" indexed="true" stored="false"/>
+ <field name="stext" type="text" indexed="true" stored="true"/>
</fields>
<defaultSearchField>text</defaultSearchField>
Modified: lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml Thu Dec 19 17:48:47 2013
@@ -24,6 +24,11 @@
<queryParser name="foo" class="FooQParserPlugin"/>
<!-- override the default "lucene" qparser -->
<queryParser name="lucene" class="org.apache.solr.search.FooQParserPlugin"/>
+ <queryParser name="notfoo" class="solr.LuceneQParserPlugin"/>
<valueSourceParser name="boost" class="org.apache.solr.core.DummyValueSourceParser"/>
+ <valueSourceParser name="countUsage" class="org.apache.solr.core.CountUsageValueSourceParser"/>
+
+ <requestHandler default="true" name="/select" class="solr.SearchHandler" />
+
</config>
Modified: lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml Thu Dec 19 17:48:47 2013
@@ -83,6 +83,42 @@
<str name="queryAnalyzerFieldType">phrase_suggest</str>
</searchComponent>
+
+ <!-- FuzzyLookup suggest component with HighFrequencyDictionary -->
+ <searchComponent class="solr.SuggestComponent" name="fuzzy_suggest_analyzing_with_high_freq_dict">
+ <lst name="suggester">
+ <str name="name">fuzzy_suggest_analyzing_with_high_freq_dict</str>
+ <str name="lookupImpl">org.apache.solr.spelling.suggest.fst.FuzzyLookupFactory</str>
+ <str name="dictionaryImpl">org.apache.solr.spelling.suggest.HighFrequencyDictionaryFactory</str>
+ <str name="storeDir">fuzzy_suggest_analyzing</str>
+ <str name="buildOnCommit">false</str>
+
+ <!-- Suggester properties -->
+ <bool name="exactMatchFirst">true</bool>
+ <str name="suggestAnalyzerFieldType">text</str>
+ <bool name="preserveSep">false</bool>
+ <str name="field">stext</str>
+ </lst>
+
+ </searchComponent>
+
+ <!-- FuzzyLookup suggest component with FileDictionaryFactory -->
+ <searchComponent class="solr.SuggestComponent" name="fuzzy_suggest_analyzing_with_file_dict">
+ <lst name="suggester">
+ <str name="name">fuzzy_suggest_analyzing_with_file_dict</str>
+ <str name="lookupImpl">org.apache.solr.spelling.suggest.fst.FuzzyLookupFactory</str>
+ <str name="dictionaryImpl">org.apache.solr.spelling.suggest.FileDictionaryFactory</str>
+ <str name="storeDir">fuzzy_suggest_analyzing</str>
+ <str name="buildOnCommit">false</str>
+
+ <!-- Suggester properties -->
+ <bool name="exactMatchFirst">true</bool>
+ <str name="suggestAnalyzerFieldType">text</str>
+ <bool name="preserveSep">false</bool>
+
+ <str name="sourceLocation">fuzzysuggest.txt</str>
+ </lst>
+ </searchComponent>
<!-- FuzzyLookup suggest component (default)-->
<searchComponent class="solr.SpellCheckComponent" name="fuzzy_suggest_analyzing">
<lst name="spellchecker">
@@ -215,6 +251,26 @@
</arr>
</requestHandler>
+ <requestHandler class="org.apache.solr.handler.component.SearchHandler" name="/fuzzy_suggest_analyzing_with_file_dict">
+ <lst name="defaults">
+ <str name="suggest">true</str>
+ <str name="suggest.dictionary">fuzzy_suggest_analyzing_with_file_dict</str>
+ </lst>
+ <arr name="components">
+ <str>fuzzy_suggest_analyzing_with_file_dict</str>
+ </arr>
+ </requestHandler>
+
+ <requestHandler class="org.apache.solr.handler.component.SearchHandler" name="/fuzzy_suggest_analyzing_with_high_freq_dict">
+ <lst name="defaults">
+ <str name="suggest">true</str>
+ <str name="suggest.dictionary">fuzzy_suggest_analyzing_with_high_freq_dict</str>
+ </lst>
+ <arr name="components">
+ <str>fuzzy_suggest_analyzing_with_high_freq_dict</str>
+ </arr>
+ </requestHandler>
+
<requestHandler class="org.apache.solr.handler.component.SearchHandler" name="/fuzzy_suggest_analyzing">
<lst name="defaults">
<str name="spellcheck">true</str>
Modified: lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml Thu Dec 19 17:48:47 2013
@@ -86,6 +86,18 @@
<requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />
+ <requestHandler name="/admin/file" class="solr.admin.ShowFileRequestHandler" >
+ <lst name="invariants">
+ <str name="hidden">bogus.txt</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/admin/fileedit" class="solr.admin.EditFileRequestHandler" >
+ <lst name="invariants">
+ <str name="hidden">bogus.txt</str>
+ </lst>
+ </requestHandler>
+
<updateRequestProcessorChain name="distrib-dup-test-chain-explicit">
<!-- explicit test using processors before and after distrib -->
<processor class="solr.RegexReplaceProcessorFactory">
Modified: lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Thu Dec 19 17:48:47 2013
@@ -234,6 +234,18 @@
<requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />
+ <requestHandler name="/admin/file" class="solr.admin.ShowFileRequestHandler" >
+ <lst name="invariants">
+ <str name="hidden">bogus.txt</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/admin/fileedit" class="solr.admin.EditFileRequestHandler" >
+ <lst name="invariants">
+ <str name="hidden">bogus.txt</str>
+ </lst>
+ </requestHandler>
+
<!-- test query parameter defaults -->
<requestHandler name="defaults" class="solr.StandardRequestHandler">
<lst name="defaults">
@@ -334,6 +346,8 @@
<str>termsComp</str>
</arr>
</requestHandler>
+
+
<!--
The SpellingQueryConverter to convert raw (CommonParams.Q) queries into tokens. Uses a simple regular expression
to strip off field markup, boosts, ranges, etc. but it is not guaranteed to match an exact parse from the query parser.
Modified: lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/solr-50-all.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/solr-50-all.xml?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/solr-50-all.xml (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/test-files/solr/solr-50-all.xml Thu Dec 19 17:48:47 2013
@@ -17,8 +17,10 @@
<solr>
<str name="adminHandler">testAdminHandler</str>
+ <str name="collectionsHandler">testCollectionsHandler</str>
<int name="coreLoadThreads">11</int>
<str name="coreRootDirectory">${coreRootDirectory:testCoreRootDirectory}</str>
+ <str name="infoHandler">testInfoHandler</str>
<str name="managementPath">testManagementPath</str>
<str name="sharedLib">testSharedLib</str>
<str name="shareSchema">${shareSchema:testShareSchema}</str>
@@ -27,6 +29,8 @@
<solrcloud>
<int name="distribUpdateConnTimeout">22</int>
<int name="distribUpdateSoTimeout">33</int>
+ <int name="maxUpdateConnections">3</int>
+ <int name="maxUpdateConnectionsPerHost">37</int>
<int name="leaderVoteWait">55</int>
<str name="host">testHost</str>
<str name="hostContext">testHostContext</str>
Modified: lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java Thu Dec 19 17:48:47 2013
@@ -121,7 +121,7 @@ public class AnalysisAfterCoreReloadTest
String configDir = core.getResourceLoader().getConfigDir();
FileUtils.moveFile(new File(configDir, "stopwords.txt"), new File(configDir, "stopwords.txt.bak"));
File file = new File(configDir, "stopwords.txt");
- FileUtils.writeStringToFile(file, stopwords);
+ FileUtils.writeStringToFile(file, stopwords, "UTF-8");
} finally {
core.close();
Modified: lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/TestDistributedSearch.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/TestDistributedSearch.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/TestDistributedSearch.java Thu Dec 19 17:48:47 2013
@@ -105,6 +105,8 @@ public class TestDistributedSearch exten
t1,"no eggs on wall, lesson learned",
oddField, "odd man out");
+ indexr(id, "1001", "lowerfilt", "toyota"); // for spellcheck
+
indexr(id, 14, "SubjectTerms_mfacet", new String[] {"mathematical models", "mathematical analysis"});
indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});
indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});
@@ -227,6 +229,9 @@ public class TestDistributedSearch exten
query("q","*:*", "fl", "id", "fl",nint, "fl",tint,"sort",i1 + " desc");
query("q","*:*", "fl",nint, "fl", "id", "fl",tint,"sort",i1 + " desc");
+ // basic spellcheck testing
+ query("q", "toyata", "fl", "id,lowerfilt", "spellcheck", true, "spellcheck.q", "toyata", "qt", "spellCheckCompRH_Direct", "shards.qt", "spellCheckCompRH_Direct");
+
stress=0; // turn off stress... we want to tex max combos in min time
for (int i=0; i<25*RANDOM_MULTIPLIER; i++) {
String f = fieldNames[random().nextInt(fieldNames.length)];
@@ -336,7 +341,7 @@ public class TestDistributedSearch exten
assertNotNull("missing shard info", sinfo);
assertEquals("should have an entry for each shard ["+sinfo+"] "+shards, cnt, sinfo.size());
-
+
// test shards.tolerant=true
for(int numDownServers = 0; numDownServers < jettys.size()-1; numDownServers++)
{
@@ -383,6 +388,22 @@ public class TestDistributedSearch exten
ShardParams.SHARDS_INFO, "true",
ShardParams.SHARDS_TOLERANT, "true");
+ queryPartialResults(upShards, upClients,
+ "q", "*:*",
+ "stats", "true",
+ "stats.field", i1,
+ ShardParams.SHARDS_INFO, "true",
+ ShardParams.SHARDS_TOLERANT, "true");
+
+ queryPartialResults(upShards, upClients,
+ "q", "toyata",
+ "spellcheck", "true",
+ "spellcheck.q", "toyata",
+ "qt", "spellCheckCompRH_Direct",
+ "shards.qt", "spellCheckCompRH_Direct",
+ ShardParams.SHARDS_INFO, "true",
+ ShardParams.SHARDS_TOLERANT, "true");
+
// restart the jettys
for (JettySolrRunner downJetty : downJettys) {
downJetty.start();
@@ -398,9 +419,18 @@ public class TestDistributedSearch exten
// Thread.sleep(10000000000L);
FieldCache.DEFAULT.purgeAllCaches(); // avoid FC insanity
+
+ del("*:*"); // delete all docs and test stats request
+ commit();
+ try {
+ query("q", "*:*", "stats", "true", "stats.field", "stats_dt", "stats.calcdistinct", "true");
+ } catch (Exception e) {
+ log.error("Exception on distrib stats request on empty index", e);
+ fail("NullPointerException with stats request on empty index");
+ }
}
- protected void queryPartialResults(final List<String> upShards,
+ protected void queryPartialResults(final List<String> upShards,
final List<SolrServer> upClients,
Object... q) throws Exception {
@@ -477,6 +507,7 @@ public class TestDistributedSearch exten
// make sure that it responded if it's up
if (upShards.contains(s)) {
assertTrue("Expected to find numFound in the up shard info",info.get("numFound") != null);
+ assertTrue("Expected to find shardAddress in the up shard info",info.get("shardAddress") != null);
}
else {
assertTrue("Expected to find error in the down shard info",info.get("error") != null);
Modified: lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java Thu Dec 19 17:48:47 2013
@@ -27,8 +27,10 @@ import org.apache.solr.client.solrj.Solr
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
@@ -241,23 +243,38 @@ public class AliasIntegrationTest extend
assertTrue(sawException);
}
- private void createAlias(String alias, String collections) throws SolrServerException, IOException {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("collections", collections);
- params.set("name", alias);
- params.set("action", CollectionAction.CREATEALIAS.toString());
- QueryRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
- NamedList<Object> result = createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0))).request(request);
+ private void createAlias(String alias, String collections)
+ throws SolrServerException, IOException {
+ if (random().nextBoolean()) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("collections", collections);
+ params.set("name", alias);
+ params.set("action", CollectionAction.CREATEALIAS.toString());
+ QueryRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+ NamedList<Object> result = createNewSolrServer("",
+ getBaseUrl((HttpSolrServer) clients.get(0))).request(request);
+ } else {
+ CollectionAdminResponse resp = CollectionAdminRequest.CreateAlias
+ .createAlias(alias, collections, createNewSolrServer("",
+ getBaseUrl((HttpSolrServer) clients.get(0))));
+ }
}
- private void deleteAlias(String alias) throws SolrServerException, IOException {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("name", alias);
- params.set("action", CollectionAction.DELETEALIAS.toString());
- QueryRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
- NamedList<Object> result = createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0))).request(request);
+ private void deleteAlias(String alias) throws SolrServerException,
+ IOException {
+ if (random().nextBoolean()) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("name", alias);
+ params.set("action", CollectionAction.DELETEALIAS.toString());
+ QueryRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+ NamedList<Object> result = createNewSolrServer("",
+ getBaseUrl((HttpSolrServer) clients.get(0))).request(request);
+ } else {
+ CollectionAdminResponse resp = CollectionAdminRequest.deleteAlias(alias,
+ createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0))));
+ }
}
protected void indexDoc(List<CloudJettyRunner> skipServers, Object... fields) throws IOException,