You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/02/09 00:53:25 UTC
svn commit: r1658277 [23/38] - in /lucene/dev/branches/lucene6005: ./
dev-tools/ dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/contrib/velocity/ dev-tools/maven/lucene/replicator/
dev-tools/maven/solr/ dev-tools/maven/solr/contrib/...
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java Sun Feb 8 23:53:14 2015
@@ -18,6 +18,7 @@
package org.apache.solr.search;
import com.carrotsearch.hppc.IntIntOpenHashMap;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -33,7 +34,7 @@ import org.apache.solr.common.util.Named
import org.apache.solr.handler.component.MergeStrategy;
import org.apache.solr.handler.component.QueryElevationComponent;
import org.apache.solr.request.SolrQueryRequest;
-
+import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.IndexSearcher;
@@ -43,6 +44,7 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.ScoreDoc;
+
import com.carrotsearch.hppc.IntFloatOpenHashMap;
import org.apache.lucene.util.Bits;
@@ -246,35 +248,24 @@ public class ReRankQParserPlugin extends
this.boostedPriority = boostedPriority;
Sort sort = cmd.getSort();
if(sort == null) {
- this.mainCollector = TopScoreDocCollector.create(Math.max(this.reRankDocs, length),true);
+ this.mainCollector = TopScoreDocCollector.create(Math.max(this.reRankDocs, length));
} else {
sort = sort.rewrite(searcher);
- this.mainCollector = TopFieldCollector.create(sort, Math.max(this.reRankDocs, length), false, true, true, true);
+ this.mainCollector = TopFieldCollector.create(sort, Math.max(this.reRankDocs, length), false, true, true);
}
this.searcher = searcher;
this.reRankWeight = reRankWeight;
}
- public boolean acceptsDocsOutOfOrder() {
- return false;
- }
-
- public void collect(int doc) throws IOException {
- mainCollector.collect(doc);
- }
-
- public void setScorer(Scorer scorer) throws IOException{
- mainCollector.setScorer(scorer);
- }
-
- public void doSetNextReader(LeafReaderContext context) throws IOException{
- mainCollector.getLeafCollector(context);
- }
-
public int getTotalHits() {
return mainCollector.getTotalHits();
}
+ @Override
+ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+ return mainCollector.getLeafCollector(context);
+ }
+
public TopDocs topDocs(int start, int howMany) {
try {
@@ -387,6 +378,7 @@ public class ReRankQParserPlugin extends
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}
+
}
public class BoostedComp implements Comparator {
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Sun Feb 8 23:53:14 2015
@@ -234,12 +234,9 @@ public class SolrIndexSearcher extends I
((DelegatingCollector)collector).finish();
}
}
- catch( TimeLimitingCollector.TimeExceededException x ) {
+ catch( TimeLimitingCollector.TimeExceededException | ExitableDirectoryReader.ExitingReaderException x ) {
log.warn( "Query: " + query + "; " + x.getMessage() );
qr.setPartialResults(true);
- } catch ( ExitableDirectoryReader.ExitingReaderException e) {
- log.warn("Query: " + query + "; " + e.getMessage());
- qr.setPartialResults(true);
}
}
@@ -1597,7 +1594,7 @@ public class SolrIndexSearcher extends I
if (null == cmd.getSort()) {
assert null == cmd.getCursorMark() : "have cursor but no sort";
- return TopScoreDocCollector.create(len, true);
+ return TopScoreDocCollector.create(len);
} else {
// we have a sort
final boolean needScores = (cmd.getFlags() & GET_SCORES) != 0;
@@ -1609,7 +1606,7 @@ public class SolrIndexSearcher extends I
final boolean fillFields = (null != cursor);
final FieldDoc searchAfter = (null != cursor ? cursor.getSearchAfterFieldDoc() : null);
return TopFieldCollector.create(weightedSort, len, searchAfter,
- fillFields, needScores, needScores, true);
+ fillFields, needScores, needScores);
}
}
@@ -1644,10 +1641,6 @@ public class SolrIndexSearcher extends I
public void collect(int doc) {
numHits[0]++;
}
- @Override
- public boolean acceptsDocsOutOfOrder() {
- return true;
- }
};
} else {
collector = new SimpleCollector() {
@@ -1662,10 +1655,6 @@ public class SolrIndexSearcher extends I
float score = scorer.score();
if (score > topscore[0]) topscore[0]=score;
}
- @Override
- public boolean acceptsDocsOutOfOrder() {
- return true;
- }
};
}
@@ -1750,11 +1739,6 @@ public class SolrIndexSearcher extends I
float score = scorer.score();
if (score > topscore[0]) topscore[0] = score;
}
-
- @Override
- public boolean acceptsDocsOutOfOrder() {
- return true;
- }
};
collector = MultiCollector.wrap(setCollector, topScoreCollector);
@@ -2055,16 +2039,17 @@ public class SolrIndexSearcher extends I
int end=0;
int readerIndex = 0;
+ LeafCollector leafCollector = null;
while (iter.hasNext()) {
int doc = iter.nextDoc();
while (doc>=end) {
LeafReaderContext leaf = leafContexts.get(readerIndex++);
base = leaf.docBase;
end = base + leaf.reader().maxDoc();
- topCollector.getLeafCollector(leaf);
+ leafCollector = topCollector.getLeafCollector(leaf);
// we should never need to set the scorer given the settings for the collector
}
- topCollector.collect(doc-base);
+ leafCollector.collect(doc-base);
}
TopDocs topDocs = topCollector.topDocs(0, nDocs);
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SortSpec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SortSpec.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SortSpec.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SortSpec.java Sun Feb 8 23:53:14 2015
@@ -19,13 +19,11 @@ package org.apache.solr.search;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
-
import org.apache.solr.schema.SchemaField;
import java.util.Arrays;
-import java.util.List;
-import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
/***
* SortSpec encapsulates a Lucene Sort and a count of the number of documents
* to return.
@@ -44,28 +42,6 @@ public class SortSpec
setSortAndFields(sort, Arrays.asList(fields));
}
- /** @deprecated Specify both Sort and SchemaField[] when constructing */
- @Deprecated
- public SortSpec(Sort sort, int num) {
- this(sort,0,num);
- }
-
- /** @deprecated Specify both Sort and SchemaField[] when constructing */
- @Deprecated
- public SortSpec(Sort sort, int offset, int num) {
- setSort(sort);
- this.offset=offset;
- this.num=num;
- }
-
- /** @deprecated use {@link #setSortAndFields} */
- @Deprecated
- public void setSort( Sort s )
- {
- sort = s;
- fields = Collections.unmodifiableList(Arrays.asList(new SchemaField[s.getSort().length]));
- }
-
/**
* the specified SchemaFields must correspond one to one with the Sort's SortFields,
* using null where appropriate.
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java Sun Feb 8 23:53:14 2015
@@ -18,7 +18,6 @@ package org.apache.solr.search;
import org.apache.lucene.search.Query;
-import com.spatial4j.core.distance.DistanceUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.SpatialParams;
@@ -76,10 +75,11 @@ public class SpatialFilterQParser extend
FieldType type = sf.getType();
if (type instanceof SpatialQueryable) {
- double radius = localParams.getDouble(SpatialParams.SPHERE_RADIUS, DistanceUtils.EARTH_MEAN_RADIUS_KM);
+ SpatialQueryable queryable = ((SpatialQueryable)type);
+ double radius = localParams.getDouble(SpatialParams.SPHERE_RADIUS, queryable.getSphereRadius());
SpatialOptions opts = new SpatialOptions(pointStr, dist, sf, measStr, radius);
opts.bbox = bbox;
- result = ((SpatialQueryable)type).createSpatialQuery(this, opts);
+ result = queryable.createSpatialQuery(this, opts);
} else {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The field " + fields[0]
+ " does not support spatial filtering");
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java Sun Feb 8 23:53:14 2015
@@ -17,6 +17,10 @@ package org.apache.solr.search.function.
* limitations under the License.
*/
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
import com.spatial4j.core.context.SpatialContext;
import com.spatial4j.core.distance.DistanceUtils;
import com.spatial4j.core.shape.Point;
@@ -33,12 +37,9 @@ import org.apache.solr.schema.SchemaFiel
import org.apache.solr.search.FunctionQParser;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.ValueSourceParser;
+import org.apache.solr.util.DistanceUnits;
import org.apache.solr.util.SpatialUtils;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
/**
* Parses "geodist" creating {@link HaversineConstFunction} or {@link HaversineFunction}
* or calling {@link SpatialStrategy#makeDistanceValueSource(com.spatial4j.core.shape.Point,double)}.
@@ -133,8 +134,11 @@ public class GeoDistValueSourceParser ex
" the point must be supplied as constants");
// note: uses Haversine by default but can be changed via distCalc=...
SpatialStrategy strategy = ((SpatialStrategyMultiValueSource) mv2).strategy;
+ DistanceUnits distanceUnits = ((SpatialStrategyMultiValueSource) mv2).distanceUnits;
Point queryPoint = strategy.getSpatialContext().makePoint(constants[1], constants[0]);
- return strategy.makeDistanceValueSource(queryPoint, DistanceUtils.DEG_TO_KM);
+ if (distanceUnits == DistanceUnits.BACKCOMPAT)
+ distanceUnits = DistanceUnits.KILOMETERS;
+ return strategy.makeDistanceValueSource(queryPoint, distanceUnits.multiplierFromDegreesToThisUnit());
}
if (constants != null && other instanceof VectorValueSource) {
@@ -180,7 +184,7 @@ public class GeoDistValueSourceParser ex
FieldType type = sf.getType();
if (type instanceof AbstractSpatialFieldType) {
AbstractSpatialFieldType asft = (AbstractSpatialFieldType) type;
- return new SpatialStrategyMultiValueSource(asft.getStrategy(sfield));
+ return new SpatialStrategyMultiValueSource(asft.getStrategy(sfield), asft.getDistanceUnits());
}
ValueSource vs = type.getValueSource(sf, fp);
if (vs instanceof MultiValueSource) {
@@ -194,10 +198,12 @@ public class GeoDistValueSourceParser ex
private static class SpatialStrategyMultiValueSource extends VectorValueSource {
final SpatialStrategy strategy;
+ final DistanceUnits distanceUnits;
- public SpatialStrategyMultiValueSource(SpatialStrategy strategy) {
+ public SpatialStrategyMultiValueSource(SpatialStrategy strategy, DistanceUnits distanceUnits) {
super(Collections.EMPTY_LIST);
this.strategy = strategy;
+ this.distanceUnits = distanceUnits;
}
@Override
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java Sun Feb 8 23:53:14 2015
@@ -228,12 +228,9 @@ public class CommandHandler {
try {
searcher.search(query, luceneFilter, collector);
- } catch (TimeLimitingCollector.TimeExceededException x) {
+ } catch (TimeLimitingCollector.TimeExceededException | ExitableDirectoryReader.ExitingReaderException x) {
partialResults = true;
logger.warn( "Query: " + query + "; " + x.getMessage() );
- } catch (ExitableDirectoryReader.ExitingReaderException e) {
- partialResults = true;
- logger.warn( "Query: " + query + "; " + e.getMessage() );
}
if (includeHitCount) {
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java Sun Feb 8 23:53:14 2015
@@ -126,9 +126,9 @@ public class QueryCommand implements Com
@Override
public List<Collector> create() throws IOException {
if (sort == null || sort == Sort.RELEVANCE) {
- collector = TopScoreDocCollector.create(docsToCollect, true);
+ collector = TopScoreDocCollector.create(docsToCollect);
} else {
- collector = TopFieldCollector.create(sort, docsToCollect, true, needScores, needScores, true);
+ collector = TopFieldCollector.create(sort, docsToCollect, true, needScores, needScores);
}
filterCollector = new FilterCollector(docSet, collector);
return Arrays.asList((Collector) filterCollector);
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java Sun Feb 8 23:53:14 2015
@@ -17,21 +17,20 @@
package org.apache.solr.servlet;
-import java.io.InputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+
/**
* A simple servlet to load the Solr Admin UI
*
@@ -63,7 +62,7 @@ public final class LoadAdminUiServlet ex
};
String[] replace = new String[] {
StringEscapeUtils.escapeJavaScript(request.getContextPath()),
- StringEscapeUtils.escapeJavaScript(cores.getAdminPath()),
+ StringEscapeUtils.escapeJavaScript(CoreContainer.CORES_HANDLER_PATH),
StringEscapeUtils.escapeJavaScript(pack.getSpecificationVersion())
};
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java Sun Feb 8 23:53:14 2015
@@ -17,9 +17,37 @@
package org.apache.solr.servlet;
+import java.io.ByteArrayInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.http.client.HttpClient;
+import org.apache.http.Header;
+import org.apache.http.HeaderIterator;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
import org.apache.http.client.methods.HttpGet;
@@ -28,13 +56,10 @@ import org.apache.http.client.methods.Ht
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
-import org.apache.http.Header;
-import org.apache.http.HeaderIterator;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpEntityEnclosingRequest;
-import org.apache.http.HttpResponse;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.Aliases;
@@ -57,7 +82,6 @@ import org.apache.solr.core.CoreContaine
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.handler.ContentStreamHandlerBase;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
@@ -72,31 +96,6 @@ import org.apache.solr.update.processor.
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import java.io.ByteArrayInputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
/**
* This filter looks at the incoming URL maps them to handlers defined in solrconfig.xml
*
@@ -113,7 +112,7 @@ public class SolrDispatchFilter extends
protected String pathPrefix = null; // strip this from the beginning of a path
protected String abortErrorMessage = null;
- protected final HttpClient httpClient = HttpClientUtil.createClient(new ModifiableSolrParams());
+ protected final CloseableHttpClient httpClient = HttpClientUtil.createClient(new ModifiableSolrParams());
public SolrDispatchFilter() {
}
@@ -179,7 +178,7 @@ public class SolrDispatchFilter extends
protected CoreContainer createCoreContainer() {
SolrResourceLoader loader = new SolrResourceLoader(SolrResourceLoader.locateSolrHome());
ConfigSolr config = loadConfigSolr(loader);
- CoreContainer cores = new CoreContainer(loader, config);
+ CoreContainer cores = new CoreContainer(config);
cores.load();
return cores;
}
@@ -190,10 +189,14 @@ public class SolrDispatchFilter extends
@Override
public void destroy() {
- if (cores != null) {
- cores.shutdown();
- cores = null;
- }
+ try {
+ if (cores != null) {
+ cores.shutdown();
+ cores = null;
+ }
+ } finally {
+ IOUtils.closeQuietly(httpClient);
+ }
}
@Override
@@ -247,22 +250,18 @@ public class SolrDispatchFilter extends
path = path.substring( 0, idx );
}
- // Check for the core admin page
- if( path.equals( cores.getAdminPath() ) ) {
- handler = cores.getMultiCoreHandler();
- solrReq = SolrRequestParsers.DEFAULT.parse(null,path, req);
- handleAdminRequest(req, response, handler, solrReq);
- return;
- }
+
boolean usingAliases = false;
List<String> collectionsList = null;
- // Check for the core admin collections url
+
+ // Check for container handlers
handler = cores.getRequestHandler(path);
- if( handler!= null ) {
- solrReq = SolrRequestParsers.DEFAULT.parse(null,path, req);
+ if (handler != null) {
+ solrReq = SolrRequestParsers.DEFAULT.parse(null, path, req);
handleAdminRequest(req, response, handler, solrReq);
return;
- } else {
+ }
+ else {
//otherwise, we should find a core from the path
idx = path.indexOf( "/", 1 );
if( idx > 1 ) {
@@ -527,7 +526,6 @@ public class SolrDispatchFilter extends
urlstr += queryString == null ? "" : "?" + queryString;
- URL url = new URL(urlstr);
boolean isPostOrPutRequest = "POST".equals(req.getMethod()) || "PUT".equals(req.getMethod());
if ("GET".equals(req.getMethod())) {
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java Sun Feb 8 23:53:14 2015
@@ -23,7 +23,6 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import org.apache.lucene.search.spell.Dictionary;
@@ -123,17 +122,19 @@ public class SolrSuggester implements Ac
});
// if store directory is provided make it or load up the lookup with its content
- if (store != null) {
+ if (store != null && !store.isEmpty()) {
storeDir = new File(store);
if (!storeDir.isAbsolute()) {
storeDir = new File(core.getDataDir() + File.separator + storeDir);
}
if (!storeDir.exists()) {
storeDir.mkdirs();
- } else {
- // attempt reload of the stored lookup
+ } else if (getStoreFile().exists()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("attempt reload of the stored lookup from file " + getStoreFile());
+ }
try {
- lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName())));
+ lookup.load(new FileInputStream(getStoreFile()));
} catch (IOException e) {
LOG.warn("Loading stored lookup data failed, possibly not cached yet");
}
@@ -156,12 +157,12 @@ public class SolrSuggester implements Ac
/** Build the underlying Lucene Suggester */
public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {
- LOG.info("build()");
+ LOG.info("SolrSuggester.build(" + name + ")");
dictionary = dictionaryFactory.create(core, searcher);
lookup.build(dictionary);
if (storeDir != null) {
- File target = new File(storeDir, factory.storeFileName());
+ File target = getStoreFile();
if(!lookup.store(new FileOutputStream(target))) {
LOG.error("Store Lookup build failed");
} else {
@@ -172,21 +173,35 @@ public class SolrSuggester implements Ac
/** Reloads the underlying Lucene Suggester */
public void reload(SolrCore core, SolrIndexSearcher searcher) throws IOException {
- LOG.info("reload()");
+ LOG.info("SolrSuggester.reload(" + name + ")");
if (dictionary == null && storeDir != null) {
- // this may be a firstSearcher event, try loading it
- FileInputStream is = new FileInputStream(new File(storeDir, factory.storeFileName()));
- try {
- if (lookup.load(is)) {
- return; // loaded ok
+ File lookupFile = getStoreFile();
+ if (lookupFile.exists()) {
+ // this may be a firstSearcher event, try loading it
+ FileInputStream is = new FileInputStream(lookupFile);
+ try {
+ if (lookup.load(is)) {
+ return; // loaded ok
+ }
+ } finally {
+ IOUtils.closeWhileHandlingException(is);
}
- } finally {
- IOUtils.closeWhileHandlingException(is);
+ } else {
+ LOG.info("lookup file doesn't exist");
}
- LOG.debug("load failed, need to build Lookup again");
}
- // loading was unsuccessful - build it again
- build(core, searcher);
+ }
+
+ /**
+ *
+ * @return the file where this suggester is stored.
+ * null if no storeDir was configured
+ */
+ public File getStoreFile() {
+ if (storeDir == null) {
+ return null;
+ }
+ return new File(storeDir, factory.storeFileName());
}
/** Returns suggestions based on the {@link SuggesterOptions} passed */
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java Sun Feb 8 23:53:14 2015
@@ -22,17 +22,16 @@ import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics.MetricsContext;
-import org.apache.hadoop.metrics.MetricsRecord;
-import org.apache.hadoop.metrics.MetricsUtil;
-import org.apache.hadoop.metrics.Updater;
-import org.apache.hadoop.metrics.jvm.JvmMetrics;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrInfoMBean;
+import org.apache.solr.search.SolrCacheBase;
/**
+ * A {@link SolrInfoMBean} that provides metrics on block cache operations.
+ *
* @lucene.experimental
*/
-public class Metrics implements Updater {
+public class Metrics extends SolrCacheBase {
public static class MethodCall {
public AtomicLong invokes = new AtomicLong();
@@ -60,14 +59,12 @@ public class Metrics implements Updater
public AtomicLong indexMemoryUsage = new AtomicLong(0);
public AtomicLong segmentCount = new AtomicLong(0);
- private MetricsRecord metricsRecord;
private long previous = System.nanoTime();
public static void main(String[] args) throws InterruptedException {
- Configuration conf = new Configuration();
- Metrics metrics = new Metrics(conf);
+ Metrics metrics = new Metrics();
MethodCall methodCall = new MethodCall();
- metrics.methodCalls.put("test",methodCall);
+ metrics.methodCalls.put("test", methodCall);
for (int i = 0; i < 100; i++) {
metrics.blockCacheHit.incrementAndGet();
metrics.blockCacheMiss.incrementAndGet();
@@ -77,53 +74,50 @@ public class Metrics implements Updater
}
}
- public Metrics(Configuration conf) {
- JvmMetrics.init("blockcache", Long.toString(System.currentTimeMillis()));
- MetricsContext metricsContext = MetricsUtil.getContext("blockcache");
- metricsRecord = MetricsUtil.createRecord(metricsContext, "metrics");
- metricsContext.registerUpdater(this);
- }
-
- @Override
- public void doUpdates(MetricsContext context) {
- synchronized (this) {
- long now = System.nanoTime();
- float seconds = (now - previous) / 1000000000.0f;
- metricsRecord.setMetric("blockcache.hit", getPerSecond(blockCacheHit.getAndSet(0), seconds));
- metricsRecord.setMetric("blockcache.miss", getPerSecond(blockCacheMiss.getAndSet(0), seconds));
- metricsRecord.setMetric("blockcache.eviction", getPerSecond(blockCacheEviction.getAndSet(0), seconds));
- metricsRecord.setMetric("blockcache.size", blockCacheSize.get());
- metricsRecord.setMetric("row.reads", getPerSecond(rowReads.getAndSet(0), seconds));
- metricsRecord.setMetric("row.writes", getPerSecond(rowWrites.getAndSet(0), seconds));
- metricsRecord.setMetric("record.reads", getPerSecond(recordReads.getAndSet(0), seconds));
- metricsRecord.setMetric("record.writes", getPerSecond(recordWrites.getAndSet(0), seconds));
- metricsRecord.setMetric("query.external", getPerSecond(queriesExternal.getAndSet(0), seconds));
- metricsRecord.setMetric("query.internal", getPerSecond(queriesInternal.getAndSet(0), seconds));
- metricsRecord.setMetric("buffercache.allocations", getPerSecond(shardBuffercacheAllocate.getAndSet(0), seconds));
- metricsRecord.setMetric("buffercache.lost", getPerSecond(shardBuffercacheLost.getAndSet(0), seconds));
- for (Entry<String,MethodCall> entry : methodCalls.entrySet()) {
- String key = entry.getKey();
- MethodCall value = entry.getValue();
- long invokes = value.invokes.getAndSet(0);
- long times = value.times.getAndSet(0);
-
- float avgTimes = (times / (float) invokes) / 1000000000.0f;
- metricsRecord.setMetric("methodcalls." + key + ".count", getPerSecond(invokes, seconds));
- metricsRecord.setMetric("methodcalls." + key + ".time", avgTimes);
- }
- metricsRecord.setMetric("tables", tableCount.get());
- metricsRecord.setMetric("rows", rowCount.get());
- metricsRecord.setMetric("records", recordCount.get());
- metricsRecord.setMetric("index.count", indexCount.get());
- metricsRecord.setMetric("index.memoryusage", indexMemoryUsage.get());
- metricsRecord.setMetric("index.segments", segmentCount.get());
- previous = now;
+ public NamedList<Number> getStatistics() {
+ NamedList<Number> stats = new NamedList<Number>();
+
+ long now = System.nanoTime();
+ float seconds = (now - previous) / 1000000000.0f;
+
+ long hits = blockCacheHit.getAndSet(0);
+ long lookups = hits + blockCacheMiss.getAndSet(0);
+
+ stats.add("lookups", getPerSecond(lookups, seconds));
+ stats.add("hits", getPerSecond(hits, seconds));
+ stats.add("hitratio", calcHitRatio(lookups, hits));
+ stats.add("evictions", getPerSecond(blockCacheEviction.getAndSet(0), seconds));
+ stats.add("size", blockCacheSize.get());
+ stats.add("row.reads", getPerSecond(rowReads.getAndSet(0), seconds));
+ stats.add("row.writes", getPerSecond(rowWrites.getAndSet(0), seconds));
+ stats.add("record.reads", getPerSecond(recordReads.getAndSet(0), seconds));
+ stats.add("record.writes", getPerSecond(recordWrites.getAndSet(0), seconds));
+ stats.add("query.external", getPerSecond(queriesExternal.getAndSet(0), seconds));
+ stats.add("query.internal", getPerSecond(queriesInternal.getAndSet(0), seconds));
+ stats.add("buffercache.allocations", getPerSecond(shardBuffercacheAllocate.getAndSet(0), seconds));
+ stats.add("buffercache.lost", getPerSecond(shardBuffercacheLost.getAndSet(0), seconds));
+ for (Entry<String,MethodCall> entry : methodCalls.entrySet()) {
+ String key = entry.getKey();
+ MethodCall value = entry.getValue();
+ long invokes = value.invokes.getAndSet(0);
+ long times = value.times.getAndSet(0);
+
+ float avgTimes = (times / (float) invokes) / 1000000000.0f;
+ stats.add("methodcalls." + key + ".count", getPerSecond(invokes, seconds));
+ stats.add("methodcalls." + key + ".time", avgTimes);
}
- metricsRecord.update();
+ stats.add("tables", tableCount.get());
+ stats.add("rows", rowCount.get());
+ stats.add("records", recordCount.get());
+ stats.add("index.count", indexCount.get());
+ stats.add("index.memoryusage", indexMemoryUsage.get());
+ stats.add("index.segments", segmentCount.get());
+ previous = now;
+
+ return stats;
}
private float getPerSecond(long value, float seconds) {
return (float) (value / seconds);
}
-
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java Sun Feb 8 23:53:14 2015
@@ -85,11 +85,11 @@ public class HdfsDirectory extends BaseD
}
continue;
}
- org.apache.solr.util.IOUtils.closeQuietly(fileSystem);
+ org.apache.solr.common.util.IOUtils.closeQuietly(fileSystem);
throw new RuntimeException(
"Problem creating directory: " + hdfsDirPath, e);
} catch (Exception e) {
- org.apache.solr.util.IOUtils.closeQuietly(fileSystem);
+ org.apache.solr.common.util.IOUtils.closeQuietly(fileSystem);
throw new RuntimeException(
"Problem creating directory: " + hdfsDirPath, e);
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java Sun Feb 8 23:53:14 2015
@@ -29,7 +29,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.LockReleaseFailedException;
-import org.apache.solr.util.IOUtils;
+import org.apache.solr.common.util.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java Sun Feb 8 23:53:14 2015
@@ -264,7 +264,7 @@ public final class DefaultSolrCoreState
}
protected SolrIndexWriter createMainIndexWriter(SolrCore core, String name) throws IOException {
- return SolrIndexWriter.create(name, core.getNewIndexDir(),
+ return SolrIndexWriter.create(core, name, core.getNewIndexDir(),
core.getDirectoryFactory(), false, core.getLatestSchema(),
core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec());
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Sun Feb 8 23:53:14 2015
@@ -21,9 +21,11 @@
package org.apache.solr.update;
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SlowCodecReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.BooleanClause;
@@ -233,11 +235,11 @@ public class DirectUpdateHandler2 extend
}
if (cmd.isBlock()) {
- writer.updateDocuments(updateTerm, cmd, schema.getIndexAnalyzer());
+ writer.updateDocuments(updateTerm, cmd);
} else {
Document luceneDocument = cmd.getLuceneDocument();
// SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer);
- writer.updateDocument(updateTerm, luceneDocument, schema.getIndexAnalyzer());
+ writer.updateDocument(updateTerm, luceneDocument);
}
// SolrCore.verbose("updateDocument",updateTerm,"DONE");
@@ -262,9 +264,9 @@ public class DirectUpdateHandler2 extend
} else {
// allow duplicates
if (cmd.isBlock()) {
- writer.addDocuments(cmd, schema.getIndexAnalyzer());
+ writer.addDocuments(cmd);
} else {
- writer.addDocument(cmd.getLuceneDocument(), schema.getIndexAnalyzer());
+ writer.addDocument(cmd.getLuceneDocument());
}
if (ulog != null) ulog.add(cmd);
@@ -440,8 +442,7 @@ public class DirectUpdateHandler2 extend
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
IndexWriter writer = iw.get();
- writer.updateDocument(idTerm, luceneDocument, cmd.getReq().getSchema()
- .getIndexAnalyzer());
+ writer.updateDocument(idTerm, luceneDocument);
for (Query q : dbqList) {
writer.deleteDocuments(new DeleteByQueryWrapper(q, core.getLatestSchema()));
@@ -467,9 +468,15 @@ public class DirectUpdateHandler2 extend
List<DirectoryReader> readers = cmd.readers;
if (readers != null && readers.size() > 0) {
+ List<CodecReader> mergeReaders = new ArrayList<>();
+ for (DirectoryReader reader : readers) {
+ for (LeafReaderContext leaf : reader.leaves()) {
+ mergeReaders.add(SlowCodecReaderWrapper.wrap(leaf.reader()));
+ }
+ }
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
- iw.get().addIndexes(readers.toArray(new IndexReader[readers.size()]));
+ iw.get().addIndexes(mergeReaders.toArray(new CodecReader[mergeReaders.size()]));
} finally {
iw.decref();
}
@@ -650,9 +657,7 @@ public class DirectUpdateHandler2 extend
if (waitSearcher!=null && waitSearcher[0] != null) {
try {
waitSearcher[0].get();
- } catch (InterruptedException e) {
- SolrException.log(log,e);
- } catch (ExecutionException e) {
+ } catch (InterruptedException | ExecutionException e) {
SolrException.log(log,e);
}
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java Sun Feb 8 23:53:14 2015
@@ -33,6 +33,8 @@ import org.apache.solr.common.util.DataI
import org.apache.solr.common.util.FastInputStream;
import org.apache.solr.common.util.FastOutputStream;
import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.apache.solr.util.FSHDFSUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -78,8 +80,9 @@ public class HdfsTransactionLog extends
}
this.tlogFile = tlogFile;
- // TODO: look into forcefully taking over any lease
if (fs.exists(tlogFile) && openExisting) {
+ FSHDFSUtils.recoverFileLease(fs, tlogFile, fs.getConf());
+
tlogOutStream = fs.append(tlogFile);
} else {
fs.delete(tlogFile, false);
@@ -114,6 +117,8 @@ public class HdfsTransactionLog extends
success = true;
+ assert ObjectReleaseTracker.track(this);
+
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} finally {
@@ -305,6 +310,7 @@ public class HdfsTransactionLog extends
log.error("Exception closing tlog.", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} finally {
+ assert ObjectReleaseTracker.release(this);
if (deleteOnClose) {
try {
fs.delete(tlogFile, true);
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java Sun Feb 8 23:53:14 2015
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Locale;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -35,10 +36,10 @@ import org.apache.hadoop.ipc.RemoteExcep
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.IOUtils;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.util.HdfsUtil;
-import org.apache.solr.util.IOUtils;
/** @lucene.experimental */
public class HdfsUpdateLog extends UpdateLog {
@@ -46,6 +47,9 @@ public class HdfsUpdateLog extends Updat
private volatile FileSystem fs;
private volatile Path tlogDir;
private final String confDir;
+
+ // used internally by tests to track total count of failed tran log loads in init
+ public static AtomicLong INIT_FAILED_LOGS_COUNT = new AtomicLong();
public HdfsUpdateLog() {
this.confDir = null;
@@ -66,9 +70,7 @@ public class HdfsUpdateLog extends Updat
if (future != null) {
try {
future.get();
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- } catch (ExecutionException e) {
+ } catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
}
@@ -191,6 +193,7 @@ public class HdfsUpdateLog extends Updat
addOldLog(oldLog, false); // don't remove old logs on startup since more
// than one may be uncapped.
} catch (Exception e) {
+ INIT_FAILED_LOGS_COUNT.incrementAndGet();
SolrException.log(log, "Failure to open existing log file (non fatal) "
+ f, e);
try {
@@ -264,8 +267,6 @@ public class HdfsUpdateLog extends Updat
return path.getName().startsWith(prefix);
}
});
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/PeerSync.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/PeerSync.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/PeerSync.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/PeerSync.java Sun Feb 8 23:53:14 2015
@@ -296,20 +296,20 @@ public class PeerSync {
boolean connectTimeoutExceptionInChain = connectTimeoutExceptionInChain(srsp.getException());
if (connectTimeoutExceptionInChain || solrException instanceof ConnectException || solrException instanceof ConnectTimeoutException
|| solrException instanceof NoHttpResponseException || solrException instanceof SocketException) {
- log.warn(msg() + " couldn't connect to " + srsp.getShardAddress() + ", counting as success");
+ log.warn(msg() + " couldn't connect to " + srsp.getShardAddress() + ", counting as success", srsp.getException());
return true;
}
}
if (cantReachIsSuccess && sreq.purpose == 1 && srsp.getException() instanceof SolrException && ((SolrException) srsp.getException()).code() == 503) {
- log.warn(msg() + " got a 503 from " + srsp.getShardAddress() + ", counting as success");
+ log.warn(msg() + " got a 503 from " + srsp.getShardAddress() + ", counting as success", srsp.getException());
return true;
}
if (cantReachIsSuccess && sreq.purpose == 1 && srsp.getException() instanceof SolrException && ((SolrException) srsp.getException()).code() == 404) {
log.warn(msg() + " got a 404 from " + srsp.getShardAddress() + ", counting as success. " +
- "Perhaps /get is not registered?");
+ "Perhaps /get is not registered?", srsp.getException());
return true;
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java Sun Feb 8 23:53:14 2015
@@ -252,14 +252,11 @@ public class SolrCmdDistributor {
private void submit(final Req req, boolean isCommit) {
if (req.synchronous) {
blockAndDoRetries();
-
- HttpSolrClient client = new HttpSolrClient(req.node.getUrl(), clients.getHttpClient());
- try {
+
+ try (HttpSolrClient client = new HttpSolrClient(req.node.getUrl(), clients.getHttpClient())) {
client.request(req.uReq);
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Failed synchronous update on shard " + req.node + " update: " + req.uReq , e);
- } finally {
- client.shutdown();
}
return;
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java Sun Feb 8 23:53:14 2015
@@ -17,28 +17,30 @@
package org.apache.solr.update;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
import org.apache.lucene.index.*;
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.util.InfoStream;
-import org.apache.lucene.util.PrintStreamInfoStream;
import org.apache.lucene.util.Version;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.MapSerializable;
-import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.util.SolrPluginUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.List;
-import java.util.Map;
+import static org.apache.solr.core.Config.assertWarnOrFail;
/**
* This config object encapsulates IndexWriter config params,
@@ -50,14 +52,8 @@ public class SolrIndexConfig implements
final String defaultMergePolicyClassName;
public static final String DEFAULT_MERGE_SCHEDULER_CLASSNAME = ConcurrentMergeScheduler.class.getName();
public final Version luceneVersion;
-
- /**
- * The explicit value of <useCompoundFile> specified on this index config
- * @deprecated use {@link #getUseCompoundFile}
- */
- @Deprecated
- public final boolean useCompoundFile;
- private boolean effectiveUseCompountFileSetting;
+
+ private boolean effectiveUseCompoundFileSetting;
public final int maxBufferedDocs;
public final int maxMergeDocs;
@@ -87,7 +83,7 @@ public class SolrIndexConfig implements
@SuppressWarnings("deprecation")
private SolrIndexConfig(SolrConfig solrConfig) {
luceneVersion = solrConfig.luceneMatchVersion;
- useCompoundFile = effectiveUseCompountFileSetting = false;
+ effectiveUseCompoundFileSetting = false;
maxBufferedDocs = -1;
maxMergeDocs = -1;
maxIndexingThreads = IndexWriterConfig.DEFAULT_MAX_THREAD_STATES;
@@ -127,18 +123,17 @@ public class SolrIndexConfig implements
// Assert that end-of-life parameters or syntax is not in our config.
// Warn for luceneMatchVersion's before LUCENE_3_6, fail fast above
assertWarnOrFail("The <mergeScheduler>myclass</mergeScheduler> syntax is no longer supported in solrconfig.xml. Please use syntax <mergeScheduler class=\"myclass\"/> instead.",
- !((solrConfig.getNode(prefix+"/mergeScheduler",false) != null) && (solrConfig.get(prefix+"/mergeScheduler/@class",null) == null)),
+ !((solrConfig.getNode(prefix + "/mergeScheduler", false) != null) && (solrConfig.get(prefix + "/mergeScheduler/@class", null) == null)),
true);
assertWarnOrFail("The <mergePolicy>myclass</mergePolicy> syntax is no longer supported in solrconfig.xml. Please use syntax <mergePolicy class=\"myclass\"/> instead.",
- !((solrConfig.getNode(prefix+"/mergePolicy",false) != null) && (solrConfig.get(prefix+"/mergePolicy/@class",null) == null)),
+ !((solrConfig.getNode(prefix + "/mergePolicy", false) != null) && (solrConfig.get(prefix + "/mergePolicy/@class", null) == null)),
true);
assertWarnOrFail("The <luceneAutoCommit>true|false</luceneAutoCommit> parameter is no longer valid in solrconfig.xml.",
- solrConfig.get(prefix+"/luceneAutoCommit", null) == null,
+ solrConfig.get(prefix + "/luceneAutoCommit", null) == null,
true);
defaultMergePolicyClassName = def.defaultMergePolicyClassName;
- useCompoundFile=solrConfig.getBool(prefix+"/useCompoundFile", def.useCompoundFile);
- effectiveUseCompountFileSetting = useCompoundFile;
+ effectiveUseCompoundFileSetting = solrConfig.getBool(prefix+"/useCompoundFile", def.getUseCompoundFile());
maxBufferedDocs=solrConfig.getInt(prefix+"/maxBufferedDocs",def.maxBufferedDocs);
maxMergeDocs=solrConfig.getInt(prefix+"/maxMergeDocs",def.maxMergeDocs);
maxIndexingThreads=solrConfig.getInt(prefix+"/maxIndexingThreads",def.maxIndexingThreads);
@@ -167,13 +162,10 @@ public class SolrIndexConfig implements
}
}
mergedSegmentWarmerInfo = getPluginInfo(prefix + "/mergedSegmentWarmer", solrConfig, def.mergedSegmentWarmerInfo);
- if (mergedSegmentWarmerInfo != null && solrConfig.nrtMode == false) {
- throw new IllegalArgumentException("Supplying a mergedSegmentWarmer will do nothing since nrtMode is false");
- }
assertWarnOrFail("Begining with Solr 5.0, <checkIntegrityAtMerge> option is no longer supported and should be removed from solrconfig.xml (these integrity checks are now automatic)",
- (null == solrConfig.getNode(prefix+"/checkIntegrityAtMerge",false)),
- true);
+ (null == solrConfig.getNode(prefix + "/checkIntegrityAtMerge", false)),
+ true);
}
@Override
public Map<String, Object> toMap() {
@@ -189,32 +181,28 @@ public class SolrIndexConfig implements
return m;
}
- /*
- * Assert that assertCondition is true.
- * If not, prints reason as log warning.
- * If failCondition is true, then throw exception instead of warning
- */
- private void assertWarnOrFail(String reason, boolean assertCondition, boolean failCondition) {
- if(assertCondition) {
- return;
- } else if(failCondition) {
- throw new SolrException(ErrorCode.FORBIDDEN, reason);
- } else {
- log.warn(reason);
- }
- }
-
private PluginInfo getPluginInfo(String path, SolrConfig solrConfig, PluginInfo def) {
List<PluginInfo> l = solrConfig.readPluginInfos(path, false, true);
return l.isEmpty() ? def : l.get(0);
}
- public IndexWriterConfig toIndexWriterConfig(IndexSchema schema) {
- // so that we can update the analyzer on core reload, we pass null
- // for the default analyzer, and explicitly pass an analyzer on
- // appropriate calls to IndexWriter
-
- IndexWriterConfig iwc = new IndexWriterConfig(null);
+ private static class DelayedSchemaAnalyzer extends DelegatingAnalyzerWrapper {
+ private final SolrCore core;
+
+ public DelayedSchemaAnalyzer(SolrCore core) {
+ super(PER_FIELD_REUSE_STRATEGY);
+ this.core = core;
+ }
+
+ @Override
+ protected Analyzer getWrappedAnalyzer(String fieldName) {
+ return core.getLatestSchema().getIndexAnalyzer();
+ }
+ }
+
+ public IndexWriterConfig toIndexWriterConfig(SolrCore core) {
+ IndexSchema schema = core.getLatestSchema();
+ IndexWriterConfig iwc = new IndexWriterConfig(new DelayedSchemaAnalyzer(core));
if (maxBufferedDocs != -1)
iwc.setMaxBufferedDocs(maxBufferedDocs);
@@ -327,7 +315,7 @@ public class SolrIndexConfig implements
}
public boolean getUseCompoundFile() {
- return effectiveUseCompountFileSetting;
+ return effectiveUseCompoundFileSetting;
}
/**
@@ -351,7 +339,7 @@ public class SolrIndexConfig implements
if (useCFSArg instanceof Boolean) {
boolean cfs = ((Boolean)useCFSArg).booleanValue();
log.warn("Please update your config to specify <useCompoundFile>"+cfs+"</useCompoundFile> directly in your <indexConfig> settings.");
- effectiveUseCompountFileSetting = cfs;
+ effectiveUseCompoundFileSetting = cfs;
} else {
log.error("MergePolicy's 'useCompoundFile' init arg is not a boolean, can not apply back compat logic to apply to the IndexWriterConfig: " + useCFSArg.toString());
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Sun Feb 8 23:53:14 2015
@@ -21,13 +21,14 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.FilterCodecReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.SlowCodecReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
@@ -120,7 +121,7 @@ public class SolrIndexSplitter {
} else {
SolrCore core = searcher.getCore();
String path = paths.get(partitionNumber);
- iw = SolrIndexWriter.create("SplittingIndexWriter"+partitionNumber + (ranges != null ? " " + ranges.get(partitionNumber) : ""), path,
+ iw = SolrIndexWriter.create(core, "SplittingIndexWriter"+partitionNumber + (ranges != null ? " " + ranges.get(partitionNumber) : ""), path,
core.getDirectoryFactory(), true, core.getLatestSchema(),
core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec());
}
@@ -129,8 +130,8 @@ public class SolrIndexSplitter {
// This removes deletions but optimize might still be needed because sub-shards will have the same number of segments as the parent shard.
for (int segmentNumber = 0; segmentNumber<leaves.size(); segmentNumber++) {
log.info("SolrIndexSplitter: partition #" + partitionNumber + " partitionCount=" + numPieces + (ranges != null ? " range=" + ranges.get(partitionNumber) : "") + " segment #"+segmentNumber + " segmentCount=" + leaves.size());
- IndexReader subReader = new LiveDocsReader( leaves.get(segmentNumber), segmentDocSets.get(segmentNumber)[partitionNumber] );
- iw.addIndexes(subReader);
+ CodecReader subReader = SlowCodecReaderWrapper.wrap(leaves.get(segmentNumber).reader());
+ iw.addIndexes(new LiveDocsReader(subReader, segmentDocSets.get(segmentNumber)[partitionNumber]));
}
success = true;
} finally {
@@ -232,12 +233,12 @@ public class SolrIndexSplitter {
// change livedocs on the reader to delete those docs we don't want
- static class LiveDocsReader extends FilterLeafReader {
+ static class LiveDocsReader extends FilterCodecReader {
final FixedBitSet liveDocs;
final int numDocs;
- public LiveDocsReader(LeafReaderContext context, FixedBitSet liveDocs) throws IOException {
- super(context.reader());
+ public LiveDocsReader(CodecReader in, FixedBitSet liveDocs) throws IOException {
+ super(in);
this.liveDocs = liveDocs;
this.numDocs = liveDocs.cardinality();
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java Sun Feb 8 23:53:14 2015
@@ -26,10 +26,11 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.InfoStream;
-import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.common.util.IOUtils;
import org.apache.solr.core.DirectoryFactory.DirContext;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.util.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,12 +57,12 @@ public class SolrIndexWriter extends Ind
private InfoStream infoStream;
private Directory directory;
- public static SolrIndexWriter create(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
+ public static SolrIndexWriter create(SolrCore core, String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
SolrIndexWriter w = null;
final Directory d = directoryFactory.get(path, DirContext.DEFAULT, config.lockType);
try {
- w = new SolrIndexWriter(name, path, d, create, schema,
+ w = new SolrIndexWriter(core, name, path, d, create, schema,
config, delPolicy, codec);
w.setDirectoryFactory(directoryFactory);
return w;
@@ -73,9 +74,9 @@ public class SolrIndexWriter extends Ind
}
}
- private SolrIndexWriter(String name, String path, Directory directory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
+ private SolrIndexWriter(SolrCore core, String name, String path, Directory directory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
super(directory,
- config.toIndexWriterConfig(schema).
+ config.toIndexWriterConfig(core).
setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND).
setIndexDeletionPolicy(delPolicy).setCodec(codec)
);
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java Sun Feb 8 23:53:14 2015
@@ -106,7 +106,7 @@ public class StreamingSolrClients {
public synchronized void shutdown() {
for (ConcurrentUpdateSolrClient client : solrClients.values()) {
- client.shutdown();
+ client.close();
}
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/TransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/TransactionLog.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/TransactionLog.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/TransactionLog.java Sun Feb 8 23:53:14 2015
@@ -40,6 +40,7 @@ import org.apache.solr.common.util.DataI
import org.apache.solr.common.util.FastInputStream;
import org.apache.solr.common.util.FastOutputStream;
import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.common.util.ObjectReleaseTracker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -177,6 +178,8 @@ public class TransactionLog {
}
success = true;
+
+ assert ObjectReleaseTracker.track(this);
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
@@ -553,6 +556,8 @@ public class TransactionLog {
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } finally {
+ assert ObjectReleaseTracker.release(this);
}
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/UpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/UpdateLog.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/UpdateLog.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/UpdateLog.java Sun Feb 8 23:53:14 2015
@@ -1302,10 +1302,6 @@ public class UpdateLog implements Plugin
// versionInfo.unblockUpdates();
}
}
- } catch (InterruptedException e) {
- SolrException.log(log,e);
- } catch (IOException e) {
- SolrException.log(log,e);
} catch (Exception e) {
SolrException.log(log,e);
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java Sun Feb 8 23:53:14 2015
@@ -22,12 +22,14 @@ import java.util.concurrent.Executors;
import org.apache.http.client.HttpClient;
import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.impl.conn.SchemeRegistryFactory;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.apache.solr.core.ConfigSolr;
import org.slf4j.Logger;
@@ -42,7 +44,7 @@ public class UpdateShardHandler {
private PoolingClientConnectionManager clientConnectionManager;
- private final HttpClient client;
+ private final CloseableHttpClient client;
public UpdateShardHandler(ConfigSolr cfg) {
@@ -52,7 +54,6 @@ public class UpdateShardHandler {
clientConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
}
-
ModifiableSolrParams params = new ModifiableSolrParams();
if (cfg != null) {
params.set(HttpClientUtil.PROP_SO_TIMEOUT,
@@ -60,7 +61,10 @@ public class UpdateShardHandler {
params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT,
cfg.getDistributedConnectionTimeout());
}
- params.set(HttpClientUtil.PROP_USE_RETRY, false);
+ // in the update case, we want to do retries, and to use
+ // the default Solr retry handler that createClient will
+ // give us
+ params.set(HttpClientUtil.PROP_USE_RETRY, true);
log.info("Creating UpdateShardHandler HTTP client with params: {}", params);
client = HttpClientUtil.createClient(params, clientConnectionManager);
}
@@ -84,6 +88,7 @@ public class UpdateShardHandler {
} catch (Exception e) {
SolrException.log(log, e);
} finally {
+ IOUtils.closeQuietly(client);
clientConnectionManager.shutdown();
}
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java Sun Feb 8 23:53:14 2015
@@ -16,31 +16,25 @@
*/
package org.apache.solr.update.processor;
+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
+
import java.io.IOException;
-import java.util.Collection;
import java.util.ArrayList;
-import java.util.List;
+import java.util.Collection;
import java.util.HashSet;
-
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.util.plugin.SolrCoreAware;
-
-import org.apache.solr.common.util.NamedList;
-
-import org.apache.solr.common.SolrInputField;
-import org.apache.solr.common.SolrInputDocument;
+import java.util.List;
import org.apache.solr.common.SolrException;
-import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
-
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
-
import org.apache.solr.update.AddUpdateCommand;
-
-import org.apache.solr.update.processor.FieldMutatingUpdateProcessorFactory.SelectorParams;
import org.apache.solr.update.processor.FieldMutatingUpdateProcessor.FieldNameSelector;
-
+import org.apache.solr.update.processor.FieldMutatingUpdateProcessorFactory.SelectorParams;
+import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -221,8 +215,11 @@ public class CloneFieldUpdateProcessorFa
boolean modified = false;
for (final String fname : doc.getFieldNames()) {
if (! srcSelector.shouldMutate(fname)) continue;
+
+ Collection<Object> srcFieldValues = doc.getFieldValues(fname);
+ if(srcFieldValues == null || srcFieldValues.isEmpty()) continue;
- for (Object val : doc.getFieldValues(fname)) {
+ for (Object val : srcFieldValues) {
// preserve existing dest boost (multiplicitive), ignore src boost
destField.addValue(val, 1.0f);
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Sun Feb 8 23:53:14 2015
@@ -17,29 +17,7 @@ package org.apache.solr.update.processor
* limitations under the License.
*/
-import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.cloud.CloudDescriptor;
@@ -76,7 +54,6 @@ import org.apache.solr.handler.component
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.CommitUpdateCommand;
@@ -96,6 +73,24 @@ import org.apache.zookeeper.KeeperExcept
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
+
+import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+
// NOT mt-safe... create a new processor for each add thread
// TODO: we really should not wait for distrib after local? unless a certain replication factor is asked for
public class DistributedUpdateProcessor extends UpdateRequestProcessor {
@@ -231,6 +226,7 @@ public class DistributedUpdateProcessor
private final SolrQueryRequest req;
private final SolrQueryResponse rsp;
private final UpdateRequestProcessor next;
+ private final AtomicUpdateDocumentMerger docMerger;
public static final String VERSION_FIELD = "_version_";
@@ -266,12 +262,20 @@ public class DistributedUpdateProcessor
//used for keeping track of replicas that have processed an add/update from the leader
private RequestReplicationTracker replicationTracker = null;
-
+
+ public DistributedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ this(req, rsp, new AtomicUpdateDocumentMerger(req), next);
+ }
+
+ /** Specification of AtomicUpdateDocumentMerger is currently experimental.
+ * @lucene.experimental
+ */
public DistributedUpdateProcessor(SolrQueryRequest req,
- SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ SolrQueryResponse rsp, AtomicUpdateDocumentMerger docMerger, UpdateRequestProcessor next) {
super(next);
this.rsp = rsp;
this.next = next;
+ this.docMerger = docMerger;
this.idField = req.getSchema().getUniqueKeyField();
// version init
@@ -784,7 +788,7 @@ public class DistributedUpdateProcessor
} else {
if (log.isWarnEnabled()) {
for (Error error : errors) {
- log.warn("Error sending update", error.e);
+ log.warn("Error sending update to " + error.req.node.getBaseUrl(), error.e);
}
}
}
@@ -949,7 +953,7 @@ public class DistributedUpdateProcessor
}
if (vinfo == null) {
- if (isAtomicUpdate(cmd)) {
+ if (AtomicUpdateDocumentMerger.isAtomicUpdate(cmd)) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"Atomic document updates are not supported unless <updateLog/> is configured");
@@ -1095,24 +1099,10 @@ public class DistributedUpdateProcessor
return false;
}
- /**
- * Utility method that examines the SolrInputDocument in an AddUpdateCommand
- * and returns true if the documents contains atomic update instructions.
- */
- public static boolean isAtomicUpdate(final AddUpdateCommand cmd) {
- SolrInputDocument sdoc = cmd.getSolrInputDocument();
- for (SolrInputField sif : sdoc.values()) {
- if (sif.getValue() instanceof Map) {
- return true;
- }
- }
- return false;
- }
-
// TODO: may want to switch to using optimistic locking in the future for better concurrency
// that's why this code is here... need to retry in a loop closely around/in versionAdd
boolean getUpdatedDocument(AddUpdateCommand cmd, long versionOnUpdate) throws IOException {
- if (!isAtomicUpdate(cmd)) return false;
+ if (!AtomicUpdateDocumentMerger.isAtomicUpdate(cmd)) return false;
SolrInputDocument sdoc = cmd.getSolrInputDocument();
BytesRef id = cmd.getIndexedId();
@@ -1129,142 +1119,10 @@ public class DistributedUpdateProcessor
} else {
oldDoc.remove(VERSION_FIELD);
}
-
- IndexSchema schema = cmd.getReq().getSchema();
- for (SolrInputField sif : sdoc.values()) {
- Object val = sif.getValue();
- if (val instanceof Map) {
- for (Entry<String,Object> entry : ((Map<String,Object>) val).entrySet()) {
- String key = entry.getKey();
- Object fieldVal = entry.getValue();
- boolean updateField = false;
- switch (key) {
- case "add":
- updateField = true;
- oldDoc.addField(sif.getName(), fieldVal, sif.getBoost());
- break;
- case "set":
- updateField = true;
- oldDoc.setField(sif.getName(), fieldVal, sif.getBoost());
- break;
- case "remove":
- updateField = true;
- doRemove(oldDoc, sif, fieldVal, schema);
- break;
- case "removeregex":
- updateField = true;
- doRemoveRegex(oldDoc, sif, fieldVal);
- break;
- case "inc":
- updateField = true;
- doInc(oldDoc, schema, sif, fieldVal);
- break;
- default:
- //Perhaps throw an error here instead?
- log.warn("Unknown operation for the an atomic update, operation ignored: " + key);
- break;
- }
- // validate that the field being modified is not the id field.
- if (updateField && idField.getName().equals(sif.getName())) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid update of id field: " + sif);
- }
-
- }
- } else {
- // normal fields are treated as a "set"
- oldDoc.put(sif.getName(), sif);
- }
-
- }
-
- cmd.solrDoc = oldDoc;
- return true;
- }
-
- private void doInc(SolrInputDocument oldDoc, IndexSchema schema, SolrInputField sif, Object fieldVal) {
- SolrInputField numericField = oldDoc.get(sif.getName());
- if (numericField == null) {
- oldDoc.setField(sif.getName(), fieldVal, sif.getBoost());
- } else {
- // TODO: fieldtype needs externalToObject?
- String oldValS = numericField.getFirstValue().toString();
- SchemaField sf = schema.getField(sif.getName());
- BytesRefBuilder term = new BytesRefBuilder();
- sf.getType().readableToIndexed(oldValS, term);
- Object oldVal = sf.getType().toObject(sf, term.get());
-
- String fieldValS = fieldVal.toString();
- Number result;
- if (oldVal instanceof Long) {
- result = ((Long) oldVal).longValue() + Long.parseLong(fieldValS);
- } else if (oldVal instanceof Float) {
- result = ((Float) oldVal).floatValue() + Float.parseFloat(fieldValS);
- } else if (oldVal instanceof Double) {
- result = ((Double) oldVal).doubleValue() + Double.parseDouble(fieldValS);
- } else {
- // int, short, byte
- result = ((Integer) oldVal).intValue() + Integer.parseInt(fieldValS);
- }
-
- oldDoc.setField(sif.getName(), result, sif.getBoost());
- }
- }
-
- private boolean doRemove(SolrInputDocument oldDoc, SolrInputField sif, Object fieldVal, IndexSchema schema) {
- final String name = sif.getName();
- SolrInputField existingField = oldDoc.get(name);
- if(existingField == null) return false;
- SchemaField sf = schema.getField(name);
- int oldSize = existingField.getValueCount();
-
- if (sf != null) {
- final Collection<Object> original = existingField.getValues();
- if (fieldVal instanceof Collection) {
- for (Object object : (Collection)fieldVal){
- original.remove(sf.getType().toNativeType(object));
- }
- } else {
- original.remove(sf.getType().toNativeType(fieldVal));
- }
-
- oldDoc.setField(name, original);
-
- }
- return oldSize > existingField.getValueCount();
- }
- private void doRemoveRegex(SolrInputDocument oldDoc, SolrInputField sif, Object valuePatterns) {
- final String name = sif.getName();
- final SolrInputField existingField = oldDoc.get(name);
- if (existingField != null) {
- final Collection<Object> valueToRemove = new HashSet<>();
- final Collection<Object> original = existingField.getValues();
- final Collection<Pattern> patterns = preparePatterns(valuePatterns);
- for (Object value : original) {
- for(Pattern pattern : patterns) {
- final Matcher m = pattern.matcher(value.toString());
- if (m.matches()) {
- valueToRemove.add(value);
- }
- }
- }
- original.removeAll(valueToRemove);
- oldDoc.setField(name, original);
- }
- }
-
- private Collection<Pattern> preparePatterns(Object fieldVal) {
- final Collection<Pattern> patterns = new LinkedHashSet<>(1);
- if (fieldVal instanceof Collection) {
- Collection<String> patternVals = (Collection<String>) fieldVal;
- for (String patternVal : patternVals) {
- patterns.add(Pattern.compile(patternVal));
- }
- } else {
- patterns.add(Pattern.compile(fieldVal.toString()));
- }
- return patterns;
+ cmd.solrDoc = docMerger.merge(sdoc, oldDoc);
+ return true;
}
@Override
@@ -1396,7 +1254,6 @@ public class DistributedUpdateProcessor
SolrParams params = req.getParams();
String route = params.get(ShardParams._ROUTE_);
- if(route == null) route = params.get(ShardParams.SHARD_KEYS);// deprecated . kept for backcompat
Collection<Slice> slices = coll.getRouter().getSearchSlices(route, params, coll);
List<Node> leaders = new ArrayList<>(slices.size());
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java Sun Feb 8 23:53:14 2015
@@ -60,7 +60,7 @@ class RunUpdateProcessor extends UpdateR
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
- if (DistributedUpdateProcessor.isAtomicUpdate(cmd)) {
+ if (AtomicUpdateDocumentMerger.isAtomicUpdate(cmd)) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"RunUpdateProcessor has received an AddUpdateCommand containing a document that appears to still contain Atomic document update operations, most likely because DistributedUpdateProcessorFactory was explicitly disabled from this updateRequestProcessorChain");
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java Sun Feb 8 23:53:14 2015
@@ -134,7 +134,7 @@ public class SignatureUpdateProcessorFac
if (enabled) {
SolrInputDocument doc = cmd.getSolrInputDocument();
List<String> currDocSigFields = null;
- boolean isPartialUpdate = DistributedUpdateProcessor.isAtomicUpdate(cmd);
+ boolean isPartialUpdate = AtomicUpdateDocumentMerger.isAtomicUpdate(cmd);
if (sigFields == null || sigFields.size() == 0) {
if (isPartialUpdate) {
throw new SolrException