You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/09/21 19:22:27 UTC
svn commit: r1388574 [38/45] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/eclipse/dot.settings/
dev-tools/idea/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/ dev-tools/idea/lucene/anal...
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/schema/UUIDField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/schema/UUIDField.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/schema/UUIDField.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/schema/UUIDField.java Fri Sep 21 17:21:34 2012
@@ -22,7 +22,9 @@ import java.util.Locale;
import java.util.Map;
import java.util.UUID;
+import org.apache.lucene.index.GeneralField;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.SortField;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
@@ -53,7 +55,7 @@ public class UUIDField extends StrField
}
@Override
- public void write(TextResponseWriter writer, String name, IndexableField f)
+ public void write(TextResponseWriter writer, String name, StorableField f)
throws IOException {
writer.writeStr(name, f.stringValue(), false);
}
@@ -88,7 +90,7 @@ public class UUIDField extends StrField
}
@Override
- public UUID toObject(IndexableField f) {
+ public UUID toObject(StorableField f) {
return UUID.fromString(f.stringValue());
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/DocIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/DocIterator.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/DocIterator.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/DocIterator.java Fri Sep 21 17:21:34 2012
@@ -34,9 +34,8 @@ public interface DocIterator extends Ite
//public boolean hasNext();
/**
- * Returns the next document id if hasNext()==true
+ * Returns the next document id if <code>hasNext()==true</code>
*
- * <code>
* This method is equivalent to <code>next()</code>, but avoids the creation
* of an Integer Object.
* @see #next()
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java Fri Sep 21 17:21:34 2012
@@ -346,17 +346,13 @@ class ExtendedDismaxQParser extends QPar
/* * * Boosting Query * * */
boostParams = solrParams.getParams(DisMaxParams.BQ);
+ //List<Query> boostQueries = U.parseQueryStrings(req, boostParams);
boostQueries=null;
if (boostParams!=null && boostParams.length>0) {
- Map<String,Float> bqBoosts = SolrPluginUtils.parseFieldBoosts(boostParams);
boostQueries = new ArrayList<Query>();
- for (Map.Entry<String,Float> bqs : bqBoosts.entrySet()) {
- if (bqs.getKey().trim().length()==0) continue;
- Query q = subQuery(bqs.getKey(), null).getQuery();
- Float b = bqs.getValue();
- if(b!=null) {
- q.setBoost(b);
- }
+ for (String qs : boostParams) {
+ if (qs.trim().length()==0) continue;
+ Query q = subQuery(qs, null).getQuery();
boostQueries.add(q);
}
}
@@ -958,7 +954,7 @@ class ExtendedDismaxQParser extends QPar
* Returns the aliases found for a field.
* Returns null if there are no aliases for the field
* @param field
- * @return
+ * @return Alias
*/
public Alias getAlias(String field) {
return aliases.get(field);
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/FunctionQParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/FunctionQParser.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/FunctionQParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/FunctionQParser.java Fri Sep 21 17:21:34 2012
@@ -213,7 +213,7 @@ public class FunctionQParser extends QPa
* Parse a list of ValueSource. Must be the final set of arguments
* to a ValueSource.
*
- * @return List<ValueSource>
+ * @return List<ValueSource>
* @throws ParseException
*/
public List<ValueSource> parseValueSourceList() throws ParseException {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/Grouping.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/Grouping.java Fri Sep 21 17:21:34 2012
@@ -19,6 +19,7 @@ package org.apache.solr.search;
import org.apache.commons.lang.ArrayUtils;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.StorableField;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
@@ -787,7 +788,7 @@ public class Grouping {
SchemaField schemaField = searcher.getSchema().getField(groupBy);
FieldType fieldType = schemaField.getType();
String readableValue = fieldType.indexedToReadable(group.groupValue.utf8ToString());
- IndexableField field = schemaField.createField(readableValue, 1.0f);
+ StorableField field = schemaField.createField(readableValue, 1.0f);
nl.add("groupValue", fieldType.toObject(field));
} else {
nl.add("groupValue", null);
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/QParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/QParser.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/QParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/QParser.java Fri Sep 21 17:21:34 2012
@@ -224,18 +224,18 @@ public abstract class QParser {
String pageScoreS = null;
String pageDocS = null;
- pageScoreS = params.get(CommonParams.PAGESCORE);
- pageDocS = params.get(CommonParams.PAGEDOC);
-
- if (pageScoreS == null || pageDocS == null)
- return null;
-
- int pageDoc = pageDocS != null ? Integer.parseInt(pageDocS) : -1;
- float pageScore = pageScoreS != null ? new Float(pageScoreS) : -1;
- if(pageDoc != -1 && pageScore != -1){
+ pageScoreS = params.get(CommonParams.PAGESCORE);
+ pageDocS = params.get(CommonParams.PAGEDOC);
+
+ if (pageScoreS == null || pageDocS == null)
+ return null;
+
+ int pageDoc = pageDocS != null ? Integer.parseInt(pageDocS) : -1;
+ float pageScore = pageScoreS != null ? new Float(pageScoreS) : -1;
+ if(pageDoc != -1 && pageScore != -1){
return new ScoreDoc(pageDoc, pageScore);
}
- else {
+ else {
return null;
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/QueryParsing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/QueryParsing.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/QueryParsing.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/QueryParsing.java Fri Sep 21 17:21:34 2012
@@ -226,9 +226,9 @@ public class QueryParsing {
* <p>
* The form of the sort specification string currently parsed is:
* </p>
- * <pre>>
+ * <pre>
* SortSpec ::= SingleSort [, SingleSort]*
- * SingleSort ::= <fieldname> SortDirection
+ * SingleSort ::= <fieldname> SortDirection
* SortDirection ::= top | desc | bottom | asc
* </pre>
* Examples:
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Fri Sep 21 17:21:34 2012
@@ -96,7 +96,7 @@ public class SolrIndexSearcher extends I
private final boolean cachingEnabled;
private final SolrCache<Query,DocSet> filterCache;
private final SolrCache<QueryResultKey,DocList> queryResultCache;
- private final SolrCache<Integer,Document> documentCache;
+ private final SolrCache<Integer,StoredDocument> documentCache;
private final SolrCache<String,UnInvertedField> fieldValueCache;
private final LuceneQueryOptimizer optimizer;
@@ -434,7 +434,7 @@ public class SolrIndexSearcher extends I
// need to open up access to its Document...
static class SetNonLazyFieldSelector extends StoredFieldVisitor {
private Set<String> fieldsToLoad;
- final Document doc = new Document();
+ final StoredDocument doc = new StoredDocument();
final LazyDocument lazyDoc;
SetNonLazyFieldSelector(Set<String> toLoad, IndexReader reader, int docID) {
@@ -503,7 +503,7 @@ public class SolrIndexSearcher extends I
* Retrieve the {@link Document} instance corresponding to the document id.
*/
@Override
- public Document doc(int i) throws IOException {
+ public StoredDocument doc(int i) throws IOException {
return doc(i, (Set<String>)null);
}
@@ -523,9 +523,9 @@ public class SolrIndexSearcher extends I
* filter is provided, only the provided fields will be loaded (the
* remainder will be available lazily).
*/
- public Document doc(int i, Set<String> fields) throws IOException {
+ public StoredDocument doc(int i, Set<String> fields) throws IOException {
- Document d;
+ StoredDocument d;
if (documentCache != null) {
d = documentCache.get(i);
if (d!=null) return d;
@@ -550,14 +550,14 @@ public class SolrIndexSearcher extends I
* Takes a list of docs (the doc ids actually), and reads them into an array
* of Documents.
*/
- public void readDocs(Document[] docs, DocList ids) throws IOException {
+ public void readDocs(StoredDocument[] docs, DocList ids) throws IOException {
readDocs(docs, ids, null);
}
/**
* Takes a list of docs (the doc ids actually) and a set of fields to load,
* and reads them into an array of Documents.
*/
- public void readDocs(Document[] docs, DocList ids, Set<String> fields) throws IOException {
+ public void readDocs(StoredDocument[] docs, DocList ids, Set<String> fields) throws IOException {
DocIterator iter = ids.iterator();
for (int i=0; i<docs.length; i++) {
docs[i] = doc(iter.nextDoc(), fields);
@@ -616,19 +616,18 @@ public class SolrIndexSearcher extends I
final AtomicReaderContext leaf = leafContexts.get(i);
final AtomicReader reader = leaf.reader();
- final Fields fields = reader.fields();
- if (fields == null) continue;
-
- final Bits liveDocs = reader.getLiveDocs();
+ final Terms terms = reader.terms(field);
+ if (terms == null) continue;
- final DocsEnum docs = reader.termDocsEnum(liveDocs, field, idBytes, 0);
-
- if (docs == null) continue;
- int id = docs.nextDoc();
- if (id == DocIdSetIterator.NO_MORE_DOCS) continue;
- assert docs.nextDoc() == DocIdSetIterator.NO_MORE_DOCS;
+ TermsEnum te = terms.iterator(null);
+ if (te.seekExact(idBytes, true)) {
+ DocsEnum docs = te.docs(reader.getLiveDocs(), null, 0);
+ int id = docs.nextDoc();
+ if (id == DocIdSetIterator.NO_MORE_DOCS) continue;
+ assert docs.nextDoc() == DocIdSetIterator.NO_MORE_DOCS;
- return (((long)i) << 32) | id;
+ return (((long)i) << 32) | id;
+ }
}
return -1;
@@ -1371,7 +1370,7 @@ public class SolrIndexSearcher extends I
TopDocsCollector topCollector;
if (cmd.getSort() == null) {
if(cmd.getScoreDoc() != null) {
- topCollector = TopScoreDocCollector.create(len, cmd.getScoreDoc(), true); //create the Collector with InOrderPagingCollector
+ topCollector = TopScoreDocCollector.create(len, cmd.getScoreDoc(), true); //create the Collector with InOrderPagingCollector
} else {
topCollector = TopScoreDocCollector.create(len, true);
}
@@ -1894,8 +1893,8 @@ public class SolrIndexSearcher extends I
* Takes a list of docs (the doc ids actually), and returns an array
* of Documents containing all of the stored fields.
*/
- public Document[] readDocs(DocList ids) throws IOException {
- Document[] docs = new Document[ids.size()];
+ public StoredDocument[] readDocs(DocList ids) throws IOException {
+ StoredDocument[] docs = new StoredDocument[ids.size()];
readDocs(docs,ids);
return docs;
}
@@ -2031,11 +2030,11 @@ public class SolrIndexSearcher extends I
public ScoreDoc getScoreDoc()
{
- return scoreDoc;
+ return scoreDoc;
}
public void setScoreDoc(ScoreDoc scoreDoc)
{
- this.scoreDoc = scoreDoc;
+ this.scoreDoc = scoreDoc;
}
//Issue 1726 end
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java Fri Sep 21 17:21:34 2012
@@ -19,7 +19,6 @@ package org.apache.solr.search;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.Query;
-import com.spatial4j.core.distance.DistanceUnits;
import com.spatial4j.core.distance.DistanceUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
@@ -79,7 +78,7 @@ public class SpatialFilterQParser extend
if (type instanceof SpatialQueryable) {
double radius = localParams.getDouble(SpatialParams.SPHERE_RADIUS, DistanceUtils.EARTH_MEAN_RADIUS_KM);
- SpatialOptions opts = new SpatialOptions(pointStr, dist, sf, measStr, radius, DistanceUnits.KILOMETERS);
+ SpatialOptions opts = new SpatialOptions(pointStr, dist, sf, measStr, radius);
opts.bbox = bbox;
result = ((SpatialQueryable)type).createSpatialQuery(this, opts);
} else {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SpatialOptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SpatialOptions.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SpatialOptions.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/SpatialOptions.java Fri Sep 21 17:21:34 2012
@@ -16,7 +16,6 @@ package org.apache.solr.search;
* limitations under the License.
*/
-import com.spatial4j.core.distance.DistanceUnits;
import org.apache.solr.schema.SchemaField;
/**
@@ -28,8 +27,7 @@ public class SpatialOptions {
public double distance;
public SchemaField field;
public String measStr;
- public double radius;
- public DistanceUnits units;
+ public double radius;//(planetRadius) effectively establishes the units
/** Just do a "bounding box" - or any other quicker method / shape that
* still encompasses all of the points of interest, but may also encompass
@@ -40,19 +38,11 @@ public class SpatialOptions {
public SpatialOptions() {
}
-
public SpatialOptions(String pointStr, double dist, SchemaField sf, String measStr, double radius) {
- this(pointStr, dist, sf, measStr, radius, DistanceUnits.MILES);
-
- }
-
-
- public SpatialOptions(String pointStr, double dist, SchemaField sf, String measStr, double radius, DistanceUnits units) {
this.pointStr = pointStr;
this.distance = dist;
this.field = sf;
this.measStr = measStr;
this.radius = radius;
- this.units = units;
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java Fri Sep 21 17:21:34 2012
@@ -16,6 +16,7 @@
*/
package org.apache.solr.search;
+import com.spatial4j.core.distance.DistanceUtils;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.BoostedQuery;
@@ -177,6 +178,23 @@ public abstract class ValueSourceParser
return new DivFloatFunction(a, b);
}
});
+ addParser("mod", new ValueSourceParser() {
+ @Override
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ ValueSource a = fp.parseValueSource();
+ ValueSource b = fp.parseValueSource();
+ return new DualFloatFunction(a, b) {
+ @Override
+ protected String name() {
+ return "mod";
+ }
+ @Override
+ protected float func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ return aVals.floatVal(doc) % bVals.floatVal(doc);
+ }
+ };
+ }
+ });
addParser("map", new ValueSourceParser() {
@Override
public ValueSource parse(FunctionQParser fp) throws ParseException {
@@ -381,13 +399,13 @@ public abstract class ValueSourceParser
addParser(new DoubleParser("rad") {
@Override
public double func(int doc, FunctionValues vals) {
- return vals.doubleVal(doc) * HaversineConstFunction.DEGREES_TO_RADIANS;
+ return vals.doubleVal(doc) * DistanceUtils.DEGREES_TO_RADIANS;
}
});
addParser(new DoubleParser("deg") {
@Override
public double func(int doc, FunctionValues vals) {
- return vals.doubleVal(doc) * HaversineConstFunction.RADIANS_TO_DEGREES;
+ return vals.doubleVal(doc) * DistanceUtils.RADIANS_TO_DEGREES;
}
});
addParser(new DoubleParser("sqrt") {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java Fri Sep 21 17:21:34 2012
@@ -19,7 +19,7 @@ package org.apache.solr.search.function.
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
-import com.spatial4j.core.util.GeohashUtils;
+import com.spatial4j.core.io.GeohashUtils;
import java.util.Map;
import java.io.IOException;
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java Fri Sep 21 17:21:34 2012
@@ -17,21 +17,19 @@ package org.apache.solr.search.function.
*/
+import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.distance.DistanceUtils;
+import com.spatial4j.core.distance.GeodesicSphereDistCalc;
+import com.spatial4j.core.io.GeohashUtils;
+import com.spatial4j.core.shape.Point;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.IndexSearcher;
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.context.simple.SimpleSpatialContext;
-import com.spatial4j.core.distance.DistanceCalculator;
-import com.spatial4j.core.distance.DistanceUnits;
-import com.spatial4j.core.distance.GeodesicSphereDistCalc;
-import com.spatial4j.core.util.GeohashUtils;
-import com.spatial4j.core.shape.Point;
-import java.util.Map;
import java.io.IOException;
+import java.util.Map;
/**
@@ -46,16 +44,16 @@ import java.io.IOException;
**/
public class GeohashHaversineFunction extends ValueSource {
- private ValueSource geoHash1, geoHash2;
- private double radius;
+ private final ValueSource geoHash1, geoHash2;
private final SpatialContext ctx;
+ private final double degreesToDist;
public GeohashHaversineFunction(ValueSource geoHash1, ValueSource geoHash2, double radius) {
this.geoHash1 = geoHash1;
this.geoHash2 = geoHash2;
- this.radius = radius;
- DistanceCalculator distCalc = new GeodesicSphereDistCalc.Haversine(radius);
- this.ctx = new SimpleSpatialContext(DistanceUnits.KILOMETERS,distCalc,null);
+ this.degreesToDist = DistanceUtils.degrees2Dist(1, radius);
+ this.ctx = SpatialContext.GEO;
+ assert this.ctx.getDistCalc() instanceof GeodesicSphereDistCalc.Haversine;
}
protected String name() {
@@ -92,7 +90,7 @@ public class GeohashHaversineFunction ex
//and avoid decoding every time
Point p1 = GeohashUtils.decode(h1,ctx);
Point p2 = GeohashUtils.decode(h2,ctx);
- result = ctx.getDistCalc().distance(p1, p2);
+ result = ctx.getDistCalc().distance(p1, p2) * degreesToDist;
} else if (h1 == null || h2 == null){
result = Double.MAX_VALUE;
}
@@ -112,7 +110,7 @@ public class GeohashHaversineFunction ex
return this.name().equals(other.name())
&& geoHash1.equals(other.geoHash1) &&
geoHash2.equals(other.geoHash2) &&
- radius == other.radius;
+ degreesToDist == other.degreesToDist;
}
@Override
@@ -121,7 +119,7 @@ public class GeohashHaversineFunction ex
result = geoHash1.hashCode();
result = 31 * result + geoHash2.hashCode();
result = 31 * result + name().hashCode();
- long temp =Double.doubleToRawLongBits(radius);
+ long temp =Double.doubleToRawLongBits(degreesToDist);
result = 31 * result + (int) (temp ^ (temp >>> 32));
return result;
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java Fri Sep 21 17:21:34 2012
@@ -26,7 +26,7 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.valuesource.VectorValueSource;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.IndexSearcher;
-import com.spatial4j.core.context.ParseUtils;
+import com.spatial4j.core.io.ParseUtils;
import com.spatial4j.core.distance.DistanceUtils;
import com.spatial4j.core.exception.InvalidShapeException;
import org.apache.solr.common.params.SpatialParams;
@@ -39,20 +39,18 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import static com.spatial4j.core.distance.DistanceUtils.DEGREES_TO_RADIANS;
/**
* Haversine function with one point constant
*/
public class HaversineConstFunction extends ValueSource {
- // TODO: these could go in spatial4j somewhere
- public static final double DEGREES_TO_RADIANS = Math.PI / 180.0;
- public static final double RADIANS_TO_DEGREES = 180.0 / Math.PI;
public static ValueSourceParser parser = new ValueSourceParser() {
@Override
public ValueSource parse(FunctionQParser fp) throws ParseException
{
- // TODO: dispatch through SpatialQueriable in the future?
+ // TODO: dispatch through SpatialQueryable in the future?
List<ValueSource> sources = fp.parseValueSourceList();
// "m" is a multi-value source, "x" is a single-value source
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java Fri Sep 21 17:21:34 2012
@@ -81,10 +81,10 @@ public class HaversineFunction extends V
double y2;
double x2;
if (convertToRadians) {
- y1 = p1D[0] * HaversineConstFunction.DEGREES_TO_RADIANS;
- x1 = p1D[1] * HaversineConstFunction.DEGREES_TO_RADIANS;
- y2 = p2D[0] * HaversineConstFunction.DEGREES_TO_RADIANS;
- x2 = p2D[1] * HaversineConstFunction.DEGREES_TO_RADIANS;
+ y1 = p1D[0] * DistanceUtils.DEGREES_TO_RADIANS;
+ x1 = p1D[1] * DistanceUtils.DEGREES_TO_RADIANS;
+ y2 = p2D[0] * DistanceUtils.DEGREES_TO_RADIANS;
+ x2 = p2D[1] * DistanceUtils.DEGREES_TO_RADIANS;
} else {
y1 = p1D[0];
x1 = p1D[1];
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java Fri Sep 21 17:21:34 2012
@@ -19,6 +19,7 @@ package org.apache.solr.search.grouping.
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
+import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
@@ -179,7 +180,7 @@ public class TopGroupsResultTransformer
NamedList<Object> document = new NamedList<Object>();
documents.add(document);
- Document doc = retrieveDocument(uniqueField, searchGroup.scoreDocs[i].doc);
+ StoredDocument doc = retrieveDocument(uniqueField, searchGroup.scoreDocs[i].doc);
document.add("id", uniqueField.getType().toExternal(doc.getField(uniqueField.getName())));
if (!Float.isNaN(searchGroup.scoreDocs[i].score)) {
document.add("score", searchGroup.scoreDocs[i].score);
@@ -232,7 +233,7 @@ public class TopGroupsResultTransformer
NamedList<Object> document = new NamedList<Object>();
documents.add(document);
- Document doc = retrieveDocument(uniqueField, scoreDoc.doc);
+ StoredDocument doc = retrieveDocument(uniqueField, scoreDoc.doc);
document.add("id", uniqueField.getType().toExternal(doc.getField(uniqueField.getName())));
if (rb.getGroupingSpec().isNeedScore()) {
document.add("score", scoreDoc.score);
@@ -265,7 +266,7 @@ public class TopGroupsResultTransformer
return queryResult;
}
- private Document retrieveDocument(final SchemaField uniqueField, int doc) throws IOException {
+ private StoredDocument retrieveDocument(final SchemaField uniqueField, int doc) throws IOException {
DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(uniqueField.getName());
rb.req.getSearcher().doc(doc, visitor);
return visitor.getDocument();
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java Fri Sep 21 17:21:34 2012
@@ -40,6 +40,7 @@ import javax.servlet.http.HttpServletRes
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -138,7 +139,7 @@ public class SolrDispatchFilter implemen
}
if (this.cores == null) {
- ((HttpServletResponse)response).sendError( 403, "Server is shutting down" );
+ ((HttpServletResponse)response).sendError( 503, "Server is shutting down" );
return;
}
CoreContainer cores = this.cores;
@@ -335,10 +336,10 @@ public class SolrDispatchFilter implemen
}
// check everyone then
- Map<String,ZkNodeProps> shards = entry.getValue().getShards();
- Set<Entry<String,ZkNodeProps>> shardEntries = shards.entrySet();
- for (Entry<String,ZkNodeProps> shardEntry : shardEntries) {
- ZkNodeProps zkProps = shardEntry.getValue();
+ Map<String,Replica> shards = entry.getValue().getReplicasMap();
+ Set<Entry<String,Replica>> shardEntries = shards.entrySet();
+ for (Entry<String,Replica> shardEntry : shardEntries) {
+ Replica zkProps = shardEntry.getValue();
core = checkProps(cores, path, zkProps);
if (core != null) {
break done;
@@ -352,8 +353,8 @@ public class SolrDispatchFilter implemen
ZkNodeProps zkProps) {
String corename;
SolrCore core = null;
- if (cores.getZkController().getNodeName().equals(zkProps.get(ZkStateReader.NODE_NAME_PROP))) {
- corename = zkProps.get(ZkStateReader.CORE_NAME_PROP);
+ if (cores.getZkController().getNodeName().equals(zkProps.getStr(ZkStateReader.NODE_NAME_PROP))) {
+ corename = zkProps.getStr(ZkStateReader.CORE_NAME_PROP);
core = cores.getCore(corename);
}
return core;
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java Fri Sep 21 17:21:34 2012
@@ -17,6 +17,16 @@
package org.apache.solr.servlet;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.URLEncoder;
+import java.util.Date;
+import java.util.List;
+
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
import org.apache.lucene.util.BytesRef;
import org.apache.noggit.CharArr;
import org.apache.noggit.JSONWriter;
@@ -28,16 +38,6 @@ import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.net.URLEncoder;
-import java.util.Date;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
/**
* Zookeeper Info
@@ -148,13 +148,7 @@ public final class ZookeeperInfoServlet
try {
zkClient = new SolrZkClient(addr, 10000);
doClose = true;
- } catch (TimeoutException e) {
- writeError(503, "Could not connect to zookeeper at '" + addr + "'\"");
- zkClient = null;
- return;
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
+ } catch (Exception e) {
writeError(503, "Could not connect to zookeeper at '" + addr + "'\"");
zkClient = null;
return;
@@ -163,12 +157,8 @@ public final class ZookeeperInfoServlet
}
public void close() {
- try {
- if (doClose) {
- zkClient.close();
- }
- } catch (InterruptedException e) {
- // ignore exception on close
+ if (doClose) {
+ zkClient.close();
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java Fri Sep 21 17:21:34 2012
@@ -142,7 +142,7 @@ public abstract class AbstractLuceneSpel
@Override
public SpellingResult getSuggestions(SpellingOptions options) throws IOException {
- SpellingResult result = new SpellingResult(options.tokens);
+ SpellingResult result = new SpellingResult(options.tokens);
IndexReader reader = determineReader(options.reader);
Term term = field != null ? new Term(field, "") : null;
float theAccuracy = (options.accuracy == Float.MIN_VALUE) ? spellChecker.getAccuracy() : options.accuracy;
@@ -187,13 +187,13 @@ public abstract class AbstractLuceneSpel
int countLimit = Math.min(options.count, suggestions.length);
if(countLimit>0)
{
- for (int i = 0; i < countLimit; i++) {
- term = new Term(field, suggestions[i]);
- result.add(token, suggestions[i], reader.docFreq(term));
- }
+ for (int i = 0; i < countLimit; i++) {
+ term = new Term(field, suggestions[i]);
+ result.add(token, suggestions[i], reader.docFreq(term));
+ }
} else {
- List<String> suggList = Collections.emptyList();
- result.add(token, suggList);
+ List<String> suggList = Collections.emptyList();
+ result.add(token, suggList);
}
} else {
if (suggestions.length > 0) {
@@ -203,8 +203,8 @@ public abstract class AbstractLuceneSpel
}
result.add(token, suggList);
} else {
- List<String> suggList = Collections.emptyList();
- result.add(token, suggList);
+ List<String> suggList = Collections.emptyList();
+ result.add(token, suggList);
}
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java Fri Sep 21 17:21:34 2012
@@ -139,7 +139,7 @@ public class ConjunctionSolrSpellChecker
Map<Token, Integer> combinedTokenFrequency = new HashMap<Token, Integer>();
Map<Token, List<LinkedHashMap<String, Integer>>> allSuggestions = new LinkedHashMap<Token, List<LinkedHashMap<String, Integer>>>();
for(SpellingResult result : results) {
- if(result.getTokenFrequency()!=null) {
+ if(result.getTokenFrequency()!=null) {
combinedTokenFrequency.putAll(result.getTokenFrequency());
}
for(Map.Entry<Token, LinkedHashMap<String, Integer>> entry : result.getSuggestions().entrySet()) {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java Fri Sep 21 17:21:34 2012
@@ -214,7 +214,7 @@ public class DirectSolrSpellChecker exte
result.add(token, empty);
} else {
for (SuggestWord suggestion : suggestions) {
- result.add(token, suggestion.string, suggestion.freq);
+ result.add(token, suggestion.string, suggestion.freq);
}
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java Fri Sep 21 17:21:34 2012
@@ -177,8 +177,6 @@ public class PossibilityIterator impleme
* Rank here is the sum of each selected term's position in its respective
* LinkedHashMap.
* </p>
- *
- * @return
*/
private RankedSpellPossibility internalNext() {
if (nextOnes != null && nextOnes.hasNext()) {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java Fri Sep 21 17:21:34 2012
@@ -19,7 +19,6 @@ package org.apache.solr.update;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
@@ -28,8 +27,6 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
-import java.util.List;
-
/**
*
*/
@@ -118,6 +115,35 @@ public class AddUpdateCommand extends Up
return "(null)";
}
+ /**
+ * @return String id to hash
+ */
+ public String getHashableId() {
+ String id = null;
+ IndexSchema schema = req.getSchema();
+ SchemaField sf = schema.getUniqueKeyField();
+ if (sf != null) {
+ if (solrDoc != null) {
+ SolrInputField field = solrDoc.getField(sf.getName());
+
+ int count = field == null ? 0 : field.getValueCount();
+ if (count == 0) {
+ if (overwrite) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Document is missing mandatory uniqueKey field: "
+ + sf.getName());
+ }
+ } else if (count > 1) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Document contains multiple values for uniqueKey field: " + field);
+ } else {
+ return field.getFirstValue().toString();
+ }
+ }
+ }
+ return id;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder(super.toString());
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java Fri Sep 21 17:21:34 2012
@@ -29,7 +29,7 @@ import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public final class DefaultSolrCoreState extends SolrCoreState {
+public final class DefaultSolrCoreState extends SolrCoreState implements RecoveryStrategy.RecoveryListener {
public static Logger log = LoggerFactory.getLogger(DefaultSolrCoreState.class);
private final boolean SKIP_AUTO_RECOVERY = Boolean.getBoolean("solrcloud.skip.autorecovery");
@@ -43,7 +43,7 @@ public final class DefaultSolrCoreState
private SolrIndexWriter indexWriter = null;
private DirectoryFactory directoryFactory;
- private boolean recoveryRunning;
+ private volatile boolean recoveryRunning;
private RecoveryStrategy recoveryStrat;
private boolean closed = false;
@@ -74,8 +74,7 @@ public final class DefaultSolrCoreState
}
if (indexWriter == null) {
- indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2",
- false, false);
+ indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2", false);
}
if (refCntWriter == null) {
refCntWriter = new RefCounted<IndexWriter>(indexWriter) {
@@ -97,29 +96,42 @@ public final class DefaultSolrCoreState
@Override
public synchronized void newIndexWriter(SolrCore core, boolean rollback) throws IOException {
-
+ log.info("Creating new IndexWriter...");
+ String coreName = core.getName();
synchronized (writerPauseLock) {
// we need to wait for the Writer to fall out of use
// first lets stop it from being lent out
pauseWriter = true;
// then lets wait until its out of use
+ log.info("Waiting until IndexWriter is unused... core=" + coreName);
while (!writerFree) {
try {
writerPauseLock.wait();
} catch (InterruptedException e) {}
}
-
+
try {
if (indexWriter != null) {
- try {
- indexWriter.close();
- } catch (Throwable t) {
- SolrException.log(log, "Error closing old IndexWriter", t);
+ if (!rollback) {
+ try {
+ log.info("Closing old IndexWriter... core=" + coreName);
+ indexWriter.close();
+ } catch (Throwable t) {
+ SolrException.log(log, "Error closing old IndexWriter. core="
+ + coreName, t);
+ }
+ } else {
+ try {
+ log.info("Rollback old IndexWriter... core=" + coreName);
+ indexWriter.rollback();
+ } catch (Throwable t) {
+ SolrException.log(log, "Error rolling back old IndexWriter. core="
+ + coreName, t);
+ }
}
}
-
- indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2",
- false, true);
+ indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2", true);
+ log.info("New IndexWriter is ready to be used.");
// we need to null this so it picks up the new writer next get call
refCntWriter = null;
} finally {
@@ -136,6 +148,7 @@ public final class DefaultSolrCoreState
refCnt--;
if (refCnt == 0) {
try {
+ log.info("SolrCoreState ref count has reached 0 - closing IndexWriter");
if (closer != null) {
closer.closeWriter(indexWriter);
} else if (indexWriter != null) {
@@ -150,6 +163,7 @@ public final class DefaultSolrCoreState
log.error("Error during shutdown of directory factory.", t);
}
try {
+ log.info("Closing SolrCoreState - canceling any ongoing recovery");
cancelRecovery();
} catch (Throwable t) {
log.error("Error cancelling recovery", t);
@@ -170,14 +184,12 @@ public final class DefaultSolrCoreState
@Override
public synchronized void rollbackIndexWriter(SolrCore core) throws IOException {
- indexWriter.rollback();
newIndexWriter(core, true);
}
- protected SolrIndexWriter createMainIndexWriter(SolrCore core, String name,
- boolean removeAllExisting, boolean forceNewDirectory) throws IOException {
- return new SolrIndexWriter(name, core.getNewIndexDir(),
- core.getDirectoryFactory(), removeAllExisting, core.getSchema(),
+ protected SolrIndexWriter createMainIndexWriter(SolrCore core, String name, boolean forceNewDirectory) throws IOException {
+ return SolrIndexWriter.create(name, core.getNewIndexDir(),
+ core.getDirectoryFactory(), false, core.getSchema(),
core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec(), forceNewDirectory);
}
@@ -199,6 +211,7 @@ public final class DefaultSolrCoreState
}
synchronized (recoveryLock) {
+ log.info("Running recovery - first canceling any ongoing recovery");
cancelRecovery();
while (recoveryRunning) {
@@ -218,7 +231,7 @@ public final class DefaultSolrCoreState
// if true, we are recovering after startup and shouldn't have (or be receiving) additional updates (except for local tlog recovery)
boolean recoveringAfterStartup = recoveryStrat == null;
- recoveryStrat = new RecoveryStrategy(cc, name);
+ recoveryStrat = new RecoveryStrategy(cc, name, this);
recoveryStrat.setRecoveringAfterStartup(recoveringAfterStartup);
recoveryStrat.start();
recoveryRunning = true;
@@ -229,12 +242,16 @@ public final class DefaultSolrCoreState
@Override
public void cancelRecovery() {
synchronized (recoveryLock) {
- if (recoveryStrat != null) {
+ if (recoveryStrat != null && recoveryRunning) {
recoveryStrat.close();
- try {
- recoveryStrat.join();
- } catch (InterruptedException e) {
-
+ while (true) {
+ try {
+ recoveryStrat.join();
+ } catch (InterruptedException e) {
+ // not interruptible - keep waiting
+ continue;
+ }
+ break;
}
recoveryRunning = false;
@@ -242,5 +259,15 @@ public final class DefaultSolrCoreState
}
}
}
+
+ @Override
+ public void recovered() {
+ recoveryRunning = false;
+ }
+
+ @Override
+ public void failed() {
+ recoveryRunning = false;
+ }
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Fri Sep 21 17:21:34 2012
@@ -738,6 +738,13 @@ public class DirectUpdateHandler2 extend
}
}
+ @Override
+ public void split(SplitIndexCommand cmd) throws IOException {
+ // TODO: do a commit first?
+ SolrIndexSplitter splitter = new SolrIndexSplitter(cmd);
+ splitter.split();
+ }
+
/////////////////////////////////////////////////////////////////////
// SolrInfoMBean stuff: Statistics and Module Info
/////////////////////////////////////////////////////////////////////
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java Fri Sep 21 17:21:34 2012
@@ -22,7 +22,10 @@ import java.util.HashMap;
import java.util.List;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.StorableField;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
@@ -56,7 +59,7 @@ public class DocumentBuilder {
// might actually want to map it to something. If createField()
// returns null, then we don't store the field.
if (sfield.isPolyField()) {
- IndexableField[] fields = sfield.createFields(val, boost);
+ StorableField[] fields = sfield.createFields(val, boost);
if (fields.length > 0) {
if (!sfield.multiValued()) {
String oldValue = map.put(sfield.getName(), val);
@@ -66,12 +69,12 @@ public class DocumentBuilder {
}
}
// Add each field
- for (IndexableField field : fields) {
- doc.add(field);
+ for (StorableField field : fields) {
+ doc.add((Field) field);
}
}
} else {
- IndexableField field = sfield.createField(val, boost);
+ StorableField field = sfield.createField(val, boost);
if (field != null) {
if (!sfield.multiValued()) {
String oldValue = map.put(sfield.getName(), val);
@@ -81,7 +84,7 @@ public class DocumentBuilder {
}
}
}
- doc.add(field);
+ doc.add((Field) field);
}
}
@@ -190,13 +193,13 @@ public class DocumentBuilder {
private static void addField(Document doc, SchemaField field, Object val, float boost) {
if (field.isPolyField()) {
- IndexableField[] farr = field.getType().createFields(field, val, boost);
- for (IndexableField f : farr) {
- if (f != null) doc.add(f); // null fields are not added
+ StorableField[] farr = field.getType().createFields(field, val, boost);
+ for (StorableField f : farr) {
+ if (f != null) doc.add((Field) f); // null fields are not added
}
} else {
- IndexableField f = field.createField(val, boost);
- if (f != null) doc.add(f); // null fields are not added
+ StorableField f = field.createField(val, boost);
+ if (f != null) doc.add((Field) f); // null fields are not added
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/PeerSync.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/PeerSync.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/PeerSync.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/PeerSync.java Fri Sep 21 17:21:34 2012
@@ -46,7 +46,6 @@ import org.apache.solr.handler.component
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributedUpdateProcessorFactory;
import org.apache.solr.update.processor.RunUpdateProcessorFactory;
import org.apache.solr.update.processor.UpdateRequestProcessor;
@@ -78,6 +77,7 @@ public class PeerSync {
private Set<Long> requestedUpdateSet;
private long ourLowThreshold; // 20th percentile
private long ourHighThreshold; // 80th percentile
+ private boolean cantReachIsSuccess;
private static final HttpClient client;
static {
ModifiableSolrParams params = new ModifiableSolrParams();
@@ -127,18 +127,21 @@ public class PeerSync {
Exception updateException;
}
-
+ public PeerSync(SolrCore core, List<String> replicas, int nUpdates) {
+ this(core, replicas, nUpdates, false);
+ }
+
/**
*
* @param core
* @param replicas
* @param nUpdates
*/
- public PeerSync(SolrCore core, List<String> replicas, int nUpdates) {
+ public PeerSync(SolrCore core, List<String> replicas, int nUpdates, boolean cantReachIsSuccess) {
this.replicas = replicas;
this.nUpdates = nUpdates;
this.maxUpdates = nUpdates;
-
+ this.cantReachIsSuccess = cantReachIsSuccess;
uhandler = core.getUpdateHandler();
@@ -214,6 +217,7 @@ public class PeerSync {
if (startingVersions != null) {
if (startingVersions.size() == 0) {
// no frame of reference to tell of we've missed updates
+ log.warn("no frame of reference to tell of we've missed updates");
return false;
}
Collections.sort(startingVersions, absComparator);
@@ -298,20 +302,25 @@ public class PeerSync {
// If the replica went down between asking for versions and asking for specific updates, that
// shouldn't be treated as success since we counted on getting those updates back (and avoided
// redundantly asking other replicas for them).
- if (sreq.purpose == 1 && srsp.getException() instanceof SolrServerException) {
+ if (cantReachIsSuccess && sreq.purpose == 1 && srsp.getException() instanceof SolrServerException) {
Throwable solrException = ((SolrServerException) srsp.getException())
.getRootCause();
if (solrException instanceof ConnectException
|| solrException instanceof NoHttpResponseException) {
- log.info(msg() + " couldn't connect to " + srsp.getShardAddress() + ", counting as success");
+ log.warn(msg() + " couldn't connect to " + srsp.getShardAddress() + ", counting as success");
return true;
}
}
+
+ if (cantReachIsSuccess && sreq.purpose == 1 && srsp.getException() instanceof SolrException && ((SolrException) srsp.getException()).code() == 503) {
+ log.warn(msg() + " got a 503 from " + srsp.getShardAddress() + ", counting as success");
+ return true;
+ }
// TODO: at least log???
// srsp.getException().printStackTrace(System.out);
-
- log.warn(msg() + " exception talking to " + srsp.getShardAddress() + ", counting as success");
+
+ log.warn(msg() + " exception talking to " + srsp.getShardAddress() + ", failed", srsp.getException());
return false;
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java Fri Sep 21 17:21:34 2012
@@ -29,9 +29,8 @@ import java.util.concurrent.CompletionSe
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
-import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrServerException;
@@ -40,24 +39,19 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequestExt;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.util.AdjustableSemaphore;
-import org.apache.solr.util.DefaultSolrThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SolrCmdDistributor {
- private static final int MAX_RETRIES_ON_FORWARD = 6;
+ private static final int MAX_RETRIES_ON_FORWARD = 10;
public static Logger log = LoggerFactory.getLogger(SolrCmdDistributor.class);
-
- // TODO: shut this thing down
- static ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 5,
- TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
- new DefaultSolrThreadFactory("cmdDistribExecutor"));;
static final HttpClient client;
static AdjustableSemaphore semaphore = new AdjustableSemaphore(8);
@@ -90,15 +84,18 @@ public class SolrCmdDistributor {
ModifiableSolrParams params;
}
- public SolrCmdDistributor(int numHosts) {
- int maxPermits = Math.max(8, (numHosts - 1) * 8);
-
+ public static interface AbortCheck {
+ public boolean abortCheck();
+ }
+
+ public SolrCmdDistributor(int numHosts, ThreadPoolExecutor executor) {
+ int maxPermits = Math.max(16, numHosts * 16);
// limits how many tasks can actually execute at once
if (maxPermits != semaphore.getMaxPermits()) {
semaphore.setMaxPermits(maxPermits);
}
- completionService = new ExecutorCompletionService<Request>(commExecutor);
+ completionService = new ExecutorCompletionService<Request>(executor);
pending = new HashSet<Future<Request>>();
}
@@ -166,6 +163,8 @@ public class SolrCmdDistributor {
addCommit(ureq, cmd);
+ log.info("Distrib commit to:" + nodes);
+
for (Node node : nodes) {
submit(ureq, node);
}
@@ -203,7 +202,7 @@ public class SolrCmdDistributor {
void addCommit(UpdateRequestExt ureq, CommitUpdateCommand cmd) {
if (cmd == null) return;
ureq.setAction(cmd.optimize ? AbstractUpdateRequest.ACTION.OPTIMIZE
- : AbstractUpdateRequest.ACTION.COMMIT, false, cmd.waitSearcher);
+ : AbstractUpdateRequest.ACTION.COMMIT, false, cmd.waitSearcher, cmd.maxOptimizeSegments, cmd.softCommit, cmd.expungeDeletes);
}
boolean flushAdds(int limit) {
@@ -310,12 +309,13 @@ public class SolrCmdDistributor {
Callable<Request> task = new Callable<Request>() {
@Override
public Request call() throws Exception {
- Request clonedRequest = new Request();
- clonedRequest.node = sreq.node;
- clonedRequest.ureq = sreq.ureq;
- clonedRequest.retries = sreq.retries;
-
+ Request clonedRequest = null;
try {
+ clonedRequest = new Request();
+ clonedRequest.node = sreq.node;
+ clonedRequest.ureq = sreq.ureq;
+ clonedRequest.retries = sreq.retries;
+
String fullUrl;
if (!url.startsWith("http://") && !url.startsWith("https://")) {
fullUrl = "http://" + url;
@@ -326,6 +326,12 @@ public class SolrCmdDistributor {
HttpSolrServer server = new HttpSolrServer(fullUrl,
client);
+ if (Thread.currentThread().isInterrupted()) {
+ clonedRequest.rspCode = 503;
+ clonedRequest.exception = new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Shutting down.");
+ return clonedRequest;
+ }
+
clonedRequest.ursp = server.request(clonedRequest.ureq);
// currently no way to get the request body.
@@ -345,9 +351,15 @@ public class SolrCmdDistributor {
try {
semaphore.acquire();
} catch (InterruptedException e) {
- throw new RuntimeException();
+ Thread.currentThread().interrupt();
+ throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Update thread interrupted", e);
+ }
+ try {
+ pending.add(completionService.submit(task));
+ } catch (RejectedExecutionException e) {
+ semaphore.release();
+ throw e;
}
- pending.add(completionService.submit(task));
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java Fri Sep 21 17:21:34 2012
@@ -54,20 +54,38 @@ public class SolrIndexWriter extends Ind
String name;
private DirectoryFactory directoryFactory;
- public SolrIndexWriter(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec, boolean forceNewDirectory) throws IOException {
- super(
- directoryFactory.get(path, config.lockType, forceNewDirectory),
- config.toIndexWriterConfig(schema).
- setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND).
- setIndexDeletionPolicy(delPolicy).setCodec(codec).setInfoStream(toInfoStream(config))
- );
+ public static SolrIndexWriter create(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec, boolean forceNewDirectory) throws IOException {
+
+ SolrIndexWriter w = null;
+ final Directory d = directoryFactory.get(path, config.lockType, forceNewDirectory);
+ try {
+ w = new SolrIndexWriter(name, path, d, create, schema,
+ config, delPolicy, codec, forceNewDirectory);
+ w.setDirectoryFactory(directoryFactory);
+ return w;
+ } finally {
+ if (null == w && null != d) {
+ directoryFactory.doneWithDirectory(d);
+ directoryFactory.release(d);
+ }
+ }
+ }
+
+ private SolrIndexWriter(String name, String path, Directory directory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec, boolean forceNewDirectory) throws IOException {
+ super(directory,
+ config.toIndexWriterConfig(schema).
+ setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND).
+ setIndexDeletionPolicy(delPolicy).setCodec(codec).setInfoStream(toInfoStream(config))
+ );
log.debug("Opened Writer " + name);
this.name = name;
-
- this.directoryFactory = directoryFactory;
numOpens.incrementAndGet();
}
+ private void setDirectoryFactory(DirectoryFactory factory) {
+ this.directoryFactory = factory;
+ }
+
private static InfoStream toInfoStream(SolrIndexConfig config) throws IOException {
String infoStreamFile = config.infoStreamFile;
if (infoStreamFile != null) {
@@ -141,6 +159,8 @@ public class SolrIndexWriter extends Ind
super.rollback();
} finally {
isClosed = true;
+ directoryFactory.release(getDirectory());
+ numCloses.incrementAndGet();
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/TransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/TransactionLog.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/TransactionLog.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/TransactionLog.java Fri Sep 21 17:21:34 2012
@@ -141,6 +141,7 @@ public class TransactionLog {
}
TransactionLog(File tlogFile, Collection<String> globalStrings, boolean openExisting) {
+ boolean success = false;
try {
if (debug) {
log.debug("New TransactionLog file=" + tlogFile + ", exists=" + tlogFile.exists() + ", size=" + tlogFile.length() + ", openExisting=" + openExisting);
@@ -175,8 +176,18 @@ public class TransactionLog {
addGlobalStrings(globalStrings);
}
+ success = true;
+
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } finally {
+ if (!success && raf != null) {
+ try {
+ raf.close();
+ } catch (Exception e) {
+ log.error("Error closing tlog file (after error opening)", e);
+ }
+ }
}
}
@@ -775,31 +786,3 @@ class ChannelFastInputStream extends Fas
}
-class MemOutputStream extends FastOutputStream {
- public List<byte[]> buffers = new LinkedList<byte[]>();
- public MemOutputStream(byte[] tempBuffer) {
- super(null, tempBuffer, 0);
- }
-
- @Override
- public void flush(byte[] arr, int offset, int len) throws IOException {
- if (arr == buf && offset==0 && len==buf.length) {
- buffers.add(buf); // steal the buffer
- buf = new byte[8192];
- } else if (len > 0) {
- byte[] newBuf = new byte[len];
- System.arraycopy(arr, offset, newBuf, 0, len);
- buffers.add(newBuf);
- }
- }
-
- public void writeAll(FastOutputStream fos) throws IOException {
- for (byte[] buffer : buffers) {
- fos.write(buffer);
- }
- if (pos > 0) {
- fos.write(buf, 0, pos);
- }
- }
-}
-
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/UpdateHandler.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/UpdateHandler.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/UpdateHandler.java Fri Sep 21 17:21:34 2012
@@ -179,4 +179,6 @@ public abstract class UpdateHandler impl
{
optimizeCallbacks.add( listener );
}
+
+ public abstract void split(SplitIndexCommand cmd) throws IOException;
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/UpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/UpdateLog.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/UpdateLog.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/UpdateLog.java Fri Sep 21 17:21:34 2012
@@ -17,9 +17,9 @@
package org.apache.solr.update;
-import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
@@ -31,6 +31,7 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributedUpdateProcessorFactory;
import org.apache.solr.update.processor.RunUpdateProcessorFactory;
import org.apache.solr.update.processor.UpdateRequestProcessor;
@@ -60,7 +61,19 @@ public class UpdateLog implements Plugin
public boolean trace = log.isTraceEnabled();
- public enum SyncLevel { NONE, FLUSH, FSYNC }
+ public enum SyncLevel { NONE, FLUSH, FSYNC;
+ public static SyncLevel getSyncLevel(String level){
+ if (level == null) {
+ return SyncLevel.FLUSH;
+ }
+ try{
+ return SyncLevel.valueOf(level.toUpperCase(Locale.ROOT));
+ } catch(Exception ex){
+ log.warn("There was an error reading the SyncLevel - default to " + SyncLevel.FLUSH, ex);
+ return SyncLevel.FLUSH;
+ }
+ }
+ }
public enum State { REPLAYING, BUFFERING, APPLYING_BUFFERED, ACTIVE }
public static final int ADD = 0x01;
@@ -82,6 +95,8 @@ public class UpdateLog implements Plugin
public int deleteByQuery;
public int errors;
+ public boolean failed;
+
@Override
public String toString() {
return "RecoveryInfo{adds="+adds+" deletes="+deletes+ " deleteByQuery="+deleteByQuery+" errors="+errors + " positionOfStart="+positionOfStart+"}";
@@ -164,6 +179,7 @@ public class UpdateLog implements Plugin
public void init(PluginInfo info) {
dataDir = (String)info.initArgs.get("dir");
+ defaultSyncLevel = SyncLevel.getSyncLevel((String)info.initArgs.get("syncLevel"));
}
public void init(UpdateHandler uhandler, SolrCore core) {
@@ -201,7 +217,7 @@ public class UpdateLog implements Plugin
addOldLog(oldLog, false); // don't remove old logs on startup since more than one may be uncapped.
} catch (Exception e) {
SolrException.log(log, "Failure to open existing log file (non fatal) " + f, e);
- f.delete();
+ deleteFile(f);
}
}
@@ -211,8 +227,14 @@ public class UpdateLog implements Plugin
newestLogsOnStartup.addFirst(ll);
if (newestLogsOnStartup.size() >= 2) break;
}
-
- versionInfo = new VersionInfo(this, 256);
+
+ try {
+ versionInfo = new VersionInfo(this, 256);
+ } catch (SolrException e) {
+ log.error("Unable to use updateLog: " + e.getMessage(), e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Unable to use updateLog: " + e.getMessage(), e);
+ }
// TODO: these startingVersions assume that we successfully recover from all non-complete tlogs.
UpdateLog.RecentUpdates startingUpdates = getRecentUpdates();
@@ -1111,6 +1133,7 @@ public class UpdateLog implements Plugin
public void run() {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(DISTRIB_UPDATE_PARAM, FROMLEADER.toString());
+ params.set(DistributedUpdateProcessor.LOG_REPLAY, "true");
req = new LocalSolrQueryRequest(uhandler.core, params);
rsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); // setting request info will help logging
@@ -1119,9 +1142,17 @@ public class UpdateLog implements Plugin
for (TransactionLog translog : translogs) {
doReplay(translog);
}
+ } catch (SolrException e) {
+ if (e.code() == ErrorCode.SERVICE_UNAVAILABLE.code) {
+ SolrException.log(log, e);
+ recoveryInfo.failed = true;
+ } else {
+ recoveryInfo.errors++;
+ SolrException.log(log, e);
+ }
} catch (Throwable e) {
recoveryInfo.errors++;
- SolrException.log(log,e);
+ SolrException.log(log, e);
} finally {
// change the state while updates are still blocked to prevent races
state = State.ACTIVE;
@@ -1269,6 +1300,13 @@ public class UpdateLog implements Plugin
recoveryInfo.errors++;
loglog.warn("REPLAY_ERR: Unexpected log entry or corrupt log. Entry=" + o, cl);
// would be caused by a corrupt transaction log
+ } catch (SolrException ex) {
+ if (ex.code() == ErrorCode.SERVICE_UNAVAILABLE.code) {
+ throw ex;
+ }
+ recoveryInfo.errors++;
+ loglog.warn("REYPLAY_ERR: IOException reading log", ex);
+ // could be caused by an incomplete flush if recovering from log
} catch (Throwable ex) {
recoveryInfo.errors++;
loglog.warn("REPLAY_ERR: Exception replaying log", ex);
@@ -1320,6 +1358,26 @@ public class UpdateLog implements Plugin
Integer.MAX_VALUE, 1, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("recoveryExecutor"));
+
+ public static void deleteFile(File file) {
+ boolean success = false;
+ try {
+ success = file.delete();
+ if (!success) {
+ log.error("Error deleting file: " + file);
+ }
+ } catch (Exception e) {
+ log.error("Error deleting file: " + file, e);
+ }
+
+ if (!success) {
+ try {
+ file.deleteOnExit();
+ } catch (Exception e) {
+ log.error("Error deleting file on exit: " + file, e);
+ }
+ }
+ }
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/VersionInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/VersionInfo.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/VersionInfo.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/VersionInfo.java Fri Sep 21 17:21:34 2012
@@ -28,6 +28,7 @@ import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.RefCounted;
@@ -41,10 +42,45 @@ public class VersionInfo {
private SchemaField idField;
final ReadWriteLock lock = new ReentrantReadWriteLock(true);
+ /**
+ * Gets and returns the {@link #VERSION_FIELD} from the specified
+ * schema, after verifying that it is indexed, stored, and single-valued.
+ * If any of these pre-conditions are not met, it throws a SolrException
+ * with a user suitable message indicating the problem.
+ */
+ public static SchemaField getAndCheckVersionField(IndexSchema schema)
+ throws SolrException {
+ final String errPrefix = VERSION_FIELD + "field must exist in schema, using indexed=\"true\" stored=\"true\" and multiValued=\"false\"";
+ SchemaField sf = schema.getFieldOrNull(VERSION_FIELD);
+
+ if (null == sf) {
+ throw new SolrException
+ (SolrException.ErrorCode.SERVER_ERROR,
+ errPrefix + " (" + VERSION_FIELD + " does not exist)");
+ }
+ if ( !sf.indexed() ) {
+ throw new SolrException
+ (SolrException.ErrorCode.SERVER_ERROR,
+ errPrefix + " (" + VERSION_FIELD + " is not indexed");
+ }
+ if ( !sf.stored() ) {
+ throw new SolrException
+ (SolrException.ErrorCode.SERVER_ERROR,
+ errPrefix + " (" + VERSION_FIELD + " is not stored");
+ }
+ if ( sf.multiValued() ) {
+ throw new SolrException
+ (SolrException.ErrorCode.SERVER_ERROR,
+ errPrefix + " (" + VERSION_FIELD + " is not multiValued");
+ }
+
+ return sf;
+ }
+
public VersionInfo(UpdateLog ulog, int nBuckets) {
this.ulog = ulog;
SolrCore core = ulog.uhandler.core;
- versionField = core.getSchema().getFieldOrNull(VERSION_FIELD);
+ versionField = getAndCheckVersionField(core.getSchema());
idField = core.getSchema().getUniqueKeyField();
buckets = new VersionBucket[ BitUtil.nextHighestPowerOfTwo(nBuckets) ];
for (int i=0; i<buckets.length; i++) {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java?rev=1388574&r1=1388573&r2=1388574&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java Fri Sep 21 17:21:34 2012
@@ -53,9 +53,9 @@ import org.slf4j.LoggerFactory;
/**
* Clones the values found in any matching <code>source</code> field into
- * the configured <code>dest<code> field.
+ * the configured <code>dest</code> field.
* <p>
- * While the <code>dest<code> field must be a single <code><str></code>,
+ * While the <code>dest</code> field must be a single <code><str></code>,
* the <code>source</code> fields can be configured as either:
* </p>
* <ul>