You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/01/29 12:14:56 UTC
svn commit: r1562403 [4/7] - in /lucene/dev/branches/lucene5376: ./
dev-tools/ dev-tools/idea/lucene/suggest/
dev-tools/idea/solr/core/src/java/ lucene/ lucene/analysis/
lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis...
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/simple-Queries-BBox.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/simple-Queries-BBox.txt?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/simple-Queries-BBox.txt (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/simple-Queries-BBox.txt Wed Jan 29 11:14:53 2014
@@ -1,13 +1,8 @@
-C5 @ IsWithin(-6 -6 6 6)
-C5 @ BBoxWithin(-6 -6 6 6)
-C10 @ Contains(-6 -6 6 6)
-C10 @ IsEqualTo(-10 -10 10 10)
-C5 C10 @ Intersects(-2 -2 2 2)
-C5 C10 @ Overlaps(-2 -2 2 2)
-C5 C10 @ BBoxIntersects(-2 -2 2 2)
-NW15 @ IsDisjointTo(-10 -10 10 10)
-
-
-
-
-
+C5 @ IsWithin(ENVELOPE(-6, 6, 6, -6))
+C5 @ BBoxWithin(ENVELOPE(-6, 6, 6, -6))
+C10 @ Contains(ENVELOPE(-6, 6, 6, -6))
+C10 @ IsEqualTo(ENVELOPE(-10, 10, 10, -10))
+C5 C10 @ Intersects(ENVELOPE(-2, 2, 2, -2))
+C5 C10 @ Overlaps(ENVELOPE(-2, 2, 2, -2))
+C5 C10 @ BBoxIntersects(ENVELOPE(-2, 2, 2, -2))
+NW15 @ IsDisjointTo(ENVELOPE(-10, 10, 10, -10))
\ No newline at end of file
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/states-Intersects-BBox.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/states-Intersects-BBox.txt?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/states-Intersects-BBox.txt (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/states-Intersects-BBox.txt Wed Jan 29 11:14:53 2014
@@ -1,3 +1,3 @@
-WY CO @ Intersects(-106.964844 39.460938 -105.734375 42.800781)
-TX @ Intersects(-99.669922 30.583984 -98.439453 32.253906)
-MS TX LA @ Intersects(-95.363281 29.792969 -90.133789 32.473633)
+WY CO @ Intersects(ENVELOPE(-106.964844, -105.734375, 42.800781, 39.460938))
+TX @ Intersects(ENVELOPE(-99.669922, -98.439453, 32.253906, 30.583984))
+MS TX LA @ Intersects(ENVELOPE(-95.363281, -90.133789, 32.473633, 29.792969))
\ No newline at end of file
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/states-IsWithin-BBox.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/states-IsWithin-BBox.txt?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/states-IsWithin-BBox.txt (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test-files/states-IsWithin-BBox.txt Wed Jan 29 11:14:53 2014
@@ -1,4 +1,4 @@
-KS @ IsWithin(-103.493164 36.208984 -93.825195 41.086914)
-WA @ IsWithin(-126.916016 44.36084 -115.314453 50.688965)
-MA CT RI @ IsWithin(-73.894043 40.825195 -69.521484 43.198242)
-AL GA @ IsWithin(-89.472656 29.311523 -80.244141 35.90332)
\ No newline at end of file
+KS @ IsWithin(ENVELOPE(-103.493164, -93.825195, 41.086914, 36.208984))
+WA @ IsWithin(ENVELOPE(-126.916016, -115.314453, 50.688965, 44.36084))
+MA CT RI @ IsWithin(ENVELOPE(-73.894043, -69.521484, 43.198242, 40.825195))
+AL GA @ IsWithin(ENVELOPE(-89.472656, -80.244141, 35.90332, 29.311523))
\ No newline at end of file
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java Wed Jan 29 11:14:53 2014
@@ -93,8 +93,8 @@ public class DistanceStrategyTest extend
adoc("103", (Shape)null);//test score for nothing
commit();
//FYI distances are in docid order
- checkDistValueSource("3,4", 2.8274937f, 5.0898066f, 180f);
- checkDistValueSource("4,0", 3.6043684f, 0.9975641f, 180f);
+ checkDistValueSource(ctx.makePoint(4, 3), 2.8274937f, 5.0898066f, 180f);
+ checkDistValueSource(ctx.makePoint(0, 4), 3.6043684f, 0.9975641f, 180f);
}
@Test
@@ -120,8 +120,7 @@ public class DistanceStrategyTest extend
return super.newDoc(id, shape);
}
- void checkDistValueSource(String ptStr, float... distances) throws IOException {
- Point pt = (Point) ctx.readShape(ptStr);
+ void checkDistValueSource(Point pt, float... distances) throws IOException {
float multiplier = random().nextFloat() * 100f;
float[] dists2 = Arrays.copyOf(distances, distances.length);
for (int i = 0; i < dists2.length; i++) {
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java Wed Jan 29 11:14:53 2014
@@ -36,7 +36,6 @@ import org.apache.lucene.spatial.query.S
import org.apache.lucene.spatial.vector.PointVectorStrategy;
import org.junit.Test;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -91,21 +90,21 @@ public class PortedSolr3Test extends Str
this.strategy = strategy;
}
- private void setupDocs() throws IOException {
+ private void setupDocs() throws Exception {
super.deleteAll();
- adoc("1", "32.7693246, -79.9289094");
- adoc("2", "33.7693246, -80.9289094");
- adoc("3", "-32.7693246, 50.9289094");
- adoc("4", "-50.7693246, 60.9289094");
- adoc("5", "0,0");
- adoc("6", "0.1,0.1");
- adoc("7", "-0.1,-0.1");
- adoc("8", "0,179.9");
- adoc("9", "0,-179.9");
- adoc("10", "89.9,50");
- adoc("11", "89.9,-130");
- adoc("12", "-89.9,50");
- adoc("13", "-89.9,-130");
+ adoc("1", ctx.makePoint(-79.9289094, 32.7693246));
+ adoc("2", ctx.makePoint(-80.9289094, 33.7693246));
+ adoc("3", ctx.makePoint(50.9289094, -32.7693246));
+ adoc("4", ctx.makePoint(60.9289094, -50.7693246));
+ adoc("5", ctx.makePoint(0, 0));
+ adoc("6", ctx.makePoint(0.1, 0.1));
+ adoc("7", ctx.makePoint(-0.1, -0.1));
+ adoc("8", ctx.makePoint(179.9, 0));
+ adoc("9", ctx.makePoint(-179.9, 0));
+ adoc("10", ctx.makePoint(50, 89.9));
+ adoc("11", ctx.makePoint(-130, 89.9));
+ adoc("12", ctx.makePoint(50, -89.9));
+ adoc("13", ctx.makePoint(-130, -89.9));
commit();
}
@@ -115,53 +114,52 @@ public class PortedSolr3Test extends Str
setupDocs();
//Try some edge cases
//NOTE: 2nd arg is distance in kilometers
- checkHitsCircle("1,1", 175, 3, 5, 6, 7);
- checkHitsCircle("0,179.8", 200, 2, 8, 9);
- checkHitsCircle("89.8, 50", 200, 2, 10, 11);//this goes over the north pole
- checkHitsCircle("-89.8, 50", 200, 2, 12, 13);//this goes over the south pole
+ checkHitsCircle(ctx.makePoint(1, 1), 175, 3, 5, 6, 7);
+ checkHitsCircle(ctx.makePoint(179.8, 0), 200, 2, 8, 9);
+ checkHitsCircle(ctx.makePoint(50, 89.8), 200, 2, 10, 11);//this goes over the north pole
+ checkHitsCircle(ctx.makePoint(50, -89.8), 200, 2, 12, 13);//this goes over the south pole
//try some normal cases
- checkHitsCircle("33.0,-80.0", 300, 2);
+ checkHitsCircle(ctx.makePoint(-80.0, 33.0), 300, 2);
//large distance
- checkHitsCircle("1,1", 5000, 3, 5, 6, 7);
+ checkHitsCircle(ctx.makePoint(1, 1), 5000, 3, 5, 6, 7);
//Because we are generating a box based on the west/east longitudes and the south/north latitudes, which then
//translates to a range query, which is slightly more inclusive. Thus, even though 0.0 is 15.725 kms away,
//it will be included, b/c of the box calculation.
- checkHitsBBox("0.1,0.1", 15, 2, 5, 6);
+ checkHitsBBox(ctx.makePoint(0.1, 0.1), 15, 2, 5, 6);
//try some more
deleteAll();
- adoc("14", "0,5");
- adoc("15", "0,15");
+ adoc("14", ctx.makePoint(5, 0));
+ adoc("15", ctx.makePoint(15, 0));
//3000KM from 0,0, see http://www.movable-type.co.uk/scripts/latlong.html
- adoc("16", "18.71111,19.79750");
- adoc("17", "44.043900,-95.436643");
+ adoc("16", ctx.makePoint(19.79750, 18.71111));
+ adoc("17", ctx.makePoint(-95.436643, 44.043900));
commit();
- checkHitsCircle("0,0", 1000, 1, 14);
- checkHitsCircle("0,0", 2000, 2, 14, 15);
- checkHitsBBox("0,0", 3000, 3, 14, 15, 16);
- checkHitsCircle("0,0", 3001, 3, 14, 15, 16);
- checkHitsCircle("0,0", 3000.1, 3, 14, 15, 16);
+ checkHitsCircle(ctx.makePoint(0, 0), 1000, 1, 14);
+ checkHitsCircle(ctx.makePoint(0, 0), 2000, 2, 14, 15);
+ checkHitsBBox(ctx.makePoint(0, 0), 3000, 3, 14, 15, 16);
+ checkHitsCircle(ctx.makePoint(0, 0), 3001, 3, 14, 15, 16);
+ checkHitsCircle(ctx.makePoint(0, 0), 3000.1, 3, 14, 15, 16);
//really fine grained distance and reflects some of the vagaries of how we are calculating the box
- checkHitsCircle("43.517030,-96.789603", 109, 0);
+ checkHitsCircle(ctx.makePoint(-96.789603, 43.517030), 109, 0);
// falls outside of the real distance, but inside the bounding box
- checkHitsCircle("43.517030,-96.789603", 110, 0);
- checkHitsBBox("43.517030,-96.789603", 110, 1, 17);
+ checkHitsCircle(ctx.makePoint(-96.789603, 43.517030), 110, 0);
+ checkHitsBBox(ctx.makePoint(-96.789603, 43.517030), 110, 1, 17);
}
//---- these are similar to Solr test methods
- private void checkHitsCircle(String ptStr, double distKM, int assertNumFound, int... assertIds) {
- _checkHits(false, ptStr, distKM, assertNumFound, assertIds);
+ private void checkHitsCircle(Point pt, double distKM, int assertNumFound, int... assertIds) {
+ _checkHits(false, pt, distKM, assertNumFound, assertIds);
}
- private void checkHitsBBox(String ptStr, double distKM, int assertNumFound, int... assertIds) {
- _checkHits(true, ptStr, distKM, assertNumFound, assertIds);
+ private void checkHitsBBox(Point pt, double distKM, int assertNumFound, int... assertIds) {
+ _checkHits(true, pt, distKM, assertNumFound, assertIds);
}
- private void _checkHits(boolean bbox, String ptStr, double distKM, int assertNumFound, int... assertIds) {
+ private void _checkHits(boolean bbox, Point pt, double distKM, int assertNumFound, int... assertIds) {
SpatialOperation op = SpatialOperation.Intersects;
- Point pt = (Point) ctx.readShape(ptStr);
double distDEG = DistanceUtils.dist2Degrees(distKM, DistanceUtils.EARTH_MEAN_RADIUS_KM);
Shape shape = ctx.makeCircle(pt, distDEG);
if (bbox)
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java Wed Jan 29 11:14:53 2014
@@ -57,11 +57,11 @@ public class SpatialExample extends Luce
//Note: Test invoked via TestTestFramework.spatialExample()
- public static void main(String[] args) throws IOException {
+ public static void main(String[] args) throws Exception {
new SpatialExample().test();
}
- public void test() throws IOException {
+ public void test() throws Exception {
init();
indexPoints();
search();
@@ -102,7 +102,7 @@ public class SpatialExample extends Luce
this.directory = new RAMDirectory();
}
- private void indexPoints() throws IOException {
+ private void indexPoints() throws Exception {
IndexWriterConfig iwConfig = new IndexWriterConfig(TEST_VERSION_CURRENT,null);
IndexWriter indexWriter = new IndexWriter(directory, iwConfig);
@@ -110,10 +110,9 @@ public class SpatialExample extends Luce
indexWriter.addDocument(newSampleDocument(
2, ctx.makePoint(-80.93, 33.77)));
- //When parsing a string to a shape, the presence of a comma means it's y-x
- // order (lon, lat)
+ //Spatial4j has a WKT parser which is also "x y" order
indexWriter.addDocument(newSampleDocument(
- 4, ctx.readShape("-50.7693246, 60.9289094")));
+ 4, ctx.readShapeFromWkt("POINT(60.9289094 -50.7693246)")));
indexWriter.addDocument(newSampleDocument(
20, ctx.makePoint(0.1,0.1), ctx.makePoint(0, 0)));
@@ -131,13 +130,15 @@ public class SpatialExample extends Luce
doc.add(f);
}
//store it too; the format is up to you
- doc.add(new StoredField(strategy.getFieldName(), ctx.toString(shape)));
+ // (assume point in this example)
+ Point pt = (Point) shape;
+ doc.add(new StoredField(strategy.getFieldName(), pt.getX()+" "+pt.getY()));
}
return doc;
}
- private void search() throws IOException {
+ private void search() throws Exception {
IndexReader indexReader = DirectoryReader.open(directory);
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
Sort idSort = new Sort(new SortField("id", SortField.Type.INT));
@@ -155,15 +156,19 @@ public class SpatialExample extends Luce
// (this computation is usually not redundant)
StoredDocument doc1 = indexSearcher.doc(docs.scoreDocs[0].doc);
String doc1Str = doc1.getField(strategy.getFieldName()).stringValue();
- Point doc1Point = (Point) ctx.readShape(doc1Str);
- double doc1DistDEG = ctx.getDistCalc().distance(args.getShape().getCenter(), doc1Point);
+ //assume doc1Str is "x y" as written in newSampleDocument()
+ int spaceIdx = doc1Str.indexOf(' ');
+ double x = Double.parseDouble(doc1Str.substring(0, spaceIdx));
+ double y = Double.parseDouble(doc1Str.substring(spaceIdx+1));
+ double doc1DistDEG = ctx.calcDistance(args.getShape().getCenter(), x, y);
assertEquals(121.6d, DistanceUtils.degrees2Dist(doc1DistDEG, DistanceUtils.EARTH_MEAN_RADIUS_KM), 0.1);
+ //or more simply:
+ assertEquals(121.6d, doc1DistDEG * DistanceUtils.DEG_TO_KM, 0.1);
}
//--Match all, order by distance ascending
{
Point pt = ctx.makePoint(60, -50);
- double degToKm = DistanceUtils.degrees2Dist(1, DistanceUtils.EARTH_MEAN_RADIUS_KM);
- ValueSource valueSource = strategy.makeDistanceValueSource(pt, degToKm);//the distance (in km)
+ ValueSource valueSource = strategy.makeDistanceValueSource(pt, DistanceUtils.DEG_TO_KM);//the distance (in km)
Sort distSort = new Sort(valueSource.getSortField(false)).rewrite(indexSearcher);//false=asc dist
TopDocs docs = indexSearcher.search(new MatchAllDocsQuery(), 10, distSort);
assertDocMatchedIds(indexSearcher, docs, 4, 20, 2);
@@ -178,7 +183,7 @@ public class SpatialExample extends Luce
{
SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects,
ctx.makeCircle(-80.0, 33.0, 1));
- SpatialArgs args2 = new SpatialArgsParser().parse("Intersects(Circle(33,-80 d=1))", ctx);
+ SpatialArgs args2 = new SpatialArgsParser().parse("Intersects(BUFFER(POINT(-80 33),1))", ctx);
assertEquals(args.toString(),args2.toString());
}
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java Wed Jan 29 11:14:53 2014
@@ -18,10 +18,10 @@ package org.apache.lucene.spatial;
*/
import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.io.LineReader;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialArgsParser;
+import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -41,20 +41,23 @@ public class SpatialTestQuery {
public List<String> ids = new ArrayList<String>();
/**
- * Get Test Queries
+ * Get Test Queries. The InputStream is closed.
*/
public static Iterator<SpatialTestQuery> getTestQueries(
final SpatialArgsParser parser,
final SpatialContext ctx,
final String name,
final InputStream in ) throws IOException {
- return new LineReader<SpatialTestQuery>(new InputStreamReader(in,"UTF-8")) {
- @Override
- public SpatialTestQuery parseLine(String line) {
+ List<SpatialTestQuery> results = new ArrayList<SpatialTestQuery>();
+
+ BufferedReader bufInput = new BufferedReader(new InputStreamReader(in,"UTF-8"));
+ try {
+ String line;
+ for (int lineNumber = 1; (line = bufInput.readLine()) != null; lineNumber++) {
SpatialTestQuery test = new SpatialTestQuery();
test.line = line;
- test.lineNumber = getLineNumber();
+ test.lineNumber = lineNumber;
try {
// skip a comment
@@ -71,13 +74,16 @@ public class SpatialTestQuery {
test.ids.add(st.nextToken().trim());
}
test.args = parser.parse(line.substring(idx + 1).trim(), ctx);
- return test;
+ results.add(test);
}
catch( Exception ex ) {
throw new RuntimeException( "invalid query line: "+test.line, ex );
}
}
- };
+ } finally {
+ bufInput.close();
+ }
+ return results.iterator();
}
@Override
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java Wed Jan 29 11:14:53 2014
@@ -19,8 +19,6 @@ package org.apache.lucene.spatial;
*/
import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.io.sample.SampleData;
-import com.spatial4j.core.io.sample.SampleDataReader;
import com.spatial4j.core.shape.Shape;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -40,6 +38,7 @@ import org.apache.lucene.spatial.query.S
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -88,21 +87,21 @@ public abstract class StrategyTestCase e
return getDocuments(getSampleData(testDataFile));
}
- protected List<Document> getDocuments(Iterator<SampleData> sampleData) {
+ protected List<Document> getDocuments(Iterator<SpatialTestData> sampleData) {
List<Document> documents = new ArrayList<Document>();
while (sampleData.hasNext()) {
- SampleData data = sampleData.next();
+ SpatialTestData data = sampleData.next();
Document document = new Document();
document.add(new StringField("id", data.id, Field.Store.YES));
document.add(new StringField("name", data.name, Field.Store.YES));
- Shape shape = ctx.readShape(data.shape);
+ Shape shape = data.shape;
shape = convertShapeFromGetDocuments(shape);
if (shape != null) {
for (Field f : strategy.createIndexableFields(shape)) {
document.add(f);
}
- if (storeShape)
- document.add(new StoredField(strategy.getFieldName(), ctx.toString(shape)));
+ if (storeShape)//just for diagnostics
+ document.add(new StoredField(strategy.getFieldName(), shape.toString()));
}
documents.add(document);
@@ -115,18 +114,18 @@ public abstract class StrategyTestCase e
return shape;
}
- protected Iterator<SampleData> getSampleData(String testDataFile) throws IOException {
+ protected Iterator<SpatialTestData> getSampleData(String testDataFile) throws IOException {
String path = "data/" + testDataFile;
InputStream stream = getClass().getClassLoader().getResourceAsStream(path);
if (stream == null)
throw new FileNotFoundException("classpath resource not found: "+path);
- return new SampleDataReader(stream);
+ return SpatialTestData.getTestData(stream, ctx);//closes the InputStream
}
protected Iterator<SpatialTestQuery> getTestQueries(String testQueryFile, SpatialContext ctx) throws IOException {
InputStream in = getClass().getClassLoader().getResourceAsStream(testQueryFile);
return SpatialTestQuery.getTestQueries(
- argsParser, ctx, testQueryFile, in );
+ argsParser, ctx, testQueryFile, in );//closes the InputStream
}
public void runTestQueries(
@@ -142,8 +141,8 @@ public abstract class StrategyTestCase e
String msg = q.toString(); //"Query: " + q.args.toString(ctx);
SearchResults got = executeQuery(strategy.makeQuery(q.args), Math.max(100, q.ids.size()+1));
if (storeShape && got.numFound > 0) {
- //check stored value is there & parses
- assertNotNull(ctx.readShape(got.results.get(0).document.get(strategy.getFieldName())));
+ //check stored value is there
+ assertNotNull(got.results.get(0).document.get(strategy.getFieldName()));
}
if (concern.orderIsImportant) {
Iterator<String> ids = q.ids.iterator();
@@ -184,8 +183,8 @@ public abstract class StrategyTestCase e
}
}
- protected void adoc(String id, String shapeStr) throws IOException {
- Shape shape = shapeStr==null ? null : ctx.readShape(shapeStr);
+ protected void adoc(String id, String shapeStr) throws IOException, ParseException {
+ Shape shape = shapeStr==null ? null : ctx.readShapeFromWkt(shapeStr);
addDocument(newDoc(id, shape));
}
protected void adoc(String id, Shape shape) throws IOException {
@@ -200,7 +199,7 @@ public abstract class StrategyTestCase e
doc.add(f);
}
if (storeShape)
- doc.add(new StoredField(strategy.getFieldName(), ctx.toString(shape)));
+ doc.add(new StoredField(strategy.getFieldName(), shape.toString()));//not to be parsed; just for debug
}
return doc;
}
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/TestTestFramework.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/TestTestFramework.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/TestTestFramework.java (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/TestTestFramework.java Wed Jan 29 11:14:53 2014
@@ -44,7 +44,7 @@ public class TestTestFramework extends L
InputStream in = getClass().getClassLoader().getResourceAsStream(name);
SpatialContext ctx = SpatialContext.GEO;
Iterator<SpatialTestQuery> iter = SpatialTestQuery.getTestQueries(
- new SpatialArgsParser(), ctx, name, in );
+ new SpatialArgsParser(), ctx, name, in );//closes the InputStream
List<SpatialTestQuery> tests = new ArrayList<SpatialTestQuery>();
while( iter.hasNext() ) {
tests.add( iter.next() );
@@ -52,15 +52,15 @@ public class TestTestFramework extends L
Assert.assertEquals( 3, tests.size() );
SpatialTestQuery sf = tests.get(0);
- // assert
+ // assert
Assert.assertEquals( 1, sf.ids.size() );
Assert.assertTrue( sf.ids.get(0).equals( "G5391959" ) );
Assert.assertTrue( sf.args.getShape() instanceof Rectangle);
- Assert.assertEquals( SpatialOperation.Intersects, sf.args.getOperation() );
+ Assert.assertEquals(SpatialOperation.Intersects, sf.args.getOperation());
}
@Test
- public void spatialExample() throws IOException {
+ public void spatialExample() throws Exception {
//kind of a hack so that SpatialExample is tested despite
// it not starting or ending with "Test".
SpatialExample.main(null);
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java Wed Jan 29 11:14:53 2014
@@ -35,7 +35,7 @@ import org.apache.lucene.spatial.query.S
import org.apache.lucene.spatial.query.SpatialOperation;
import org.junit.Test;
-import java.io.IOException;
+import java.text.ParseException;
import java.util.HashMap;
public class JtsPolygonTest extends StrategyTestCase {
@@ -59,7 +59,7 @@ public class JtsPolygonTest extends Stra
@Test
/** LUCENE-4464 */
- public void testCloseButNoMatch() throws IOException {
+ public void testCloseButNoMatch() throws Exception {
getAddAndVerifyIndexedDocuments("LUCENE-4464.txt");
SpatialArgs args = q(
"POLYGON((-93.18100824442227 45.25676372469945," +
@@ -73,8 +73,8 @@ public class JtsPolygonTest extends Stra
//did not find poly 1 !
}
- private SpatialArgs q(String shapeStr, double distErrPct) {
- Shape shape = ctx.readShape(shapeStr);
+ private SpatialArgs q(String shapeStr, double distErrPct) throws ParseException {
+ Shape shape = ctx.readShapeFromWkt(shapeStr);
SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, shape);
args.setDistErrPct(distErrPct);
return args;
@@ -87,7 +87,7 @@ public class JtsPolygonTest extends Stra
@Test
public void testBadPrefixTreePrune() throws Exception {
- Shape area = ctx.readShape("POLYGON((-122.83 48.57, -122.77 48.56, -122.79 48.53, -122.83 48.57))");
+ Shape area = ctx.readShapeFromWkt("POLYGON((-122.83 48.57, -122.77 48.56, -122.79 48.53, -122.83 48.57))");
SpatialPrefixTree trie = new QuadPrefixTree(ctx, 12);
TermQueryPrefixTreeStrategy strategy = new TermQueryPrefixTreeStrategy(trie, "geo");
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java Wed Jan 29 11:14:53 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.spatial.prefix
import com.carrotsearch.randomizedtesting.annotations.Repeat;
import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.context.SpatialContextFactory;
import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Rectangle;
import com.spatial4j.core.shape.Shape;
@@ -71,7 +72,10 @@ public class SpatialOpRecursivePrefixTre
public void mySetup(int maxLevels) throws IOException {
//non-geospatial makes this test a little easier (in gridSnap), and using boundary values 2^X raises
// the prospect of edge conditions we want to test, plus makes for simpler numbers (no decimals).
- this.ctx = new SpatialContext(false, null, new RectangleImpl(0, 256, -128, 128, null));
+ SpatialContextFactory factory = new SpatialContextFactory();
+ factory.geo = false;
+ factory.worldBounds = new RectangleImpl(0, 256, -128, 128, null);
+ this.ctx = factory.newSpatialContext();
//A fairly shallow grid, and default 2.5% distErrPct
if (maxLevels == -1)
maxLevels = randomIntBetween(1, 8);
@@ -179,8 +183,8 @@ public class SpatialOpRecursivePrefixTre
doc.add(f);
}
}
- if (storeShape)
- doc.add(new StoredField(strategy.getFieldName(), ctx.toString(shape)));
+ if (storeShape)//just for diagnostics
+ doc.add(new StoredField(strategy.getFieldName(), shape.toString()));
}
return doc;
}
@@ -412,6 +416,16 @@ public class SpatialOpRecursivePrefixTre
}
@Override
+ public Shape getBuffered(double distance, SpatialContext ctx) {
+ throw new UnsupportedOperationException("TODO unimplemented");//TODO
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return false;
+ }
+
+ @Override
public String toString() {
return "ShapePair(" + shape1 + " , " + shape2 + ")";
}
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java Wed Jan 29 11:14:53 2014
@@ -46,7 +46,7 @@ public class TestTermQueryPrefixGridStra
for (Field field : prefixGridStrategy.createIndexableFields(point)) {
losAngeles.add(field);
}
- losAngeles.add(new StoredField(prefixGridStrategy.getFieldName(), ctx.toString(point)));
+ losAngeles.add(new StoredField(prefixGridStrategy.getFieldName(), point.toString()));//just for diagnostics
addDocumentsAndCommit(Arrays.asList(losAngeles));
Modified: lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/spatial/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java Wed Jan 29 11:14:53 2014
@@ -34,7 +34,7 @@ public class SpatialArgsParserTest exten
public void testArgsParser() throws Exception {
SpatialArgsParser parser = new SpatialArgsParser();
- String arg = SpatialOperation.IsWithin + "(-10 -20 10 20)";
+ String arg = SpatialOperation.IsWithin + "(Envelope(-10, 10, 20, -20))";
SpatialArgs out = parser.parse(arg, ctx);
assertEquals(SpatialOperation.IsWithin, out.getOperation());
Rectangle bounds = (Rectangle) out.getShape();
@@ -42,7 +42,7 @@ public class SpatialArgsParserTest exten
assertEquals(10.0, bounds.getMaxX(), 0D);
// Disjoint should not be scored
- arg = SpatialOperation.IsDisjointTo + " (-10 10 -20 20)";
+ arg = SpatialOperation.IsDisjointTo + " (Envelope(-10,-20,20,10))";
out = parser.parse(arg, ctx);
assertEquals(SpatialOperation.IsDisjointTo, out.getOperation());
@@ -54,7 +54,7 @@ public class SpatialArgsParserTest exten
}
try {
- parser.parse("XXXX(-10 10 -20 20)", ctx);
+ parser.parse("XXXX(Envelope(-10, 10, 20, -20))", ctx);
fail("unknown operation!");
}
catch (Exception ex) {//expected
Modified: lucene/dev/branches/lucene5376/lucene/suggest/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/build.xml?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/build.xml (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/build.xml Wed Jan 29 11:14:53 2014
@@ -31,13 +31,10 @@
<path id="classpath">
<pathelement path="${analyzers-common.jar}"/>
<pathelement path="${misc.jar}"/>
- <pathelement path="${expressions.jar}"/>
<pathelement path="${queries.jar}"/>
- <fileset dir="${common.dir}/expressions/lib"/>
<path refid="base.classpath"/>
</path>
-
<target name="javadocs" depends="javadocs-queries,javadocs-analyzers-common,compile-core">
<invoke-module-javadoc>
<links>
Modified: lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java (original)
+++ lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java Wed Jan 29 11:14:53 2014
@@ -41,17 +41,6 @@ public final class Lucene40RWCodec exten
}
};
- private final SegmentInfoFormat infosFormat = new Lucene40SegmentInfoFormat() {
- @Override
- public org.apache.lucene.codecs.SegmentInfoWriter getSegmentInfoWriter() {
- if (!LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE) {
- return super.getSegmentInfoWriter();
- } else {
- return new Lucene40SegmentInfoWriter();
- }
- }
- };
-
private final DocValuesFormat docValues = new Lucene40RWDocValuesFormat();
private final NormsFormat norms = new Lucene40RWNormsFormat();
@@ -70,9 +59,4 @@ public final class Lucene40RWCodec exten
return norms;
}
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return infosFormat;
- }
-
}
Modified: lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java (original)
+++ lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java Wed Jan 29 11:14:53 2014
@@ -6,14 +6,11 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosWriter;
import org.apache.lucene.codecs.lucene40.Lucene40RWDocValuesFormat;
import org.apache.lucene.codecs.lucene40.Lucene40RWNormsFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoWriter;
import org.apache.lucene.util.LuceneTestCase;
/*
@@ -53,17 +50,6 @@ public class Lucene41RWCodec extends Luc
private final DocValuesFormat docValues = new Lucene40RWDocValuesFormat();
private final NormsFormat norms = new Lucene40RWNormsFormat();
- private final SegmentInfoFormat segmentInfosFormat = new Lucene40SegmentInfoFormat() {
- @Override
- public org.apache.lucene.codecs.SegmentInfoWriter getSegmentInfoWriter() {
- if (!LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE) {
- return super.getSegmentInfoWriter();
- } else {
- return new Lucene40SegmentInfoWriter();
- }
- }
- };
-
@Override
public FieldInfosFormat fieldInfosFormat() {
return fieldInfos;
@@ -84,9 +70,4 @@ public class Lucene41RWCodec extends Luc
return norms;
}
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return segmentInfosFormat;
- }
-
}
Modified: lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java (original)
+++ lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java Wed Jan 29 11:14:53 2014
@@ -23,9 +23,6 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoWriter;
import org.apache.lucene.util.LuceneTestCase;
/**
@@ -47,17 +44,6 @@ public class Lucene42RWCodec extends Luc
}
}
};
-
- private final SegmentInfoFormat segmentInfosFormat = new Lucene40SegmentInfoFormat() {
- @Override
- public org.apache.lucene.codecs.SegmentInfoWriter getSegmentInfoWriter() {
- if (!LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE) {
- return super.getSegmentInfoWriter();
- } else {
- return new Lucene40SegmentInfoWriter();
- }
- }
- };
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
@@ -72,11 +58,5 @@ public class Lucene42RWCodec extends Luc
@Override
public FieldInfosFormat fieldInfosFormat() {
return fieldInfosFormat;
- }
-
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return segmentInfosFormat;
- }
-
+ }
}
Modified: lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java (original)
+++ lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java Wed Jan 29 11:14:53 2014
@@ -21,9 +21,6 @@ import java.io.IOException;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosWriter;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoWriter;
import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosFormat;
import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosWriter;
import org.apache.lucene.util.LuceneTestCase;
@@ -44,26 +41,9 @@ public class Lucene45RWCodec extends Luc
}
}
};
-
- private final SegmentInfoFormat segmentInfosFormat = new Lucene40SegmentInfoFormat() {
- @Override
- public org.apache.lucene.codecs.SegmentInfoWriter getSegmentInfoWriter() {
- if (!LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE) {
- return super.getSegmentInfoWriter();
- } else {
- return new Lucene40SegmentInfoWriter();
- }
- }
- };
@Override
public FieldInfosFormat fieldInfosFormat() {
return fieldInfosFormat;
}
-
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return segmentInfosFormat;
- }
-
}
Modified: lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java (original)
+++ lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java Wed Jan 29 11:14:53 2014
@@ -151,7 +151,11 @@ public class AssertingScorer extends Sco
@Override
public Collection<ChildScorer> getChildren() {
- return in.getChildren();
+ // We cannot hide that we hold a single child, else
+ // collectors (e.g. ToParentBlockJoinCollector) that
+ // need to walk the scorer tree will miss/skip the
+ // Scorer we wrap:
+ return Collections.singletonList(new ChildScorer(in, "SHOULD"));
}
@Override
Modified: lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/store/MockLockFactoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/store/MockLockFactoryWrapper.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/store/MockLockFactoryWrapper.java (original)
+++ lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/store/MockLockFactoryWrapper.java Wed Jan 29 11:14:53 2014
@@ -78,8 +78,8 @@ public class MockLockFactoryWrapper exte
}
@Override
- public void release() throws IOException {
- delegateLock.release();
+ public void close() throws IOException {
+ delegateLock.close();
dir.openLocks.remove(name);
}
Modified: lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Wed Jan 29 11:14:53 2014
@@ -334,12 +334,12 @@ public abstract class LuceneTestCase ext
/**
* When {@code true}, Codecs for old Lucene version will support writing
- * indexes in that format. Defaults to {@code true}, can be disabled by
- * spdecific tests on demand.
+ * indexes in that format. Defaults to {@code false}, can be disabled by
+ * specific tests on demand.
*
* @lucene.internal
*/
- public static boolean OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
+ public static boolean OLD_FORMAT_IMPERSONATION_IS_ACTIVE = false;
// -----------------------------------------------------------------
@@ -1365,6 +1365,18 @@ public abstract class LuceneTestCase ext
}
}
+ /** Returns true if the default codec supports single valued docvalues with missing values */
+ public static boolean defaultCodecSupportsMissingDocValues() {
+ String name = Codec.getDefault().getName();
+ if (name.equals("Lucene3x") ||
+ name.equals("Lucene40") || name.equals("Appending") ||
+ name.equals("Lucene41") ||
+ name.equals("Lucene42")) {
+ return false;
+ }
+ return true;
+ }
+
/** Returns true if the default codec supports SORTED_SET docvalues */
public static boolean defaultCodecSupportsSortedSet() {
String name = Codec.getDefault().getName();
Modified: lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java (original)
+++ lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java Wed Jan 29 11:14:53 2014
@@ -155,6 +155,9 @@ final class TestRuleSetupAndRestoreClass
avoidCodecs.addAll(Arrays.asList(a.value()));
}
+ // set back to default
+ LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = false;
+
savedCodec = Codec.getDefault();
int randomVal = random.nextInt(10);
if ("Lucene40".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) &&
@@ -163,6 +166,7 @@ final class TestRuleSetupAndRestoreClass
randomVal == 0 &&
!shouldAvoidCodec("Lucene40"))) {
codec = Codec.forName("Lucene40");
+ LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
assert codec instanceof Lucene40RWCodec : "fix your classpath to have tests-framework.jar before lucene-core.jar";
assert (PostingsFormat.forName("Lucene40") instanceof Lucene40RWPostingsFormat) : "fix your classpath to have tests-framework.jar before lucene-core.jar";
} else if ("Lucene41".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) &&
@@ -171,6 +175,7 @@ final class TestRuleSetupAndRestoreClass
randomVal == 1 &&
!shouldAvoidCodec("Lucene41"))) {
codec = Codec.forName("Lucene41");
+ LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
assert codec instanceof Lucene41RWCodec : "fix your classpath to have tests-framework.jar before lucene-core.jar";
} else if ("Lucene42".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) &&
"random".equals(TEST_POSTINGSFORMAT) &&
@@ -178,6 +183,7 @@ final class TestRuleSetupAndRestoreClass
randomVal == 2 &&
!shouldAvoidCodec("Lucene42"))) {
codec = Codec.forName("Lucene42");
+ LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
assert codec instanceof Lucene42RWCodec : "fix your classpath to have tests-framework.jar before lucene-core.jar";
} else if ("Lucene45".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) &&
"random".equals(TEST_POSTINGSFORMAT) &&
@@ -185,6 +191,7 @@ final class TestRuleSetupAndRestoreClass
randomVal == 5 &&
!shouldAvoidCodec("Lucene45"))) {
codec = Codec.forName("Lucene45");
+ LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
assert codec instanceof Lucene45RWCodec : "fix your classpath to have tests-framework.jar before lucene-core.jar";
} else if (("random".equals(TEST_POSTINGSFORMAT) == false) || ("random".equals(TEST_DOCVALUESFORMAT) == false)) {
// the user wired postings or DV: this is messy
Modified: lucene/dev/branches/lucene5376/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/CHANGES.txt?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5376/solr/CHANGES.txt Wed Jan 29 11:14:53 2014
@@ -115,8 +115,8 @@ New Features
* SOLR-5506: Support docValues in CollationField and ICUCollationField.
(Robert Muir)
- * SOLR-5518: Added EditFileRequestHandler to deal with security issues around modifying
- solr configuration files.
+* SOLR-5518: Added EditFileRequestHandler to deal with security issues around modifying
+ solr configuration files.
* SOLR-5023: Add support for deleteInstanceDir to be passed from SolrJ for Core
Unload action. (Lyubov Romanchuk, shalin)
@@ -151,6 +151,16 @@ New Features
* SOLR-5476: Overseer Role for nodes (Noble Paul)
+* SOLR-5594: Allow FieldTypes to specify custom PrefixQuery behavior
+ (Anshum Gupta via hossman)
+
+* LUCENE-5395: Upgrade to Spatial4j 0.4. Various new options are now exposed
+ automatically for an RPT field type. See Spatial4j CHANGES & javadocs.
+ https://github.com/spatial4j/spatial4j/blob/master/CHANGES.md (David Smiley)
+
+* SOLR-5670: allow _version_ to use DocValues. (Per Steffensen via yonik)
+
+
Bug Fixes
----------------------
@@ -206,13 +216,29 @@ Bug Fixes
* SOLR-4992: Solr eats OutOfMemoryError exceptions in many cases.
(Mark Miller, Daniel Collins)
-* SOLR-5636: SolrRequestParsers does some xpath lookups on every request, which
- can cause concurrency issues. (Mark Miller)
-
* LUCENE-5399, SOLR-5354 sort wouldn't work correctly with
distributed searching for some field types such as legacy numeric
types (Rob Muir, Mike McCandless)
+* SOLR-5643: ConcurrentUpdateSolrServer will sometimes not spawn a new Runner
+ thread even though there are updates in the queue. (Mark Miller)
+
+* SOLR-5650: When a replica becomes a leader, only peer sync with other replicas
+ that last published an ACTIVE state. (Mark Miller)
+
+* SOLR-5657: When a SolrCore starts on HDFS, it should gracefully handle HDFS
+ being in safe mode. (Mark Miller)
+
+* SOLR-5663: example-DIH uses non-existing column for mapping (case-sensitive)
+ (steffkes)
+
+* SOLR-5666: Using the hdfs write cache can result in appearance of corrupted
+ index. (Mark Miller)
+
+* SOLR-5230: Call DelegatingCollector.finish() during grouping.
+ (Joel Bernstein, ehatcher)
+
+
Optimizations
----------------------
@@ -282,6 +308,9 @@ Other Changes
* SOLR-5632: Improve response message for reloading a non-existent core.
(Anshum Gupta via Mark Miller)
+* SOLR-5633: HttpShardHandlerFactory should make its http client available to subclasses.
+ (Ryan Ernst)
+
================== 4.6.1 ==================
Versions of Major Components
@@ -355,7 +384,7 @@ Bug Fixes
necessary. (Mark Miller)
* SOLR-4709: The core reload after replication if config files have changed
- can fail due to a race condition. (Mark Miller, Hossman))
+ can fail due to a race condition. (Mark Miller, Hossman)
* SOLR-5503: Retry 'forward to leader' requests less aggressively - rather
than on IOException and status 500, ConnectException. (Mark Miller)
@@ -383,6 +412,16 @@ Bug Fixes
updates meant for replicas to be lost. (Markus Jelsma, Timothy Potter,
Joel Bernstein, Mark Miller)
+* SOLR-5645: A SolrCore reload via the CoreContainer will try and register in
+ zk again with the new SolrCore. (Mark Miller)
+
+* SOLR-5636: SolrRequestParsers does some xpath lookups on every request, which
+ can cause concurrency issues. (Mark Miller)
+
+* SOLR-5658: commitWithin and overwrite are not being distributed to replicas
+ now that SolrCloud uses javabin to distribute updates.
+ (Mark Miller, Varun Thacker, Elodie Sannier, shalin)
+
Optimizations
----------------------
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Wed Jan 29 11:14:53 2014
@@ -208,7 +208,7 @@ final class ShardLeaderElectionContext e
boolean success = false;
try {
- success = syncStrategy.sync(zkController, core, leaderProps);
+ success = syncStrategy.sync(zkController, core, leaderProps, weAreReplacement);
} catch (Exception e) {
SolrException.log(log, "Exception while trying to sync", e);
success = false;
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Wed Jan 29 11:14:53 2014
@@ -19,6 +19,7 @@ package org.apache.solr.cloud;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
@@ -67,6 +68,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
@@ -222,71 +224,104 @@ public class OverseerCollectionProcessor
List overseerDesignates = (List) m.get("overseer");
if(overseerDesignates==null || overseerDesignates.isEmpty()) return;
- log.debug("overseer designates {}", overseerDesignates);
-
- List<String> children = zk.getChildren(OverseerElectionContext.PATH + LeaderElector.ELECTION_NODE, null, true);
+ log.info("overseer designates {}", overseerDesignates);
+ List<String> nodeNames = getSortedNodeNames(zk);
+ if(nodeNames.size()<2) return;
- if(children.size()<2) return;
-
- LeaderElector.sortSeqs(children);
- ArrayList<String> nodeNames = new ArrayList<>(children.size());
- for (String c : children) nodeNames.add(LeaderElector.getNodeName(c));
- boolean overseerDesignateAvailable = false;
+//
+ Set<String> nodesTobePushedBack = new HashSet<String>();
//ensure that the node right behind the leader , i.r at position 1 is a Overseer
- String newOverseerDesignate = null;
- log.debug("sorted nodes {}", nodeNames);
- for (int i = 1; i < nodeNames.size(); i++) {
+ Set<String> availableDesignates = new HashSet<String>();
+
+ log.debug("sorted nodes {}", nodeNames);//TODO to be removed
+ for (int i = 0; i < nodeNames.size(); i++) {
String s = nodeNames.get(i);
+
if (overseerDesignates.contains(s)) {
- log.info(" found an overseer designate to be promoted to the front : {}, pushing others back", s);
- overseerDesignateAvailable = true;
- newOverseerDesignate = s;
- for (int j = 1; j < i; j++) {
- String nodeName = nodeNames.get(j);
- log.debug("pushing back {} ", nodeName);
- invokeRejoinOverseer(nodeName);
+ availableDesignates.add(s);
+
+ for(int j=0;j<i;j++){
+ if(!overseerDesignates.contains(nodeNames.get(j))) {
+ nodesTobePushedBack.add(nodeNames.get(j));
+ }
}
- break;
+
}
- if(overseerDesignateAvailable) break;
+ if(availableDesignates.size()>1) break;
}
- if(overseerDesignateAvailable){
+ if(!availableDesignates.isEmpty()){
+ for (String s : nodesTobePushedBack) {
+ log.info("pushing back {} ", s);
+ invokeRejoinOverseer(s);
+ }
+
//wait for a while to ensure the designate has indeed come in front
boolean prioritizationComplete = false;
- long timeout = System.currentTimeMillis() + 5000;
+ long timeout = System.currentTimeMillis() + 2500;
for(;System.currentTimeMillis()< timeout ;){
- children = zk.getChildren(OverseerElectionContext.PATH + LeaderElector.ELECTION_NODE, null, true);
- LeaderElector.sortSeqs(children);
+ List<String> currentNodeNames = getSortedNodeNames(zk);
- String frontRunner = LeaderElector.getNodeName(children.get(1));
- log.debug("Frontrunner : {}", frontRunner);
- if(newOverseerDesignate.equals(frontRunner)){
+ int totalLeaders = 0;
+
+ for(int i=0;i<availableDesignates.size();i++) {
+ if(overseerDesignates.contains(currentNodeNames.get(i))) totalLeaders++;
+ }
+ if(totalLeaders == availableDesignates.size()){
prioritizationComplete = true;
break;
}
- Thread.sleep(50);
+ try {
+ Thread.sleep(50);
+ } catch (InterruptedException e) {
+ log.warn("Thread interrupted",e);
+ break;
+
+ }
}
if(!prioritizationComplete) {
- log.warn("Could not make the Overseer designate '{}' the frontrunner", newOverseerDesignate);
+ log.warn("available designates and current state {} {} ", availableDesignates, getSortedNodeNames(zk));
}
} else {
- log.warn("No overseer designates are available, overseerDesignates: {}, nodes : ",overseerDesignates,nodeNames);
+ log.warn("No overseer designates are available, overseerDesignates: {}, live nodes : {}",overseerDesignates,nodeNames);
return;
}
-
-
- if(!overseerDesignates.contains( nodeNames.get(0)) && overseerDesignateAvailable){
+ String leaderNode = getLeaderNode(zkStateReader.getZkClient());
+ if(leaderNode ==null) return;
+ if(!overseerDesignates.contains(leaderNode) && !availableDesignates.isEmpty()){
//this means there are designated Overseer nodes and I am not one of them , kill myself
- invokeRejoinOverseer(nodeNames.get(0));
+ log.info("I am not an overseerdesignate , rejoining election {} ", leaderNode);
+ invokeRejoinOverseer(leaderNode);
}
+ }
+
+ public static List<String> getSortedNodeNames(SolrZkClient zk) throws KeeperException, InterruptedException {
+ List<String> children = zk.getChildren(OverseerElectionContext.PATH + LeaderElector.ELECTION_NODE, null, true);
+ LeaderElector.sortSeqs(children);
+ ArrayList<String> nodeNames = new ArrayList<>(children.size());
+ for (String c : children) nodeNames.add(LeaderElector.getNodeName(c));
+ return nodeNames;
+ }
+ public static String getLeaderNode(SolrZkClient zkClient) throws KeeperException, InterruptedException {
+ byte[] data = new byte[0];
+ try {
+ data = zkClient.getData("/overseer_elect/leader", null, new Stat(), true);
+ } catch (KeeperException.NoNodeException e) {
+ return null;
+ }
+ Map m = (Map) ZkStateReader.fromJSON(data);
+ String s = (String) m.get("id");
+// log.info("leader-id {}",s);
+ String nodeName = LeaderElector.getNodeName(s);
+// log.info("Leader {}", nodeName);
+ return nodeName;
}
private void invokeRejoinOverseer(String nodeName) {
@@ -517,7 +552,7 @@ public class OverseerCollectionProcessor
Thread.sleep(100);
removed = !zkStateReader.getClusterState().hasCollection(message.getStr(collection));
if (removed) {
- Thread.sleep(100); // just a bit of time so it's more likely other
+ Thread.sleep(500); // just a bit of time so it's more likely other
// readers see on return
break;
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java Wed Jan 29 11:14:53 2014
@@ -74,8 +74,11 @@ public class SyncStrategy {
public String baseUrl;
}
- public boolean sync(ZkController zkController, SolrCore core,
- ZkNodeProps leaderProps) {
+ public boolean sync(ZkController zkController, SolrCore core, ZkNodeProps leaderProps) {
+ return sync(zkController, core, leaderProps, false);
+ }
+
+ public boolean sync(ZkController zkController, SolrCore core, ZkNodeProps leaderProps, boolean peerSyncOnlyWithActive) {
if (SKIP_AUTO_RECOVERY) {
return true;
}
@@ -95,7 +98,7 @@ public class SyncStrategy {
return false;
}
- success = syncReplicas(zkController, core, leaderProps);
+ success = syncReplicas(zkController, core, leaderProps, peerSyncOnlyWithActive);
} finally {
SolrRequestInfo.clearRequestInfo();
}
@@ -103,7 +106,7 @@ public class SyncStrategy {
}
private boolean syncReplicas(ZkController zkController, SolrCore core,
- ZkNodeProps leaderProps) {
+ ZkNodeProps leaderProps, boolean peerSyncOnlyWithActive) {
boolean success = false;
CloudDescriptor cloudDesc = core.getCoreDescriptor().getCloudDescriptor();
String collection = cloudDesc.getCollectionName();
@@ -117,7 +120,7 @@ public class SyncStrategy {
// first sync ourselves - we are the potential leader after all
try {
success = syncWithReplicas(zkController, core, leaderProps, collection,
- shardId);
+ shardId, peerSyncOnlyWithActive);
} catch (Exception e) {
SolrException.log(log, "Sync Failed", e);
}
@@ -145,7 +148,7 @@ public class SyncStrategy {
}
private boolean syncWithReplicas(ZkController zkController, SolrCore core,
- ZkNodeProps props, String collection, String shardId) {
+ ZkNodeProps props, String collection, String shardId, boolean peerSyncOnlyWithActive) {
List<ZkCoreNodeProps> nodes = zkController.getZkStateReader()
.getReplicaProps(collection, shardId,core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName(),
props.getStr(ZkStateReader.CORE_NAME_PROP));
@@ -163,7 +166,7 @@ public class SyncStrategy {
// if we can't reach a replica for sync, we still consider the overall sync a success
// TODO: as an assurance, we should still try and tell the sync nodes that we couldn't reach
// to recover once more?
- PeerSync peerSync = new PeerSync(core, syncWith, core.getUpdateHandler().getUpdateLog().numRecordsToKeep, true, true);
+ PeerSync peerSync = new PeerSync(core, syncWith, core.getUpdateHandler().getUpdateLog().numRecordsToKeep, true, true, peerSyncOnlyWithActive);
return peerSync.sync();
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/core/CoreContainer.java Wed Jan 29 11:14:53 2014
@@ -719,7 +719,7 @@ public class CoreContainer {
SolrCore newCore = core.reload(solrLoader, core);
// keep core to orig name link
solrCores.removeCoreToOrigName(newCore, core);
- registerCore(false, name, newCore, false);
+ registerCore(false, name, newCore, false, false);
} finally {
solrCores.removeFromPendingOps(name);
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java Wed Jan 29 11:14:53 2014
@@ -128,12 +128,10 @@ public class HdfsDirectoryFactory extend
new Object[] {slabSize, bankCount,
((long) bankCount * (long) slabSize)});
- int _1024Size = params.getInt("solr.hdfs.blockcache.bufferstore.1024",
- 8192);
- int _8192Size = params.getInt("solr.hdfs.blockcache.bufferstore.8192",
- 8192);
+ int bufferSize = params.getInt("solr.hdfs.blockcache.bufferstore.buffersize", 128);
+ int bufferCount = params.getInt("solr.hdfs.blockcache.bufferstore.buffercount", 128 * 128);
- BufferStore.init(_1024Size, _8192Size, metrics);
+ BufferStore.initNewBuffer(bufferSize, bufferCount);
long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank
* (long) blockSize;
try {
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/SnapShooter.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/SnapShooter.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/SnapShooter.java Wed Jan 29 11:14:53 2014
@@ -122,7 +122,7 @@ public class SnapShooter {
replicationHandler.snapShootDetails = details;
if (lock != null) {
try {
- lock.release();
+ lock.close();
} catch (IOException e) {
LOG.error("Unable to release snapshoot lock: " + directoryName + ".lock");
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Wed Jan 29 11:14:53 2014
@@ -827,7 +827,7 @@ public class CoreAdminHandler extends Re
props.put(ZkStateReader.CORE_NAME_PROP, cname);
props.put(ZkStateReader.NODE_NAME_PROP, zkController.getNodeName());
- boolean success = syncStrategy.sync(zkController, core, new ZkNodeProps(props));
+ boolean success = syncStrategy.sync(zkController, core, new ZkNodeProps(props), true);
// solrcloud_debug
if (log.isDebugEnabled()) {
try {
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java Wed Jan 29 11:14:53 2014
@@ -61,7 +61,7 @@ public class HttpShardHandlerFactory ext
new DefaultSolrThreadFactory("httpShardExecutor")
);
- private HttpClient defaultClient;
+ protected HttpClient defaultClient;
private LBHttpSolrServer loadbalancer;
//default values:
int soTimeout = 0;
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java Wed Jan 29 11:14:53 2014
@@ -40,6 +40,7 @@ import org.apache.solr.common.SolrInputD
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
@@ -539,6 +540,17 @@ public class RealTimeGetComponent extend
public void processSync(ResponseBuilder rb, int nVersions, String sync) {
+
+ boolean onlyIfActive = rb.req.getParams().getBool("onlyIfActive", false);
+
+ if (onlyIfActive) {
+ if (!rb.req.getCore().getCoreDescriptor().getCloudDescriptor().getLastPublished().equals(ZkStateReader.ACTIVE)) {
+ log.info("Last published state was not ACTIVE, cannot sync.");
+ rb.rsp.add("sync", "false");
+ return;
+ }
+ }
+
List<String> replicas = StrUtils.splitSmart(sync, ",", true);
boolean cantReachIsSuccess = rb.req.getParams().getBool("cantReachIsSuccess", false);
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java Wed Jan 29 11:14:53 2014
@@ -70,7 +70,7 @@ public class JavabinLoader extends Conte
private AddUpdateCommand addCmd = null;
@Override
- public void update(SolrInputDocument document, UpdateRequest updateRequest) {
+ public void update(SolrInputDocument document, UpdateRequest updateRequest, Integer commitWithin, Boolean overwrite) {
if (document == null) {
// Perhaps commit from the parameters
try {
@@ -85,6 +85,13 @@ public class JavabinLoader extends Conte
addCmd = getAddCommand(req, updateRequest.getParams());
}
addCmd.solrDoc = document;
+ if (commitWithin != null) {
+ addCmd.commitWithin = commitWithin;
+ }
+ if (overwrite != null) {
+ addCmd.overwrite = overwrite;
+ }
+
try {
processor.processAdd(addCmd);
addCmd.clear();
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java Wed Jan 29 11:14:53 2014
@@ -24,6 +24,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.postingshighlight.DefaultPassageFormatter;
@@ -68,6 +69,7 @@ import org.apache.solr.util.plugin.Plugi
* <str name="hl.bs.type">SENTENCE</str>
* <int name="hl.maxAnalyzedChars">10000</int>
* <str name="hl.multiValuedSeparatorChar"> </str>
+ * <bool name="hl.highlightMultiTerm">false</bool>
* </lst>
* </requestHandler>
* </pre>
@@ -98,6 +100,7 @@ import org.apache.solr.util.plugin.Plugi
* <li>hl.bs.variant (string) specifies country code for BreakIterator. default is empty string (root locale)
* <li>hl.maxAnalyzedChars specifies how many characters at most will be processed in a document.
* <li>hl.multiValuedSeparatorChar specifies the logical separator between values for multi-valued fields.
+ * <li>hl.highlightMultiTerm enables highlighting for range/wildcard/fuzzy/prefix queries.
* NOTE: currently hl.maxAnalyzedChars cannot yet be specified per-field
* </ul>
*
@@ -132,6 +135,8 @@ public class PostingsSolrHighlighter ext
maxPassages[i] = params.getFieldInt(fieldNames[i], HighlightParams.SNIPPETS, 1);
}
+ final IndexSchema schema = req.getSchema();
+
PostingsHighlighter highlighter = new PostingsHighlighter(maxLength) {
@Override
protected Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
@@ -178,6 +183,15 @@ public class PostingsSolrHighlighter ext
}
return sep.charAt(0);
}
+
+ @Override
+ protected Analyzer getIndexAnalyzer(String field) {
+ if (params.getFieldBool(field, HighlightParams.HIGHLIGHT_MULTI_TERM, false)) {
+ return schema.getAnalyzer();
+ } else {
+ return null;
+ }
+ }
};
Map<String,String[]> snippets = highlighter.highlightFields(fieldNames, query, searcher, docIDs, maxPassages);
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java Wed Jan 29 11:14:53 2014
@@ -28,7 +28,6 @@ import org.apache.lucene.search.MatchAll
import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.WildcardQuery;
@@ -419,10 +418,8 @@ public abstract class SolrQueryParserBas
* @return new PrefixQuery instance
*/
protected Query newPrefixQuery(Term prefix){
- PrefixQuery query = new PrefixQuery(prefix);
SchemaField sf = schema.getField(prefix.field());
- query.setRewriteMethod(sf.getType().getRewriteMethod(parser, sf));
- return query;
+ return sf.getType().getPrefixQuery(parser, sf, prefix.text());
}
/**
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java Wed Jan 29 11:14:53 2014
@@ -23,8 +23,7 @@ import com.google.common.cache.CacheBuil
import com.spatial4j.core.context.SpatialContext;
import com.spatial4j.core.context.SpatialContextFactory;
import com.spatial4j.core.distance.DistanceUtils;
-import com.spatial4j.core.exception.InvalidShapeException;
-import com.spatial4j.core.io.ParseUtils;
+import com.spatial4j.core.io.LegacyShapeReadWriterFormat;
import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Rectangle;
import com.spatial4j.core.shape.Shape;
@@ -47,14 +46,17 @@ import org.apache.solr.response.TextResp
import org.apache.solr.search.QParser;
import org.apache.solr.search.SpatialOptions;
import org.apache.solr.util.MapListener;
+import org.apache.solr.util.SpatialUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -89,12 +91,32 @@ public abstract class AbstractSpatialFie
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Must specify units=\"degrees\" on field types with class "+getClass().getSimpleName());
+ //replace legacy rect format with ENVELOPE
+ String wbStr = args.get("worldBounds");
+ if (wbStr != null && !wbStr.toUpperCase(Locale.ROOT).startsWith("ENVELOPE")) {
+ log.warn("Using old worldBounds format? Should use ENVELOPE(xMin, xMax, yMax, yMin).");
+ String[] parts = wbStr.split(" ");//"xMin yMin xMax yMax"
+ if (parts.length == 4) {
+ args.put("worldBounds",
+ "ENVELOPE(" + parts[0] + ", " + parts[2] + ", " + parts[3] + ", " + parts[1] + ")");
+ } //else likely eventual exception
+ }
+
//Solr expects us to remove the parameters we've used.
MapListener<String, String> argsWrap = new MapListener<String, String>(args);
ctx = SpatialContextFactory.makeSpatialContext(argsWrap, schema.getResourceLoader().getClassLoader());
args.keySet().removeAll(argsWrap.getSeenKeys());
- argsParser = new SpatialArgsParser();//might make pluggable some day?
+ argsParser = newSpatialArgsParser();
+ }
+
+ protected SpatialArgsParser newSpatialArgsParser() {
+ return new SpatialArgsParser() {
+ @Override
+ protected Shape parseShape(String str, SpatialContext ctx) throws ParseException {
+ return AbstractSpatialFieldType.this.parseShape(str);
+ }
+ };
}
//--------------------------------------------------------------
@@ -136,16 +158,39 @@ public abstract class AbstractSpatialFie
return result;
}
- protected Shape parseShape(String shapeStr) {
+ protected Shape parseShape(String str) {
+ if (str.length() == 0)
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty string shape");
+ //In Solr trunk we only support "lat, lon" (or x y) as an additional format; in v4.0 we do the
+ // weird Circle & Rect formats too (Spatial4j LegacyShapeReadWriterFormat).
try {
- return ctx.readShape(shapeStr);
- } catch (InvalidShapeException e) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ Shape shape = LegacyShapeReadWriterFormat.readShapeOrNull(str, ctx);
+ if (shape != null)
+ return shape;
+ return ctx.readShapeFromWkt(str);
+ } catch (Exception e) {
+ String message = e.getMessage();
+ if (!message.contains(str))
+ message = "Couldn't parse shape '" + str + "' because: " + message;
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message, e);
}
}
+ /**
+ * Returns a String version of a shape to be used for the stored value. This method in Solr is only called if for some
+ * reason a Shape object is passed to the field type (perhaps via a custom UpdateRequestProcessor),
+ * *and* the field is marked as stored. <em>The default implementation throws an exception.</em>
+ * <p/>
+ * Spatial4j 0.4 is probably the last release to support SpatialContext.toString(shape) but it's deprecated with no
+ * planned replacement. Shapes do have a toString() method but they are generally internal/diagnostic and not
+ * standard WKT.
+ * The solution is subclassing and calling ctx.toString(shape) or directly using LegacyShapeReadWriterFormat or
+ * passing in some sort of custom wrapped shape that holds a reference to a String or can generate it.
+ */
protected String shapeToString(Shape shape) {
- return ctx.toString(shape);
+// return ctx.toString(shape);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Getting a String from a Shape is no longer possible. See javadocs for commentary.");
}
/** Called from {@link #getStrategy(String)} upon first use by fieldName. } */
@@ -161,27 +206,16 @@ public abstract class AbstractSpatialFie
//--------------------------------------------------------------
/**
- * Implemented for compatibility with Solr 3 spatial geofilt & bbox query parsers:
+ * Implemented for compatibility with geofilt & bbox query parsers:
* {@link SpatialQueryable}.
*/
@Override
public Query createSpatialQuery(QParser parser, SpatialOptions options) {
- //--WARNING: the code from here to the next marker is identical to LatLonType's impl.
- double[] point = null;
- try {
- point = ParseUtils.parseLatitudeLongitude(options.pointStr);
- } catch (InvalidShapeException e) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
- }
-
- // lat & lon in degrees
- double latCenter = point[0];
- double lonCenter = point[1];
+ Point pt = SpatialUtils.parsePointSolrException(options.pointStr, ctx);
double distDeg = DistanceUtils.dist2Degrees(options.distance, options.radius);
- //--END-WARNING
- Shape shape = ctx.makeCircle(lonCenter, latCenter, distDeg);
+ Shape shape = ctx.makeCircle(pt, distDeg);
if (options.bbox)
shape = shape.getBoundingBox();
@@ -193,12 +227,9 @@ public abstract class AbstractSpatialFie
public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
if (!minInclusive || !maxInclusive)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Both sides of spatial range query must be inclusive: " + field.getName());
- Shape shape1 = parseShape(part1);
- Shape shape2 = parseShape(part2);
- if (!(shape1 instanceof Point) || !(shape2 instanceof Point))
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Both sides of spatial range query must be points: " + field.getName());
- Point p1 = (Point) shape1;
- Point p2 = (Point) shape2;
+ Point p1 = SpatialUtils.parsePointSolrException(part1, ctx);
+ Point p2 = SpatialUtils.parsePointSolrException(part2, ctx);
+
Rectangle bbox = ctx.makeRectangle(p1, p2);
SpatialArgs spatialArgs = new SpatialArgs(SpatialOperation.Intersects, bbox);
return getQueryFromSpatialArgs(parser, field, spatialArgs);//won't score by default
@@ -220,6 +251,8 @@ public abstract class AbstractSpatialFie
protected SpatialArgs parseSpatialArgs(String externalVal) {
try {
return argsParser.parse(externalVal, ctx);
+ } catch (SolrException e) {
+ throw e;
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/BinaryField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/BinaryField.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/BinaryField.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/BinaryField.java Wed Jan 29 11:14:53 2014
@@ -21,8 +21,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.GeneralField;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/schema/FieldType.java Wed Jan 29 11:14:53 2014
@@ -35,6 +35,7 @@ import org.apache.lucene.search.DocTermO
import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.FieldCacheRewriteMethod;
import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
@@ -427,6 +428,25 @@ public abstract class FieldType extends
return getClass().getName();
}
+
+ /**
+ * Returns a Query instance for doing prefix searches on this field type.
+ * Also, other QueryParser implementations may have different semantics.
+ * <p/>
+ * Sub-classes should override this method to provide their own range query implementation.
+ *
+ * @param parser the {@link org.apache.solr.search.QParser} calling the method
+ * @param sf the schema field
+ * @param termStr the term string for prefix query
+ * @return a Query instance to perform prefix search
+ *
+ */
+ public Query getPrefixQuery(QParser parser, SchemaField sf, String termStr) {
+ PrefixQuery query = new PrefixQuery(new Term(sf.getName(), termStr));
+ query.setRewriteMethod(sf.getType().getRewriteMethod(parser, sf));
+ return query;
+ }
+
/**
* Default analyzer for types that only produce 1 verbatim token...
* A maximum size of chars to be read must be specified