You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ds...@apache.org on 2015/05/21 15:16:53 UTC
svn commit: r1680869 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/spatial/
lucene/spatial/src/java/org/apache/lucene/spatial/composite/ solr/
solr/core/ solr/core/src/java/org/apache/solr/handler/component/
solr/core/src/java/org/apache/solr/...
Author: dsmiley
Date: Thu May 21 13:16:52 2015
New Revision: 1680869
URL: http://svn.apache.org/r1680869
Log:
SOLR-7379: Spatial RptWithGeometrySpatialField (based on CompositeSpatialStrategy)
Added:
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
- copied, changed from r1680862, lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml
- copied unchanged from r1680862, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java
lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial2.java
Modified: lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java?rev=1680869&r1=1680868&r2=1680869&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java (original)
+++ lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java Thu May 21 13:16:52 2015
@@ -62,6 +62,18 @@ public class CompositeSpatialStrategy ex
this.geometryStrategy = geometryStrategy;
}
+ public RecursivePrefixTreeStrategy getIndexStrategy() {
+ return indexStrategy;
+ }
+
+ public SerializedDVStrategy getGeometryStrategy() {
+ return geometryStrategy;
+ }
+
+ public boolean isOptimizePredicates() {
+ return optimizePredicates;
+ }
+
/** Set to false to NOT use optimized search predicates that avoid checking the geometry sometimes. Only useful for
* benchmarking. */
public void setOptimizePredicates(boolean optimizePredicates) {
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1680869&r1=1680868&r2=1680869&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Thu May 21 13:16:52 2015
@@ -155,6 +155,10 @@ New Features
* SOLR-7274: Pluggable authentication module in Solr. This defines an interface and a mechanism to create,
load, and use an Authentication plugin. (Noble Paul, Ishan Chattopadhyaya, Gregory Chanan, Anshum Gupta)
+* SOLR-7379: (experimental) New spatial RptWithGeometrySpatialField, based on CompositeSpatialStrategy,
+ which blends RPT indexes for speed with serialized geometry for accuracy. Includes a Lucene segment based
+ in-memory shape cache. (David Smiley)
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java?rev=1680869&r1=1680868&r2=1680869&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java Thu May 21 13:16:52 2015
@@ -47,10 +47,12 @@ import org.apache.solr.common.util.Named
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.AbstractSpatialPrefixTreeFieldType;
import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.RptWithGeometrySpatialField;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.SpatialRecursivePrefixTreeFieldType;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.QueryParsing;
+import org.apache.solr.util.DistanceUnits;
import org.apache.solr.util.SpatialUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,13 +77,24 @@ public class SpatialHeatmapFacets {
//get the strategy from the field type
final SchemaField schemaField = rb.req.getSchema().getField(fieldName);
final FieldType type = schemaField.getType();
- if (!(type instanceof AbstractSpatialPrefixTreeFieldType)) {
+
+ final PrefixTreeStrategy strategy;
+ final DistanceUnits distanceUnits;
+ // note: the two instanceof conditions is not ideal, versus one. If we start needing to add more then refactor.
+ if ((type instanceof AbstractSpatialPrefixTreeFieldType)) {
+ AbstractSpatialPrefixTreeFieldType rptType = (AbstractSpatialPrefixTreeFieldType) type;
+ strategy = (PrefixTreeStrategy) rptType.getStrategy(fieldName);
+ distanceUnits = rptType.getDistanceUnits();
+ } else if (type instanceof RptWithGeometrySpatialField) {
+ RptWithGeometrySpatialField rptSdvType = (RptWithGeometrySpatialField) type;
+ strategy = rptSdvType.getStrategy(fieldName).getIndexStrategy();
+ distanceUnits = rptSdvType.getDistanceUnits();
+ } else {
//FYI we support the term query one too but few people use that one
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "heatmap field needs to be of type "
- + SpatialRecursivePrefixTreeFieldType.class);
+ + SpatialRecursivePrefixTreeFieldType.class + " or " + RptWithGeometrySpatialField.class);
}
- AbstractSpatialPrefixTreeFieldType rptType = (AbstractSpatialPrefixTreeFieldType) type;
- final PrefixTreeStrategy strategy = (PrefixTreeStrategy) rptType.getStrategy(fieldName);
+
final SpatialContext ctx = strategy.getSpatialContext();
//get the bbox (query Rectangle)
@@ -106,7 +119,7 @@ public class SpatialHeatmapFacets {
final Double distErrObj = params.getFieldDouble(fieldKey, FacetParams.FACET_HEATMAP_DIST_ERR);
if (distErrObj != null) {
// convert distErr units based on configured units
- spatialArgs.setDistErr(distErrObj * rptType.getDistanceUnits().multiplierFromThisUnitToDegrees());
+ spatialArgs.setDistErr(distErrObj * distanceUnits.multiplierFromThisUnitToDegrees());
}
spatialArgs.setDistErrPct(params.getFieldDouble(fieldKey, FacetParams.FACET_HEATMAP_DIST_ERR_PCT));
double distErr = spatialArgs.resolveDistErr(ctx, DEFAULT_DIST_ERR_PCT);
Copied: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java (from r1680862, lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java?p2=lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java&p1=lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java&r1=1680862&r2=1680869&rev=1680869&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java Thu May 21 13:16:52 2015
@@ -142,7 +142,7 @@ public class RptWithGeometrySpatialField
@SuppressWarnings("unchecked")
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public FunctionValues getValues(Map context, final LeafReaderContext readerContext) throws IOException {
final FunctionValues targetFuncValues = targetValueSource.getValues(context, readerContext);
// The key is a pair of leaf reader with a docId relative to that reader. The value is a Map from field to Shape.
final SolrCache<PerSegCacheKey,Shape> cache =
Modified: lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml?rev=1680869&r1=1680868&r2=1680869&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml Thu May 21 13:16:52 2015
@@ -50,6 +50,8 @@
<fieldType name="pointvector" class="solr.SpatialPointVectorFieldType"
numberType="tdouble" distanceUnits="degrees"/>
+ <fieldType name="srptgeom" class="solr.RptWithGeometrySpatialField" />
+
<fieldType name="bbox" class="solr.BBoxField"
numberType="tdoubleDV" distanceUnits="degrees" storeSubFields="false"/>
</types>
@@ -64,6 +66,7 @@
<field name="srpt_packedquad" type="srpt_packedquad" multiValued="true" />
<field name="stqpt_geohash" type="stqpt_geohash" multiValued="true" />
<field name="pointvector" type="pointvector" />
+ <field name="srptgeom" type="srptgeom" />
<field name="bbox" type="bbox" />
<dynamicField name="bboxD_*" type="bbox" indexed="true" />
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial2.java?rev=1680869&r1=1680868&r2=1680869&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial2.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial2.java Thu May 21 13:16:52 2015
@@ -19,6 +19,8 @@ package org.apache.solr.search;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.FacetParams;
+import org.apache.solr.request.SolrQueryRequest;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -28,7 +30,7 @@ public class TestSolr4Spatial2 extends S
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-basic.xml", "schema-spatial.xml");
+ initCore("solrconfig-spatial.xml", "schema-spatial.xml");
}
@Override
@@ -72,19 +74,19 @@ public class TestSolr4Spatial2 extends S
);
//area2D
- assertJQ(req("q", "{!field f="+fieldName+" filter=false score=area2D}" +
+ assertJQ(req("q", "{!field f=" + fieldName + " filter=false score=area2D}" +
"Intersects(ENVELOPE(0,0,12,12))",//pt
"fl", "id,score",
"debug", "results"),//explain info
- "/response/docs/[0]/id=='1'" ,
+ "/response/docs/[0]/id=='1'",
"/response/docs/[0]/score==" + (30f * 5f) + "]"//150
);
//area (not 2D)
- assertJQ(req("q", "{!field f="+fieldName+" filter=false score=area}" +
+ assertJQ(req("q", "{!field f=" + fieldName + " filter=false score=area}" +
"Intersects(ENVELOPE(0,0,12,12))",//pt
"fl", "id,score",
"debug", "results"),//explain info
- "/response/docs/[0]/id=='1'" ,
+ "/response/docs/[0]/id=='1'",
"/response/docs/[0]/score==" + 146.39793f + "]"//a bit less than 150
);
}
@@ -94,8 +96,53 @@ public class TestSolr4Spatial2 extends S
String fieldName = "bbox";
assertQEx("expect friendly error message",
"area2D",
- req("{!field f="+fieldName+" filter=false score=bogus}Intersects(ENVELOPE(0,0,12,12))"),
+ req("{!field f=" + fieldName + " filter=false score=bogus}Intersects(ENVELOPE(0,0,12,12))"),
SolrException.ErrorCode.BAD_REQUEST);
}
+ @Test
+ public void testRptWithGeometryField() throws Exception {
+ String fieldName = "srptgeom"; //note: fails with "srpt_geohash" because it's not as precise
+ assertU(adoc("id", "0", fieldName, "ENVELOPE(-10, 20, 15, 10)"));
+ assertU(adoc("id", "1", fieldName, "BUFFER(POINT(-10 15), 5)"));//circle at top-left corner
+ assertU(optimize());// one segment.
+ assertU(commit());
+
+ // Search to the edge but not quite touching the indexed envelope of id=0. It requires geom validation to
+ // eliminate id=0. id=1 is found and doesn't require validation. cache=false means no query cache.
+ final SolrQueryRequest sameReq = req(
+ "q", "{!cache=false field f=" + fieldName + "}Intersects(ENVELOPE(-20, -10.0001, 30, 15.0001))",
+ "sort", "id asc");
+ assertJQ(sameReq, "/response/numFound==1", "/response/docs/[0]/id=='1'");
+
+ // The tricky thing is verifying the cache works correctly...
+
+ SolrCache cache = (SolrCache) h.getCore().getInfoRegistry().get("perSegSpatialFieldCache_srptgeom");
+ assertEquals("1", cache.getStatistics().get("cumulative_inserts").toString());
+ assertEquals("0", cache.getStatistics().get("cumulative_hits").toString());
+
+ // Repeat the query earlier
+ assertJQ(sameReq, "/response/numFound==1", "/response/docs/[0]/id=='1'");
+ assertEquals("1", cache.getStatistics().get("cumulative_hits").toString());
+
+ assertEquals("1 segment",
+ 1, ((SolrIndexSearcher) h.getCore().getInfoRegistry().get("searcher")).getRawReader().leaves().size());
+ // add new segment
+ assertU(adoc("id", "3"));
+ assertU(commit()); // sometimes merges (to one seg), sometimes won't
+ boolean newSeg =
+ (((SolrIndexSearcher)h.getCore().getInfoRegistry().get("searcher")).getRawReader().leaves().size() > 1);
+
+ // can still find the same document
+ assertJQ(sameReq, "/response/numFound==1", "/response/docs/[0]/id=='1'");
+
+ // when there are new segments, we accumulate another hit. This tests the cache was not blown away on commit.
+ assertEquals(newSeg ? "2" : "1", cache.getStatistics().get("cumulative_hits").toString());
+
+ // Now try to see if heatmaps work:
+ assertJQ(req("q", "*:*", "facet", "true", FacetParams.FACET_HEATMAP, fieldName, "json.nl", "map"),
+ "/facet_counts/facet_heatmaps/" + fieldName + "/minX==-180.0");
+
+ }
+
}