You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/07/01 19:02:52 UTC
incubator-usergrid git commit: Revert of geo per shard. Geo has a
hashing bug on write,
and therefore does not correctly allocate the same shard as the rest of the
entity on write. Aggregate + filter is the only mechanism to return correct
results witho
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-752 9971068a0 -> 595aa71d6
Revert of geo per shard. Geo has a hashing bug on write, and therefore does not correctly allocate the same shard as the rest of the entity on write. Aggregate + filter is the only mechanism to return correct results without triggering a complete re-index.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/595aa71d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/595aa71d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/595aa71d
Branch: refs/heads/USERGRID-752
Commit: 595aa71d6512ed5f43f9ff252d28824094cf7223
Parents: 9971068
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jul 1 11:02:48 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jul 1 11:02:48 2015 -0600
----------------------------------------------------------------------
.../persistence/geo/CollectionGeoSearch.java | 4 +-
.../persistence/geo/ConnectionGeoSearch.java | 4 +-
.../persistence/geo/GeoIndexSearcher.java | 13 +-
.../query/ir/result/CollectionShardFilter.java | 53 ++++++
.../ir/result/SearchCollectionVisitor.java | 12 +-
.../ir/result/SearchConnectionVisitor.java | 9 +-
.../org/apache/usergrid/persistence/GeoIT.java | 161 +++++++++----------
7 files changed, 154 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
index d8635d9..c823e20 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
@@ -44,9 +44,9 @@ public class CollectionGeoSearch extends GeoIndexSearcher {
private final EntityRef headEntity;
- public CollectionGeoSearch( EntityManager entityManager, final String shard, CassandraService cass,
+ public CollectionGeoSearch( EntityManager entityManager, IndexBucketLocator locator, CassandraService cass,
EntityRef headEntity, String collectionName ) {
- super( entityManager, shard, cass );
+ super( entityManager, locator, cass );
this.collectionName = collectionName;
this.headEntity = headEntity;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
index 37b96dd..a1ad71e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
@@ -43,9 +43,9 @@ public class ConnectionGeoSearch extends GeoIndexSearcher {
private final UUID connectionId;
- public ConnectionGeoSearch( EntityManager entityManager, final String shard, CassandraService cass,
+ public ConnectionGeoSearch( EntityManager entityManager, IndexBucketLocator locator, CassandraService cass,
UUID connectionId ) {
- super( entityManager, shard, cass );
+ super( entityManager, locator, cass );
this.connectionId = connectionId;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
index e61aa64..eb20ffa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
@@ -65,12 +65,12 @@ public abstract class GeoIndexSearcher {
private static final int MAX_FETCH_SIZE = 1000;
protected final EntityManager em;
- protected final String shard;
+ protected final IndexBucketLocator locator;
protected final CassandraService cass;
- public GeoIndexSearcher(final EntityManager entityManager, final String shard, final CassandraService cass ) {
+ public GeoIndexSearcher( EntityManager entityManager, IndexBucketLocator locator, CassandraService cass ) {
this.em = entityManager;
- this.shard = shard;
+ this.locator = locator;
this.cass = cass;
}
@@ -325,7 +325,12 @@ public abstract class GeoIndexSearcher {
for ( String geoCell : curGeocellsUnique ) {
// add buckets for each geoCell
- keys.add( key( key, DICTIONARY_GEOCELL, geoCell, shard ) );
+
+ //TODO, use merge logic here
+
+ for ( String indexBucket : locator.getBuckets( ) ) {
+ keys.add( key( key, DICTIONARY_GEOCELL, geoCell, indexBucket ) );
+ }
}
DynamicComposite start = null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java
new file mode 100644
index 0000000..cc8b7c7
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+
+
+/**
+ * Class that performs validation on an entity to ensure it's in the shard we expect
+ */
+public final class CollectionShardFilter implements ShardFilter {
+ private final IndexBucketLocator indexBucketLocator;
+ private final String expectedBucket;
+
+
+ public CollectionShardFilter( final IndexBucketLocator indexBucketLocator, final String expectedBucket ) {
+ this.indexBucketLocator = indexBucketLocator;
+ this.expectedBucket = expectedBucket;
+ }
+
+
+
+
+
+ public boolean isInShard( final ScanColumn scanColumn ) {
+
+ final UUID entityId = scanColumn.getUUID();
+
+ //not for our current processing shard, discard
+ final String shard = indexBucketLocator.getBucket( entityId );
+
+ return expectedBucket.equals( shard );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index 73eb148..32c3a6e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -128,16 +128,16 @@ public class SearchCollectionVisitor extends SearchVisitor {
final int size = queryProcessor.getPageSizeHint( node );
GeoIterator itr = new GeoIterator(
- new CollectionGeoSearch( em, bucket, cassandraService, headEntity, collection.getName() ),
+ new CollectionGeoSearch( em, indexBucketLocator, cassandraService, headEntity, collection.getName() ),
size, slice, node.getPropertyName(), new Point( node.getLattitude(), node.getLongitude() ),
node.getDistance() );
- this.results.push( itr );
+// this.results.push( itr );
-// final CollectionShardFilter
-// validator = new CollectionShardFilter(indexBucketLocator, bucket );
-//
-// this.results.push( new ShardFilterIterator( validator, itr, size));
+ final CollectionShardFilter
+ validator = new CollectionShardFilter(indexBucketLocator, bucket );
+
+ this.results.push( new ShardFilterIterator( validator, itr, size));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 404420a..5505914 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -115,17 +115,16 @@ public class SearchConnectionVisitor extends SearchVisitor {
//TODO, make search take a shard
GeoIterator itr =
- new GeoIterator( new ConnectionGeoSearch( em, bucket, cassandraService, connection.getIndexId() ),
+ new GeoIterator( new ConnectionGeoSearch( em, indexBucketLocator, cassandraService, connection.getIndexId() ),
size, slice, node.getPropertyName(),
new Point( node.getLattitude(), node.getLongitude() ), node.getDistance() );
- this.results.push( itr );
-// final CollectionShardFilter
-// validator = new CollectionShardFilter(indexBucketLocator, bucket );
+ final ConnectionShardFilter
+ validator = new ConnectionShardFilter(indexBucketLocator, bucket, connection );
-// this.results.push( new ShardFilterIterator( validator, itr, size ) );
+ this.results.push( new ShardFilterIterator( validator, itr, size ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
index 8e6fb80..8c9e9b7 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
@@ -76,10 +76,8 @@ public class GeoIT extends AbstractCoreIT {
Point center = new Point( 37.774277, -122.404744 );
- final String shard = setup.getIbl().getBucket( user.getUuid() );
-
CollectionGeoSearch connSearch =
- new CollectionGeoSearch( em, shard, setup.getCassSvc(), em.getApplicationRef(), "users" );
+ new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(), "users" );
List<EntityLocationRef> listResults =
@@ -421,86 +419,83 @@ public class GeoIT extends AbstractCoreIT {
}
-// @Test
-// public void testDenseSearch() throws Exception {
-//
-// UUID applicationId = setup.createApplication( "testOrganization", "testDenseSearch" );
-// assertNotNull( applicationId );
-//
-// EntityManager em = setup.getEmf().getEntityManager( applicationId );
-// assertNotNull( em );
-//
-// // save objects in a diagonal line from -90 -180 to 90 180
-//
-// int numEntities = 500;
-//
-// float minLattitude = 48.32455f;
-// float maxLattitude = 48.46481f;
-// float minLongitude = 9.89561f;
-// float maxLongitude = 10.0471f;
-//
-// float lattitudeDelta = ( maxLattitude - minLattitude ) / numEntities;
-//
-// float longitudeDelta = ( maxLongitude - minLongitude ) / numEntities;
-//
-// for ( int i = 0; i < numEntities; i++ ) {
-// float lattitude = minLattitude + lattitudeDelta * i;
-// float longitude = minLongitude + longitudeDelta * i;
-//
-// Map<String, Float> location = MapUtils.hashMap( "latitude", lattitude ).map( "longitude", longitude );
-//
-// Map<String, Object> data = new HashMap<String, Object>( 2 );
-// data.put( "name", String.valueOf( i ) );
-// data.put( "location", location );
-//
-// em.create( "store", data );
-// }
-//
-// //do a direct geo iterator test. We need to make sure that we short circuit on the correct tile.
-//
-// float lattitude = 48.38626f;
-// float longtitude = 9.94175f;
-// int distance = 1000;
-// int limit = 8;
-//
-//
-// QuerySlice slice = new QuerySlice( "location", 0 );
-//
-// //TODO test this
-// final String shard = setup.getIbl().getBucket( user.getUuid() );
-//
-// GeoIterator itr = new GeoIterator(
-// new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(), "stores" ),
-// limit, slice, "location", new Point( lattitude, longtitude ), distance );
-//
-//
-// // check we got back all 500 entities
-// assertFalse( itr.hasNext() );
-//
-// List<String> cells = itr.getLastCellsSearched();
-//
-// assertEquals( 1, cells.size() );
-//
-// assertEquals( 4, cells.get( 0 ).length() );
-//
-//
-// long startTime = System.currentTimeMillis();
-//
-// //now test at the EM level, there should be 0 results.
-// Query query = new Query();
-//
-// query.addFilter( "location within 1000 of 48.38626, 9.94175" );
-// query.setLimit( 8 );
-//
-//
-// Results results = em.searchCollection( em.getApplicationRef(), "stores", query );
-//
-// assertEquals( 0, results.size() );
-//
-// long endTime = System.currentTimeMillis();
-//
-// LOG.info( "Runtime took {} milliseconds to search", endTime - startTime );
-// }
+ @Test
+ public void testDenseSearch() throws Exception {
+
+ UUID applicationId = setup.createApplication( "testOrganization", "testDenseSearch" );
+ assertNotNull( applicationId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ assertNotNull( em );
+
+ // save objects in a diagonal line from -90 -180 to 90 180
+
+ int numEntities = 500;
+
+ float minLattitude = 48.32455f;
+ float maxLattitude = 48.46481f;
+ float minLongitude = 9.89561f;
+ float maxLongitude = 10.0471f;
+
+ float lattitudeDelta = ( maxLattitude - minLattitude ) / numEntities;
+
+ float longitudeDelta = ( maxLongitude - minLongitude ) / numEntities;
+
+ for ( int i = 0; i < numEntities; i++ ) {
+ float lattitude = minLattitude + lattitudeDelta * i;
+ float longitude = minLongitude + longitudeDelta * i;
+
+ Map<String, Float> location = MapUtils.hashMap( "latitude", lattitude ).map( "longitude", longitude );
+
+ Map<String, Object> data = new HashMap<String, Object>( 2 );
+ data.put( "name", String.valueOf( i ) );
+ data.put( "location", location );
+
+ em.create( "store", data );
+ }
+
+ //do a direct geo iterator test. We need to make sure that we short circuit on the correct tile.
+
+ float lattitude = 48.38626f;
+ float longtitude = 9.94175f;
+ int distance = 1000;
+ int limit = 8;
+
+
+ QuerySlice slice = new QuerySlice( "location", 0 );
+
+ GeoIterator itr = new GeoIterator(
+ new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(), "stores" ),
+ limit, slice, "location", new Point( lattitude, longtitude ), distance );
+
+
+ // check we got back all 500 entities
+ assertFalse( itr.hasNext() );
+
+ List<String> cells = itr.getLastCellsSearched();
+
+ assertEquals( 1, cells.size() );
+
+ assertEquals( 4, cells.get( 0 ).length() );
+
+
+ long startTime = System.currentTimeMillis();
+
+ //now test at the EM level, there should be 0 results.
+ Query query = new Query();
+
+ query.addFilter( "location within 1000 of 48.38626, 9.94175" );
+ query.setLimit( 8 );
+
+
+ Results results = em.searchCollection( em.getApplicationRef(), "stores", query );
+
+ assertEquals( 0, results.size() );
+
+ long endTime = System.currentTimeMillis();
+
+ LOG.info( "Runtime took {} milliseconds to search", endTime - startTime );
+ }
public Map<String, Object> getLocation( double latitude, double longitude ) throws Exception {