You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/07/30 19:19:39 UTC
[01/25] incubator-usergrid git commit: First pass of refactor to
allow concurrent tree execution.
Repository: incubator-usergrid
Updated Branches:
refs/heads/master 7fa7f4e68 -> 2547ee88f
First pass of refactor to allow concurrent tree execution.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9b21332d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9b21332d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9b21332d
Branch: refs/heads/master
Commit: 9b21332d1db5160f280fd74a3a64d82eca05e5cd
Parents: 107a465
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jun 16 11:58:39 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jun 16 11:58:39 2015 -0600
----------------------------------------------------------------------
stack/awscluster/aws.properties | 19 +++++++
.../apache/usergrid/persistence/Results.java | 10 ++--
.../persistence/cassandra/CassandraService.java | 45 ++++++++-------
.../persistence/cassandra/QueryProcessor.java | 13 +++--
.../cassandra/index/ConnectedIndexScanner.java | 38 +++----------
.../cassandra/index/IndexBucketScanner.java | 40 ++++++--------
.../cassandra/index/IndexScanner.java | 4 +-
.../persistence/geo/GeoIndexSearcher.java | 2 +
.../persistence/query/ir/SearchVisitor.java | 5 +-
.../result/CollectionSearchVisitorFactory.java | 35 ++++++++++++
.../result/ConnectionSearchVisitorFactory.java | 56 +++++++++++++++++++
.../query/ir/result/GatherIterator.java | 58 ++++++++++++++++++++
.../query/ir/result/SearchVisitorFactory.java | 37 +++++++++++++
13 files changed, 274 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/awscluster/aws.properties
----------------------------------------------------------------------
diff --git a/stack/awscluster/aws.properties b/stack/awscluster/aws.properties
new file mode 100644
index 0000000..182168a
--- /dev/null
+++ b/stack/awscluster/aws.properties
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+#
+accesskey=AKIAJBSDUBUSCVQKLNCQ
+secretkey=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
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
index 64e081e..d64489d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
@@ -33,7 +33,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion;
import org.apache.usergrid.persistence.cassandra.QueryProcessor;
-import org.apache.usergrid.persistence.query.ir.SearchVisitor;
+import org.apache.usergrid.persistence.query.ir.result.SearchVisitorFactory;
import org.apache.usergrid.utils.MapUtils;
import org.apache.usergrid.utils.StringUtils;
@@ -85,7 +85,7 @@ public class Results implements Iterable<Entity> {
String dataName;
private QueryProcessor queryProcessor;
- private SearchVisitor searchVisitor;
+ private SearchVisitorFactory searchVisitorFactory;
public Results() {
@@ -1273,8 +1273,8 @@ public class Results implements Iterable<Entity> {
}
- public void setSearchVisitor( SearchVisitor searchVisitor ) {
- this.searchVisitor = searchVisitor;
+ public void setSearchVisitorFactory( SearchVisitorFactory searchVisitor ) {
+ this.searchVisitorFactory = searchVisitor;
}
@@ -1288,6 +1288,6 @@ public class Results implements Iterable<Entity> {
q.setCursor( getCursor() );
queryProcessor.setQuery( q );
- return queryProcessor.getResults( searchVisitor );
+ return queryProcessor.getResults( searchVisitorFactory );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
index 43bfd2d..58d5d0f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
@@ -1027,27 +1027,30 @@ public class CassandraService {
*
* @throws Exception the exception
*/
- public IndexScanner getIdList( Keyspace ko, Object key, UUID start, UUID finish, int count, boolean reversed,
- IndexBucketLocator locator, UUID applicationId, String collectionName, boolean keepFirst )
- throws Exception {
-
- if ( count <= 0 ) {
- count = DEFAULT_COUNT;
- }
-
- if ( NULL_ID.equals( start ) ) {
- start = null;
- }
-
-
- final boolean skipFirst = start != null && !keepFirst;
-
- IndexScanner scanner =
- new IndexBucketScanner( this, locator, ENTITY_ID_SETS, applicationId, IndexType.COLLECTION, key, start,
- finish, reversed, count, skipFirst, collectionName );
-
- return scanner;
- }
+ public IndexScanner getIdList( Keyspace ko, Object key, UUID start, UUID finish, int count, boolean reversed,
+ IndexBucketLocator locator, UUID applicationId, String collectionName, boolean keepFirst )
+ throws Exception {
+
+ //TODO, refactor this
+ throw new UnsupportedOperationException( "Implement me" );
+
+ // if ( count <= 0 ) {
+ // count = DEFAULT_COUNT;
+ // }
+ //
+ // if ( NULL_ID.equals( start ) ) {
+ // start = null;
+ // }
+ //
+ //
+ // final boolean skipFirst = start != null && !keepFirst;
+ //
+ // IndexScanner scanner =
+ // new IndexBucketScanner( this, locator, ENTITY_ID_SETS, applicationId, IndexType.COLLECTION, key, start,
+ // finish, reversed, count, skipFirst, collectionName );
+ //
+ // return scanner;
+ }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
index c2b65b7..78392a4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
@@ -19,7 +19,6 @@ package org.apache.usergrid.persistence.cassandra;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import java.util.Stack;
import java.util.UUID;
@@ -47,14 +46,15 @@ import org.apache.usergrid.persistence.query.ir.OrNode;
import org.apache.usergrid.persistence.query.ir.OrderByNode;
import org.apache.usergrid.persistence.query.ir.QueryNode;
import org.apache.usergrid.persistence.query.ir.QuerySlice;
-import org.apache.usergrid.persistence.query.ir.SearchVisitor;
import org.apache.usergrid.persistence.query.ir.SliceNode;
import org.apache.usergrid.persistence.query.ir.UuidIdentifierNode;
import org.apache.usergrid.persistence.query.ir.WithinNode;
+import org.apache.usergrid.persistence.query.ir.result.GatherIterator;
import org.apache.usergrid.persistence.query.ir.result.ResultIterator;
import org.apache.usergrid.persistence.query.ir.result.ResultsLoader;
import org.apache.usergrid.persistence.query.ir.result.ResultsLoaderFactory;
import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
+import org.apache.usergrid.persistence.query.ir.result.SearchVisitorFactory;
import org.apache.usergrid.persistence.query.tree.AndOperand;
import org.apache.usergrid.persistence.query.tree.ContainsOperand;
import org.apache.usergrid.persistence.query.tree.Equal;
@@ -138,6 +138,7 @@ public class QueryProcessor {
if ( rootOperand != null ) {
// visit the tree
+ //TODO, evaluate for each shard and generate a tree in this class?
TreeEvaluator visitor = new TreeEvaluator();
rootOperand.visit( visitor );
@@ -259,16 +260,16 @@ public class QueryProcessor {
/**
* Return the iterator results, ordered if required
*/
- public Results getResults( SearchVisitor visitor ) throws Exception {
+ public Results getResults( final SearchVisitorFactory searchVisitorFactory ) throws Exception {
// if we have no order by just load the results
if ( rootNode == null ) {
return null;
}
- rootNode.visit( visitor );
+ //use the gather iterator to collect all the
- ResultIterator itr = visitor.getResults();
+ ResultIterator itr = new GatherIterator(rootNode, searchVisitorFactory.createVisitors() );
List<ScanColumn> entityIds = new ArrayList<ScanColumn>( Math.min( size, Query.MAX_LIMIT ) );
@@ -304,7 +305,7 @@ public class QueryProcessor {
results.setQuery( query );
results.setQueryProcessor( this );
- results.setSearchVisitor( visitor );
+ results.setSearchVisitorFactory( searchVisitorFactory );
return results;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
index b412df8..27689ce 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
@@ -63,7 +63,7 @@ public class ConnectedIndexScanner implements IndexScanner {
/**
* Iterator for our results from the last page load
*/
- private LinkedHashSet<HColumn<ByteBuffer, ByteBuffer>> lastResults;
+ private List<HColumn<ByteBuffer, ByteBuffer>> lastResults;
/**
* True if our last load loaded a full page size.
@@ -131,16 +131,6 @@ public class ConnectedIndexScanner implements IndexScanner {
}
- lastResults = new LinkedHashSet<HColumn<ByteBuffer, ByteBuffer>>();
-
-
- //cleanup columns for later logic
- //pointer to the first col we load
- HColumn<ByteBuffer, ByteBuffer> first = null;
-
- //pointer to the last column we load
- HColumn<ByteBuffer, ByteBuffer> last = null;
-
//go through each connection type until we exhaust the result sets
while ( currentConnectionType != null ) {
@@ -157,16 +147,7 @@ public class ConnectedIndexScanner implements IndexScanner {
final int resultSize = results.size();
- if(resultSize > 0){
-
- last = results.get( resultSize -1 );
-
- if(first == null ){
- first = results.get( 0 );
- }
- }
-
- lastResults.addAll( results );
+ lastResults = results;
// we loaded a full page, there might be more
@@ -193,14 +174,13 @@ public class ConnectedIndexScanner implements IndexScanner {
}
//remove the first element, we need to skip it
- if ( skipFirst && first != null) {
- lastResults.remove( first );
+ if ( skipFirst && lastResults.size() > 0) {
+ lastResults.remove( 0 );
}
- if ( hasMore && last != null ) {
+ if ( hasMore && lastResults.size() > 0 ) {
// set the bytebuffer for the next pass
- start = last.getName();
- lastResults.remove( last );
+ lastResults.remove( lastResults.size() - 1 );
}
return lastResults != null && lastResults.size() > 0;
@@ -213,7 +193,7 @@ public class ConnectedIndexScanner implements IndexScanner {
* @see java.lang.Iterable#iterator()
*/
@Override
- public Iterator<Set<HColumn<ByteBuffer, ByteBuffer>>> iterator() {
+ public Iterator<List<HColumn<ByteBuffer, ByteBuffer>>> iterator() {
return this;
}
@@ -250,8 +230,8 @@ public class ConnectedIndexScanner implements IndexScanner {
*/
@Override
@Metered( group = "core", name = "IndexBucketScanner_load" )
- public Set<HColumn<ByteBuffer, ByteBuffer>> next() {
- Set<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
+ public List<HColumn<ByteBuffer, ByteBuffer>> next() {
+ List<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
lastResults = null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
index b2ca591..8a9b709 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
@@ -47,16 +47,14 @@ import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtil
public class IndexBucketScanner implements IndexScanner {
private final CassandraService cass;
- private final IndexBucketLocator indexBucketLocator;
private final UUID applicationId;
private final Object keyPrefix;
private final ApplicationCF columnFamily;
private final Object finish;
private final boolean reversed;
private final int pageSize;
- private final String[] indexPath;
- private final IndexType indexType;
private final boolean skipFirst;
+ private final String bucket;
/** Pointer to our next start read */
private Object start;
@@ -65,18 +63,17 @@ public class IndexBucketScanner implements IndexScanner {
private Object scanStart;
/** Iterator for our results from the last page load */
- private TreeSet<HColumn<ByteBuffer, ByteBuffer>> lastResults;
+ private List<HColumn<ByteBuffer, ByteBuffer>> lastResults;
/** True if our last load loaded a full page size. */
private boolean hasMore = true;
- public IndexBucketScanner( CassandraService cass, IndexBucketLocator locator, ApplicationCF columnFamily,
- UUID applicationId, IndexType indexType, Object keyPrefix, Object start, Object finish,
- boolean reversed, int pageSize, boolean skipFirst, String... indexPath) {
+ public IndexBucketScanner( CassandraService cass, ApplicationCF columnFamily,
+ UUID applicationId, Object keyPrefix, Object start, Object finish,
+ boolean reversed, int pageSize, boolean skipFirst, String bucket) {
this.cass = cass;
- this.indexBucketLocator = locator;
this.applicationId = applicationId;
this.keyPrefix = keyPrefix;
this.columnFamily = columnFamily;
@@ -84,11 +81,10 @@ public class IndexBucketScanner implements IndexScanner {
this.finish = finish;
this.reversed = reversed;
this.skipFirst = skipFirst;
+ this.bucket = bucket;
//we always add 1 to the page size. This is because we pop the last column for the next page of results
this.pageSize = pageSize+1;
- this.indexPath = indexPath;
- this.indexType = indexType;
this.scanStart = start;
}
@@ -117,13 +113,7 @@ public class IndexBucketScanner implements IndexScanner {
return false;
}
- List<String> keys = indexBucketLocator.getBuckets( applicationId, indexType, indexPath );
-
- List<Object> cassKeys = new ArrayList<Object>( keys.size() );
-
- for ( String bucket : keys ) {
- cassKeys.add( key( keyPrefix, bucket ) );
- }
+ final Object rowKey = key( keyPrefix, bucket );
//if we skip the first we need to set the load to page size +2, since we'll discard the first
//and start paging at the next entity, otherwise we'll just load the page size we need
@@ -138,8 +128,10 @@ public class IndexBucketScanner implements IndexScanner {
selectSize++;
}
- TreeSet<HColumn<ByteBuffer, ByteBuffer>> resultsTree = IndexMultiBucketSetLoader
- .load( cass, columnFamily, applicationId, cassKeys, start, finish, selectSize, reversed );
+
+ final List<HColumn<ByteBuffer, ByteBuffer>>
+ resultsTree = cass.getColumns( cass.getApplicationKeyspace( applicationId ), columnFamily, rowKey,
+ start, finish, selectSize, reversed );
//remove the first element, it's from a cursor value and we don't want to retain it
@@ -150,7 +142,7 @@ public class IndexBucketScanner implements IndexScanner {
// set the bytebuffer for the next pass
- start = resultsTree.pollLast().getName();
+ start = resultsTree.get( resultsTree.size() - 1 ).getName();
}
else {
hasMore = false;
@@ -158,7 +150,7 @@ public class IndexBucketScanner implements IndexScanner {
//remove the first element since it needs to be skipped AFTER the size check. Otherwise it will fail
if ( firstPageSkipFirst ) {
- resultsTree.pollFirst();
+ resultsTree.remove( 0 );
}
lastResults = resultsTree;
@@ -173,7 +165,7 @@ public class IndexBucketScanner implements IndexScanner {
* @see java.lang.Iterable#iterator()
*/
@Override
- public Iterator<Set<HColumn<ByteBuffer, ByteBuffer>>> iterator() {
+ public Iterator<List<HColumn<ByteBuffer, ByteBuffer>>> iterator() {
return this;
}
@@ -210,8 +202,8 @@ public class IndexBucketScanner implements IndexScanner {
*/
@Override
@Metered(group = "core", name = "IndexBucketScanner_load")
- public NavigableSet<HColumn<ByteBuffer, ByteBuffer>> next() {
- NavigableSet<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
+ public List<HColumn<ByteBuffer, ByteBuffer>> next() {
+ List<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
lastResults = null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexScanner.java
index a938ca3..1307361 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexScanner.java
@@ -19,7 +19,7 @@ package org.apache.usergrid.persistence.cassandra.index;
import java.nio.ByteBuffer;
import java.util.Iterator;
-import java.util.Set;
+import java.util.List;
import me.prettyprint.hector.api.beans.HColumn;
@@ -30,7 +30,7 @@ import me.prettyprint.hector.api.beans.HColumn;
* @author tnine
*/
public interface IndexScanner
- extends Iterable<Set<HColumn<ByteBuffer, ByteBuffer>>>, Iterator<Set<HColumn<ByteBuffer, ByteBuffer>>> {
+ extends Iterable<List<HColumn<ByteBuffer, ByteBuffer>>>, Iterator<List<HColumn<ByteBuffer, ByteBuffer>>> {
/** Reset the scanner back to the start */
public void reset();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
index 4bc160d..0bbace0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
@@ -328,6 +328,8 @@ public abstract class GeoIndexSearcher {
// add buckets for each geoCell
+ //TODO, use merge logic here
+
for ( String indexBucket : locator.getBuckets( appId, IndexType.GEO, geoCell ) ) {
keys.add( key( key, DICTIONARY_GEOCELL, geoCell, indexBucket ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
index e8c189a..0038689 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
@@ -56,14 +56,17 @@ public abstract class SearchVisitor implements NodeVisitor {
protected final Stack<ResultIterator> results = new Stack<ResultIterator>();
+ protected final String bucket;
+
/**
* @param queryProcessor
*/
- public SearchVisitor( QueryProcessor queryProcessor ) {
+ public SearchVisitor( QueryProcessor queryProcessor, final String bucket ) {
this.query = queryProcessor.getQuery();
this.queryProcessor = queryProcessor;
this.em = queryProcessor.getEntityManager();
+ this.bucket = bucket;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
new file mode 100644
index 0000000..1fd1604
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.Collection;
+
+import org.apache.usergrid.persistence.cassandra.QueryProcessor;
+import org.apache.usergrid.persistence.query.ir.SearchVisitor;
+
+
+public class CollectionSearchVisitorFactory implements SearchVisitorFactory {
+
+
+ @Override
+ public Collection<SearchVisitor> createVisitors() {
+// SearchConnectionVisitor visitor = new SearchConnectionVisitor( qp, connectionRef, true );
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
new file mode 100644
index 0000000..371f79b
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.Collection;
+
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.QueryProcessor;
+import org.apache.usergrid.persistence.query.ir.SearchVisitor;
+
+
+public class ConnectionSearchVisitorFactory implements SearchVisitorFactory {
+
+ private final
+ private final IndexBucketLocator indexBucketLocator;
+ private final QueryProcessor queryProcessor;
+ private final ConnectionRef connectionRef;
+ private final boolean outgoing;
+
+ private ConnectionSearchVisitorFactory( final IndexBucketLocator indexBucketLocator,
+ final QueryProcessor queryProcessor, final ConnectionRef connectionRef,
+ final boolean outgoing ){
+// SearchConnectionVisitor visitor = new SearchConnectionVisitor( indexBucketLocator, qp, connectionRef, true );
+
+ this.indexBucketLocator = indexBucketLocator;
+ this.queryProcessor = queryProcessor;
+ this.connectionRef = connectionRef;
+ this.outgoing = outgoing;
+ }
+
+
+ @Override
+ public Collection<SearchVisitor> createVisitors() {
+
+ indexBucketLocator.getBuckets( )
+
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
new file mode 100644
index 0000000..48e2301
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -0,0 +1,58 @@
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.cassandra.CursorCache;
+import org.apache.usergrid.persistence.query.ir.QueryNode;
+import org.apache.usergrid.persistence.query.ir.SearchVisitor;
+
+
+/**
+ * Used to gather results from multiple sub iterators
+ */
+public class GatherIterator implements ResultIterator {
+
+
+ private final Collection<SearchVisitor> searchVisitors;
+ private final QueryNode rootNode;
+
+
+ public GatherIterator( final QueryNode rootNode, final Collection<SearchVisitor> searchVisitors) {
+ this.rootNode = rootNode;
+ this.searchVisitors = searchVisitors;
+ }
+
+
+ @Override
+ public void reset() {
+ throw new UnsupportedOperationException( "Gather iterators cannot be reset" );
+ }
+
+
+ @Override
+ public void finalizeCursor( final CursorCache cache, final UUID lastValue ) {
+ //find the last value in the tree, and return it's cursor
+ }
+
+
+ @Override
+ public Iterator<Set<ScanColumn>> iterator() {
+ return this;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
+
+
+ @Override
+ public Set<ScanColumn> next() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b21332d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchVisitorFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchVisitorFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchVisitorFactory.java
new file mode 100644
index 0000000..26603b4
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchVisitorFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.Collection;
+
+import org.apache.usergrid.persistence.query.ir.SearchVisitor;
+
+
+/**
+ * A factory for generating search visitors for a shard
+ */
+public interface SearchVisitorFactory {
+
+ /**
+ * Creates a collection of search visitors. Each visitor will evaluate the tree, and the results will be aggregated together
+ *
+ * @return 1 or more search visitors to visit our IR tree and return searches
+ */
+ Collection<SearchVisitor> createVisitors();
+}
+
[15/25] incubator-usergrid git commit: Fixes multi order by
Posted by sn...@apache.org.
Fixes multi order by
Cleans up unused code and adds license headers
Fixes export tools.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/44448037
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/44448037
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/44448037
Branch: refs/heads/master
Commit: 4444803727620198045fd06143255495227bac4a
Parents: 501da72
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Jul 6 16:43:05 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Jul 6 16:43:05 2015 -0600
----------------------------------------------------------------------
.../query/ir/result/AbstractScanColumn.java | 15 +-
.../query/ir/result/GatherIterator.java | 16 ++
.../query/ir/result/GeoIterator.java | 20 --
.../query/ir/result/OrderByIterator.java | 144 ++++++++------
.../persistence/query/ir/result/ScanColumn.java | 17 --
.../ir/result/SearchCollectionVisitor.java | 16 ++
.../ir/result/SearchConnectionVisitor.java | 16 ++
.../query/ir/result/SliceCursorGenerator.java | 4 +-
.../persistence/query/ir/result/UUIDColumn.java | 16 ++
.../query/ir/result/UnionIterator.java | 17 --
.../apache/usergrid/tools/EntityCleanup.java | 80 ++++----
.../usergrid/tools/EntityInsertBenchMark.java | 3 +-
.../usergrid/tools/EntityReadBenchMark.java | 11 +-
.../usergrid/tools/UniqueIndexCleanup.java | 186 ++++++++++---------
14 files changed, 300 insertions(+), 261 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
index 35672d9..cabb710 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
@@ -52,7 +52,7 @@ public abstract class AbstractScanColumn implements ScanColumn {
}
- @Override
+
public ByteBuffer getCursorValue() {
return buffer == null ? null : buffer.duplicate();
}
@@ -87,19 +87,6 @@ public abstract class AbstractScanColumn implements ScanColumn {
'}';
}
-
- @Override
- public void setChild( final ScanColumn childColumn ) {
- this.child = childColumn;
- }
-
-
- @Override
- public ScanColumn getChild() {
- return child;
- }
-
-
@Override
public void addToCursor( final CursorCache cache ) {
this.sliceCursorGenerator.addToCursor( cache, this );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index 9d2e4da..e199d33 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.persistence.query.ir.result;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
index 6b5f527..7f09bb3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
@@ -259,9 +259,6 @@ public class GeoIterator implements ResultIterator {
private final EntityLocationRef location;
private final GeoCursorGenerator geoCursorGenerator;
- private ScanColumn child;
-
-
public LocationScanColumn( EntityLocationRef location, final GeoCursorGenerator geoCursorGenerator ) {
this.location = location;
@@ -275,18 +272,6 @@ public class GeoIterator implements ResultIterator {
}
- @Override
- public ByteBuffer getCursorValue() {
- throw new UnsupportedOperationException(
- "This is not supported for location scan columns. It requires iterator information" );
- }
-
-
- @Override
- public void setChild( final ScanColumn childColumn ) {
- this.child = childColumn;
- }
-
@Override
public void addToCursor( final CursorCache cache ) {
@@ -294,11 +279,6 @@ public class GeoIterator implements ResultIterator {
}
- @Override
- public ScanColumn getChild() {
- return this.child;
- }
-
@Override
public boolean equals( Object o ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
index 6d05bd6..13fd306 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
@@ -21,17 +21,17 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.collections.comparators.ComparatorChain;
+
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityPropertyComparator;
@@ -40,9 +40,8 @@ import org.apache.usergrid.persistence.Query.SortPredicate;
import org.apache.usergrid.persistence.cassandra.CursorCache;
import org.apache.usergrid.persistence.query.ir.QuerySlice;
-import org.apache.commons.collections.comparators.ComparatorChain;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
* 1) Take a result set iterator as the child 2) Iterate only over candidates and create a cursor from the candidates
@@ -63,6 +62,8 @@ public class OrderByIterator extends MergeIterator {
//our last result from in memory sorting
private SortedEntitySet entries;
+ private final UUIDCursorGenerator<MultiColumnSort> generator;
+
/**
* @param pageSize
@@ -91,6 +92,8 @@ public class OrderByIterator extends MergeIterator {
// paging
this.secondaryFields.add( NAME_UUID );
this.subSortCompare.addComparator( new EntityPropertyComparator( NAME_UUID, false ) );
+
+ this.generator = new UUIDCursorGenerator<MultiColumnSort>( this.slice.hashCode() );
}
@@ -105,7 +108,7 @@ public class OrderByIterator extends MergeIterator {
minEntryId = ue.fromByteBuffer( cursor );
}
- entries = new SortedEntitySet( subSortCompare, em, secondaryFields, pageSize, minEntryId );
+ entries = new SortedEntitySet( subSortCompare, em, secondaryFields, pageSize, minEntryId, generator );
/**
* keep looping through our peek iterator. We need to inspect each forward page to ensure we have performed a
@@ -134,45 +137,75 @@ public class OrderByIterator extends MergeIterator {
// no op
}
-//
-// @Override
-// public void finalizeCursor( CursorCache cache, UUID lastValue ) {
-// int sliceHash = slice.hashCode();
-//
-// ByteBuffer bytes = ue.toByteBuffer( lastValue );
-//
-// if ( bytes == null ) {
-// return;
-// }
-//
-// cache.setNextCursor( sliceHash, bytes );
-// }
+ //
+ // @Override
+ // public void finalizeCursor( CursorCache cache, UUID lastValue ) {
+ // int sliceHash = slice.hashCode();
+ //
+ // ByteBuffer bytes = ue.toByteBuffer( lastValue );
+ //
+ // if ( bytes == null ) {
+ // return;
+ // }
+ //
+ // cache.setNextCursor( sliceHash, bytes );
+ // }
/** A Sorted set with a max size. When a new entry is added, the max is removed */
- public static final class SortedEntitySet extends TreeSet<Entity> {
+ public static final class SortedEntitySet {
private final int maxSize;
- private final Map<UUID, ScanColumn> cursorVal = new HashMap<UUID, ScanColumn>();
+ private final UUIDCursorGenerator<MultiColumnSort> generator;
+ private final Set<UUID> uuidBuffer = new LinkedHashSet<UUID>();
+ private final TreeSet<ScanColumn> sortedEntities = new TreeSet<ScanColumn>();
private final EntityManager em;
private final List<String> fields;
private final Entity minEntity;
private final Comparator<Entity> comparator;
- public SortedEntitySet( Comparator<Entity> comparator, EntityManager em, List<String> fields, int maxSize,
- UUID minEntityId ) {
- super( comparator );
+ public SortedEntitySet( final Comparator<Entity> comparator, final EntityManager em, final List<String> fields,
+ final int maxSize, final UUID minEntityId,
+ final UUIDCursorGenerator<MultiColumnSort> generator ) {
this.maxSize = maxSize;
this.em = em;
this.fields = fields;
this.comparator = comparator;
+ this.generator = generator;
this.minEntity = getPartialEntity( minEntityId );
}
- @Override
- public boolean add( Entity entity ) {
+ /** Add the current scancolumn to be loaded **/
+ public void add( ScanColumn col ) {
+ uuidBuffer.add( col.getUUID() );
+ }
+
+
+ public void load() {
+ try {
+
+
+ //load all the entities
+ for ( Entity e : em.getPartialEntities( uuidBuffer, fields ) ) {
+ add( e );
+ }
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to load partial entities", e );
+ throw new RuntimeException( e );
+ }
+ }
+
+
+ /** Turn our sorted entities into a set of ids */
+ public Set<ScanColumn> toIds() {
+ return sortedEntities;
+ }
+
+
+ private boolean add( Entity entity ) {
// don't add this entity. We get it in our scan range, but it's <= the minimum value that
//should be allowed in the result set
@@ -180,25 +213,17 @@ public class OrderByIterator extends MergeIterator {
return false;
}
- boolean added = super.add( entity );
+ boolean added = sortedEntities.add( new MultiColumnSort( entity, comparator, generator ) );
- while ( size() > maxSize ) {
+ while ( sortedEntities.size() > maxSize ) {
//remove our last element, we're over size
- Entity e = this.pollLast();
- //remove it from the cursors as well
- cursorVal.remove( e.getUuid() );
+ sortedEntities.pollLast();
}
return added;
}
- /** add the id to be loaded, and the dynamiccomposite column that belongs with it */
- public void add( ScanColumn col ) {
- cursorVal.put( col.getUUID(), col );
- }
-
-
private Entity getPartialEntity( UUID minEntityId ) {
List<Entity> entities;
@@ -216,35 +241,42 @@ public class OrderByIterator extends MergeIterator {
return entities.get( 0 );
}
+ }
- public void load() {
- try {
- for ( Entity e : em.getPartialEntities( cursorVal.keySet(), fields ) ) {
- add( e );
- }
- }
- catch ( Exception e ) {
- logger.error( "Unable to load partial entities", e );
- throw new RuntimeException( e );
- }
+ private static final class MultiColumnSort implements ScanColumn {
+
+ private final CursorGenerator<MultiColumnSort> generator;
+ private final Entity partialCompareEntity;
+ private final Comparator<Entity> entityComparator;
+
+
+ private MultiColumnSort( final Entity partialCompareEntity, final Comparator<Entity> entityComparator,
+ final CursorGenerator<MultiColumnSort> generator ) {
+ this.generator = generator;
+ this.partialCompareEntity = partialCompareEntity;
+ this.entityComparator = entityComparator;
}
- /** Turn our sorted entities into a set of ids */
- public Set<ScanColumn> toIds() {
- Iterator<Entity> itr = iterator();
+ @Override
+ public UUID getUUID() {
+ return partialCompareEntity.getUuid();
+ }
- Set<ScanColumn> columns = new LinkedHashSet<ScanColumn>( this.size() );
- while ( itr.hasNext() ) {
+ @Override
+ public void addToCursor( final CursorCache cache ) {
+ this.generator.addToCursor( cache, this );
+ }
- UUID id = itr.next().getUuid();
- columns.add( cursorVal.get( id ) );
- }
+ @Override
+ public int compareTo( final ScanColumn o ) {
+
+ final MultiColumnSort other = ( MultiColumnSort ) o;
- return columns;
+ return entityComparator.compare( this.partialCompareEntity, other.partialCompareEntity );
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
index 577d34a..eaed2d3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
@@ -30,17 +30,6 @@ public interface ScanColumn extends Comparable<ScanColumn> {
/** Get the uuid from the column */
UUID getUUID();
- /** Get the cursor value of this column */
- ByteBuffer getCursorValue();
-
- /**
- * Append the child column used in tree iterator to this column, along with the comparator used to compare them
- *
- * for instance, a term search of A = 1 AND B = 2 would generate a ScanColumn of A-- child -> B
- * @param childColumn
- */
- void setChild( final ScanColumn childColumn );
-
/**
* Use the generator to add this value to the cursor cache
@@ -49,10 +38,4 @@ public interface ScanColumn extends Comparable<ScanColumn> {
void addToCursor( final CursorCache cache );
- /**
- * Returns the childl column if present, can return null
- * @return
- */
- ScanColumn getChild();
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index 32c3a6e..c991783 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.persistence.query.ir.result;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 5949ee4..f518297 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.persistence.query.ir.result;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
index cad0a0f..d3fd534 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
@@ -29,7 +29,7 @@ import org.apache.usergrid.persistence.query.ir.QuerySlice;
/**
* A cursor generator for the specified query slice
*/
-public class SliceCursorGenerator implements CursorGenerator<ScanColumn> {
+public class SliceCursorGenerator implements CursorGenerator<AbstractScanColumn> {
private final QuerySlice slice;
@@ -38,7 +38,7 @@ public class SliceCursorGenerator implements CursorGenerator<ScanColumn> {
@Override
- public void addToCursor( final CursorCache cache, final ScanColumn col ) {
+ public void addToCursor( final CursorCache cache, final AbstractScanColumn col ) {
if ( col == null ) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
index 264a9b6..f5d0091 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.persistence.query.ir.result;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
index 58e0e9a..526fced 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -253,28 +253,11 @@ public class UnionIterator extends MultiIterator {
@Override
- public ByteBuffer getCursorValue() {
- return ue.toByteBuffer( delegate.getUUID() );
- }
-
-
- @Override
- public void setChild( final ScanColumn childColumn ) {
- //intentionally a no-op, since child is on the delegate
- }
-
-
- @Override
public void addToCursor( final CursorCache cache ) {
this.uuidCursorGenerator.addToCursor( cache, this );
}
- @Override
- public ScanColumn getChild() {
- return delegate.getChild();
- }
-
@Override
public boolean equals( final Object o ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
index 368119c..3691031 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
@@ -25,9 +25,14 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.cassandra.CassandraService;
@@ -38,11 +43,6 @@ import org.apache.usergrid.persistence.query.ir.result.ScanColumnTransformer;
import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
import org.apache.usergrid.persistence.query.ir.result.UUIDIndexSliceParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.mutation.Mutator;
@@ -51,9 +51,9 @@ import static org.apache.usergrid.persistence.Schema.DICTIONARY_COLLECTIONS;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_ID_SETS;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
@@ -122,53 +122,57 @@ public class EntityCleanup extends ToolBase {
// go through each collection and audit the value
for ( String collectionName : collectionNames ) {
- IndexScanner scanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
- key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
- indexBucketLocator, applicationId, collectionName, false );
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser( uuidCursorGenerator ) );
+ for ( final String bucketName : indexBucketLocator.getBuckets() ) {
- while ( itr.hasNext() ) {
+ IndexScanner scanner =
+ cass.getIdList( key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null,
+ PAGE_SIZE, false, bucketName, applicationId, false );
- // load all entity ids from the index itself.
+ SliceIterator itr = new SliceIterator( scanner, new UUIDIndexSliceParser( null ) );
- Set<ScanColumn> copy = new LinkedHashSet<ScanColumn>( itr.next() );
+ while ( itr.hasNext() ) {
- results = em.get( ScanColumnTransformer.getIds( copy ) );
- // nothing to do they're the same size so there's no
- // orphaned uuid's in the entity index
- if ( copy.size() == results.size() ) {
- continue;
- }
+ // load all entity ids from the index itself.
- // they're not the same, we have some orphaned records,
- // remove them
+ Set<ScanColumn> copy = new LinkedHashSet<ScanColumn>( itr.next() );
- for ( Entity returned : results.getEntities() ) {
- copy.remove( returned.getUuid() );
- }
+ results = em.get( ScanColumnTransformer.getIds( copy ) );
+ // nothing to do they're the same size so there's no
+ // orphaned uuid's in the entity index
+ if ( copy.size() == results.size() ) {
+ continue;
+ }
- // what's left needs deleted, do so
+ // they're not the same, we have some orphaned records,
+ // remove them
- logger.info( "Cleaning up {} orphaned entities for app {}", copy.size(), app.getValue() );
+ for ( Entity returned : results.getEntities() ) {
+ copy.remove( returned.getUuid() );
+ }
- Keyspace ko = cass.getApplicationKeyspace( applicationId );
- Mutator<ByteBuffer> m = createMutator( ko, be );
+ // what's left needs deleted, do so
- for ( ScanColumn col : copy ) {
+ logger.info( "Cleaning up {} orphaned entities for app {}", copy.size(), app.getValue() );
- final UUID id = col.getUUID();
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
- Object collections_key = key( applicationId, Schema.DICTIONARY_COLLECTIONS, collectionName,
- indexBucketLocator
- .getBucket( applicationId, IndexType.COLLECTION, id, collectionName ) );
+ for ( ScanColumn col : copy ) {
- addDeleteToMutator( m, ENTITY_ID_SETS, collections_key, id, timestamp );
+ final UUID id = col.getUUID();
- logger.info( "Deleting entity with id '{}' from collection '{}'", id, collectionName );
- }
+ Object collections_key = key( applicationId, Schema.DICTIONARY_COLLECTIONS, collectionName,
+ indexBucketLocator
+ .getBucket( id ) );
+
+ addDeleteToMutator( m, ENTITY_ID_SETS, collections_key, id, timestamp );
- m.execute();
+ logger.info( "Deleting entity with id '{}' from collection '{}'", id, collectionName );
+ }
+
+ m.execute();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java
index fe1edbf..bf3d5d4 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java
@@ -29,7 +29,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.DynamicEntity;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
import org.apache.usergrid.utils.UUIDUtils;
@@ -173,7 +172,7 @@ public class EntityInsertBenchMark extends ToolBase {
String bucketId =
- indexBucketLocator.getBucket( appId, IndexType.COLLECTION, dynEntity.getUuid(), "test" );
+ indexBucketLocator.getBucket( dynEntity.getUuid());
Object index_name = key( appId, "tests", "test", bucketId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java
index 1b54495..83f8df4 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java
@@ -31,15 +31,15 @@ import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
-import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+
import com.yammer.metrics.Metrics;
import com.yammer.metrics.core.MetricPredicate;
import com.yammer.metrics.core.Timer;
@@ -60,8 +60,9 @@ import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_IND
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.IndexUpdate.indexValueCode;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.dce;
import static org.apache.usergrid.utils.ConversionUtils.bytebuffers;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
@@ -262,7 +263,7 @@ public class EntityReadBenchMark extends ToolBase {
private boolean read( String value ) {
- List<String> buckets = indexBucketLocator.getBuckets( appId, IndexType.UNIQUE, "tests" );
+ List<String> buckets = indexBucketLocator.getBuckets( );
List<Object> cassKeys = new ArrayList<Object>( buckets.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 6b669b9..994e850 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -29,13 +29,18 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.Identifier;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
@@ -45,11 +50,6 @@ import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
import org.apache.usergrid.persistence.query.ir.result.UUIDIndexSliceParser;
import org.apache.usergrid.persistence.schema.CollectionInfo;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.AbstractComposite.ComponentEquality;
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -64,10 +64,10 @@ import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_IND
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.CassandraService.INDEX_ENTRY_LIST_COUNT;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.utils.CompositeUtils.setEqualityFlag;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
@@ -177,123 +177,129 @@ public class UniqueIndexCleanup extends ToolBase {
for ( String collectionName : getCollectionNames( em, line ) ) {
- IndexScanner scanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
- key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
- indexBucketLocator, applicationId, collectionName, false );
+ for ( final String bucketName : indexBucketLocator.getBuckets() ) {
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser( uuidCursorGenerator ) );
+ IndexScanner scanner =
+ cass.getIdList( key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null,
+ PAGE_SIZE, false, bucketName, applicationId, false );
+ SliceIterator itr = new SliceIterator( scanner, new UUIDIndexSliceParser( null ) );
- while ( itr.hasNext() ) {
- Set<ScanColumn> ids = itr.next();
+ while ( itr.hasNext() ) {
- CollectionInfo collection = getDefaultSchema().getCollection( "application", collectionName );
+ Set<ScanColumn> ids = itr.next();
+ CollectionInfo collection = getDefaultSchema().getCollection( "application", collectionName );
- //We shouldn't have to do this, but otherwise the cursor won't work
- Set<String> indexed = collection.getPropertiesIndexed();
- // what's left needs deleted, do so
+ //We shouldn't have to do this, but otherwise the cursor won't work
+ Set<String> indexed = collection.getPropertiesIndexed();
- logger.info( "Auditing {} entities for collection {} in app {}", new Object[] {
- ids.size(), collectionName, app.getValue()
- } );
+ // what's left needs deleted, do so
- for ( ScanColumn col : ids ) {
- final UUID id = col.getUUID();
- boolean reIndex = false;
+ logger.info( "Auditing {} entities for collection {} in app {}", new Object[] {
+ ids.size(), collectionName, app.getValue()
+ } );
- Mutator<ByteBuffer> m = createMutator( ko, be );
+ for ( ScanColumn col : ids ) {
+ final UUID id = col.getUUID();
+ boolean reIndex = false;
- try {
+ Mutator<ByteBuffer> m = createMutator( ko, be );
- for ( String prop : indexed ) {
+ try {
- String bucket =
- indexBucketLocator.getBucket( applicationId, IndexType.COLLECTION, id, prop );
+ for ( String prop : indexed ) {
- Object rowKey = key( applicationId, collection.getName(), prop, bucket );
+ String bucket = indexBucketLocator
+ .getBucket( id );
- List<HColumn<ByteBuffer, ByteBuffer>> indexCols =
- scanIndexForAllTypes( ko, indexBucketLocator, applicationId, rowKey, id, prop );
+ Object rowKey = key( applicationId, collection.getName(), prop, bucket );
- // loop through the indexed values and verify them as present in
- // our entity_index_entries. If they aren't, we need to delete the
- // from the secondary index, and mark
- // this object for re-index via n update
- for ( HColumn<ByteBuffer, ByteBuffer> index : indexCols ) {
+ List<HColumn<ByteBuffer, ByteBuffer>> indexCols =
+ scanIndexForAllTypes( ko, indexBucketLocator, applicationId, rowKey, id,
+ prop );
- DynamicComposite secondaryIndexValue =
- DynamicComposite.fromByteBuffer( index.getName().duplicate() );
+ // loop through the indexed values and verify them as present in
+ // our entity_index_entries. If they aren't, we need to delete the
+ // from the secondary index, and mark
+ // this object for re-index via n update
+ for ( HColumn<ByteBuffer, ByteBuffer> index : indexCols ) {
- Object code = secondaryIndexValue.get( 0 );
- Object propValue = secondaryIndexValue.get( 1 );
- UUID timestampId = ( UUID ) secondaryIndexValue.get( 3 );
+ DynamicComposite secondaryIndexValue = DynamicComposite.fromByteBuffer( index
+ .getName().duplicate() );
- DynamicComposite existingEntryStart =
- new DynamicComposite( prop, code, propValue, timestampId );
- DynamicComposite existingEntryFinish =
- new DynamicComposite( prop, code, propValue, timestampId );
+ Object code = secondaryIndexValue.get( 0 );
+ Object propValue = secondaryIndexValue.get( 1 );
+ UUID timestampId = ( UUID ) secondaryIndexValue.get( 3 );
- setEqualityFlag( existingEntryFinish, ComponentEquality.GREATER_THAN_EQUAL );
+ DynamicComposite existingEntryStart = new DynamicComposite( prop, code,
+ propValue, timestampId );
+ DynamicComposite existingEntryFinish = new DynamicComposite( prop, code,
+ propValue, timestampId );
- // now search our EntityIndexEntry for previous values, see if
- // they don't match this one
+ setEqualityFlag( existingEntryFinish, ComponentEquality.GREATER_THAN_EQUAL );
- List<HColumn<ByteBuffer, ByteBuffer>> entries =
- cass.getColumns( ko, ENTITY_INDEX_ENTRIES, id, existingEntryStart,
- existingEntryFinish, INDEX_ENTRY_LIST_COUNT, false );
+ // now search our EntityIndexEntry for previous values, see if
+ // they don't match this one
- // we wouldn't find this column in our entity_index_entries
- // audit. Delete it, then mark this entity for update
- if ( entries.size() == 0 ) {
- logger.info(
- "Could not find reference to value '{}' for property '{}' on entity " +
- "{} in collection {}. " + " Forcing reindex",
- new Object[] { propValue, prop, id, collectionName } );
+ List<HColumn<ByteBuffer, ByteBuffer>> entries =
+ cass.getColumns( ko, ENTITY_INDEX_ENTRIES, id, existingEntryStart,
+ existingEntryFinish, INDEX_ENTRY_LIST_COUNT, false );
- addDeleteToMutator( m, ENTITY_INDEX, rowKey, index.getName().duplicate(),
- timestamp );
+ // we wouldn't find this column in our entity_index_entries
+ // audit. Delete it, then mark this entity for update
+ if ( entries.size() == 0 ) {
+ logger.info(
+ "Could not find reference to value '{}' for property '{}' on entity "
+ +
+ "{} in collection {}. " + " Forcing reindex", new Object[] { propValue, prop, id, collectionName } );
- reIndex = true;
- }
+ addDeleteToMutator( m, ENTITY_INDEX, rowKey, index.getName().duplicate(),
+ timestamp );
+
+ reIndex = true;
+ }
- if ( entries.size() > 1 ) {
- logger.info(
- "Found more than 1 entity referencing unique index for property '{}' " +
- "with value " + "'{}'", prop, propValue );
- reIndex = true;
+ if ( entries.size() > 1 ) {
+ logger.info(
+ "Found more than 1 entity referencing unique index for property "
+ + "'{}' "
+ +
+ "with value " + "'{}'", prop, propValue );
+ reIndex = true;
+ }
}
}
- }
- //force this entity to be updated
- if ( reIndex ) {
- Entity entity = em.get( id );
-
- //entity may not exist, but we should have deleted rows from the index
- if ( entity == null ) {
- logger.warn( "Entity with id {} did not exist in app {}", id, applicationId );
- //now execute the cleanup. In this case the entity is gone,
- // so we'll want to remove references from
- // the secondary index
- m.execute();
- continue;
- }
+ //force this entity to be updated
+ if ( reIndex ) {
+ Entity entity = em.get( id );
+
+ //entity may not exist, but we should have deleted rows from the index
+ if ( entity == null ) {
+ logger.warn( "Entity with id {} did not exist in app {}", id, applicationId );
+ //now execute the cleanup. In this case the entity is gone,
+ // so we'll want to remove references from
+ // the secondary index
+ m.execute();
+ continue;
+ }
- logger.info( "Reindex complete for entity with id '{} ", id );
- em.update( entity );
+ logger.info( "Reindex complete for entity with id '{} ", id );
+ em.update( entity );
- //now execute the cleanup. This way if the above update fails,
- // we still have enough data to run again
- // later
- m.execute();
+ //now execute the cleanup. This way if the above update fails,
+ // we still have enough data to run again
+ // later
+ m.execute();
+ }
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to process entity with id '{}'", id, e );
}
- }
- catch ( Exception e ) {
- logger.error( "Unable to process entity with id '{}'", id, e );
}
}
}
[17/25] incubator-usergrid git commit: Updates tests to be cleaner.
Posted by sn...@apache.org.
Updates tests to be cleaner.
Fixes stack trace on exception
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b2e1fe68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b2e1fe68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b2e1fe68
Branch: refs/heads/master
Commit: b2e1fe686a11f87e0de4abbe043e7bb28f9f6083
Parents: b14a9ae
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jul 8 11:36:53 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jul 8 11:36:53 2015 -0600
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 2 +-
.../query/ir/result/GatherIterator.java | 2 +-
.../collection/activities/OrderByTest.java | 24 ++++++++++++++------
3 files changed, 19 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b2e1fe68/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 0a4f218..1aeb67f 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -32,7 +32,7 @@
cassandra.url=localhost:9160
# The number of thrift connections to open per cassandra node.
-cassandra.connections=50
+cassandra.connections=100
# Name of Cassandra cluster
cassandra.cluster=Test Cluster
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b2e1fe68/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index f4ffcd2..4187ae0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -93,7 +93,7 @@ public class GatherIterator implements ResultIterator {
future.get();
}
catch ( Exception e ) {
- throw new RuntimeException( "Unable to aggregate results" );
+ throw new RuntimeException( "Unable to aggregate results", e );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b2e1fe68/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/OrderByTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/OrderByTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/OrderByTest.java
index c9e0143..23c3e23 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/OrderByTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/OrderByTest.java
@@ -89,20 +89,30 @@ public class OrderByTest extends AbstractRestIT {
props.put( "verb", "go" );
props.put( "content", "bragh" );
+ final List<JsonNode> saved = new ArrayList<JsonNode>( 20 );
+
for ( int i = 0; i < 20; i++ ) {
props.put( "ordinal", i );
JsonNode activity = activities.create( props );
+
+ saved.add( getEntity( activity, 0 ) );
}
String query = "select * where created > " + 1 + " order by created desc";
- JsonNode incorrectNode = activities.withQuery( query ).withLimit( 5 ).get();
+ JsonNode results = activities.withQuery( query ).withLimit( 5 ).get();
+
+ assertEquals( 5, results.get( "entities" ).size() );
+
- assertEquals( 5, incorrectNode.get( "entities" ).size() );
while ( checkResultsNum < 5 ) {
- assertEquals( activities.entityIndex( query, checkResultsNum ),
- activities.entityIndexLimit( query, 5, checkResultsNum ) );
+
+
+ final JsonNode expected = saved.get( saved.size() - checkResultsNum -1 );
+ final JsonNode returned = getEntity( results, checkResultsNum );
+
+ assertEquals(expected , returned);
checkResultsNum++;
}
}
@@ -119,7 +129,7 @@ public class OrderByTest extends AbstractRestIT {
CustomCollection activities = context.collection( "activities" );
- int size = 200;
+ int size = 50;
Map<String, String> actor = hashMap( "displayName", "Erin" );
Map<String, Object> props = new HashMap<String, Object>();
@@ -138,12 +148,12 @@ public class OrderByTest extends AbstractRestIT {
long lastCreated = activites.get( activites.size() - 1 ).get( "created" ).asLong();
- String errorQuery = String.format( "select * where created <= %d order by created desc", lastCreated );
+ String query = String.format( "select * where created <= %d order by created desc", lastCreated );
String cursor = null;
int index = size - 1;
do {
- JsonNode response = activities.withQuery( errorQuery ).get();
+ JsonNode response = activities.withQuery( query ).get();
JsonNode cursorNode = response.get( "cursor" );
cursor = cursorNode != null ? cursorNode.asText() : null;
[14/25] incubator-usergrid git commit: Fixes but with comparator
ordering and uuid comparison when secondary compare fails
Posted by sn...@apache.org.
Fixes but with comparator ordering and uuid comparison when secondary compare fails
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/501da721
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/501da721
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/501da721
Branch: refs/heads/master
Commit: 501da721b6f0e2ec45c94f5e21d7fad723121dc1
Parents: 84d7760
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Jul 6 12:33:05 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Jul 6 12:33:05 2015 -0600
----------------------------------------------------------------------
.../ir/result/SecondaryIndexSliceParser.java | 93 ++++++++++++--------
1 file changed, 56 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/501da721/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
index d309c3d..bd1cc9f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
@@ -100,7 +100,7 @@ public class SecondaryIndexSliceParser implements SliceParser {
*/
public SecondaryIndexColumn( final UUID uuid, final Object value, final ByteBuffer columnNameBuffer,
final Comparator<SecondaryIndexColumn> valueComparator,
- final SliceCursorGenerator sliceCursorGenerator) {
+ final SliceCursorGenerator sliceCursorGenerator ) {
super( uuid, columnNameBuffer, sliceCursorGenerator );
this.value = value;
this.valueComparator = valueComparator;
@@ -130,24 +130,24 @@ public class SecondaryIndexSliceParser implements SliceParser {
static {
final LongComparator longComparator = new LongComparator();
- COMPARATOR_MAP.put( new MapKey( Long.class, false ), longComparator );
- COMPARATOR_MAP.put( new MapKey( Long.class, true ), new ReverseComparator( longComparator ) );
+ COMPARATOR_MAP.put( new MapKey( Long.class, false ), new ForwardValueComparator( longComparator ) );
+ COMPARATOR_MAP.put( new MapKey( Long.class, true ), new ReverseValueComparator( longComparator ) );
final StringComparator stringComparator = new StringComparator();
- COMPARATOR_MAP.put( new MapKey( String.class, false ), stringComparator );
- COMPARATOR_MAP.put( new MapKey( String.class, true ), new ReverseComparator( stringComparator ) );
+ COMPARATOR_MAP.put( new MapKey( String.class, false ), new ForwardValueComparator( stringComparator ) );
+ COMPARATOR_MAP.put( new MapKey( String.class, true ), new ReverseValueComparator( stringComparator ) );
final UUIDComparator uuidComparator = new UUIDComparator();
- COMPARATOR_MAP.put( new MapKey( UUID.class, false ), uuidComparator );
- COMPARATOR_MAP.put( new MapKey( UUID.class, true ), new ReverseComparator( uuidComparator ) );
+ COMPARATOR_MAP.put( new MapKey( UUID.class, false ), new ForwardValueComparator( uuidComparator ) );
+ COMPARATOR_MAP.put( new MapKey( UUID.class, true ), new ReverseValueComparator( uuidComparator ) );
final BigIntegerComparator bigIntegerComparator = new BigIntegerComparator();
- COMPARATOR_MAP.put( new MapKey( BigInteger.class, false ), bigIntegerComparator );
- COMPARATOR_MAP.put( new MapKey( BigInteger.class, true ), new ReverseComparator( bigIntegerComparator ) );
+ COMPARATOR_MAP.put( new MapKey( BigInteger.class, false ), new ForwardValueComparator( bigIntegerComparator ) );
+ COMPARATOR_MAP.put( new MapKey( BigInteger.class, true ), new ReverseValueComparator( bigIntegerComparator ) );
}
@@ -192,23 +192,7 @@ public class SecondaryIndexSliceParser implements SliceParser {
}
- private static abstract class SecondaryIndexColumnComparator implements Comparator<SecondaryIndexColumn> {
-
- /**
- * If the result of compare is != 0 it is returned, otherwise the uuids are compared
- */
- protected int compareValues( final int compare, final SecondaryIndexColumn first,
- SecondaryIndexColumn second ) {
- if ( compare != 0 ) {
- return compare;
- }
-
- return com.fasterxml.uuid.UUIDComparator.staticCompare( first.uuid, second.uuid );
- }
- }
-
-
- private static final class LongComparator extends SecondaryIndexColumnComparator {
+ private static final class LongComparator implements Comparator<SecondaryIndexColumn> {
@Override
public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
@@ -217,12 +201,12 @@ public class SecondaryIndexSliceParser implements SliceParser {
final Long secondLong = ( Long ) second.value;
- return compareValues( Long.compare( firstLong, secondLong ), first, second );
+ return Long.compare( firstLong, secondLong );
}
}
- private static final class StringComparator extends SecondaryIndexColumnComparator {
+ private static final class StringComparator implements Comparator<SecondaryIndexColumn> {
@Override
public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
@@ -234,49 +218,84 @@ public class SecondaryIndexSliceParser implements SliceParser {
final String secondString = ( String ) second.value;
- return compareValues( firstString.compareTo( secondString ), first, second );
+ return firstString.compareTo( secondString );
}
}
- private static final class UUIDComparator extends SecondaryIndexColumnComparator {
+ private static final class UUIDComparator implements Comparator<SecondaryIndexColumn> {
@Override
public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
final UUID firstUUID = ( UUID ) first.value;
final UUID secondUUID = ( UUID ) second.value;
- return compareValues( UUIDUtils.compare( firstUUID, secondUUID ), first, second );
+ return UUIDUtils.compare( firstUUID, secondUUID );
}
}
- private static final class BigIntegerComparator extends SecondaryIndexColumnComparator {
+ private static final class BigIntegerComparator implements Comparator<SecondaryIndexColumn> {
@Override
public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
final BigInteger firstInt = ( BigInteger ) first.value;
final BigInteger secondInt = ( BigInteger ) second.value;
- return compareValues( firstInt.compareTo( secondInt ), first, second );
+ return firstInt.compareTo( secondInt );
+ }
+ }
+
+
+ /**
+ * Delegates to the type comparator, if equal, sorts by UUID ascending always
+ */
+ private static final class ForwardValueComparator implements Comparator<SecondaryIndexColumn> {
+
+ private final Comparator<SecondaryIndexColumn> comparator;
+
+
+ private ForwardValueComparator( final Comparator<SecondaryIndexColumn> comparator ) {
+ this.comparator = comparator;
+ }
+
+
+ @Override
+ public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
+
+ int compare = comparator.compare( first, second );
+
+ if ( compare == 0 ) {
+ return com.fasterxml.uuid.UUIDComparator.staticCompare( first.uuid, second.uuid );
+ }
+
+ return compare;
}
}
/**
- * Reversed our comparator
+ * Reversed our delegate comparator, if equal, compares by uuid ascending
*/
- private static final class ReverseComparator implements Comparator<SecondaryIndexColumn> {
+ private static final class ReverseValueComparator implements Comparator<SecondaryIndexColumn> {
private final Comparator<SecondaryIndexColumn> comparator;
- private ReverseComparator( final Comparator<SecondaryIndexColumn> comparator ) {this.comparator = comparator;}
+ private ReverseValueComparator( final Comparator<SecondaryIndexColumn> comparator ) {
+ this.comparator = comparator;
+ }
@Override
public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
- return comparator.compare( first, second ) * -1;
+ int compare = comparator.compare( first, second ) * -1;
+
+ if ( compare == 0 ) {
+ return com.fasterxml.uuid.UUIDComparator.staticCompare( first.uuid, second.uuid );
+ }
+
+ return compare;
}
}
}
[16/25] incubator-usergrid git commit: First pass at concurrent merge
Posted by sn...@apache.org.
First pass at concurrent merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b14a9ae1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b14a9ae1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b14a9ae1
Branch: refs/heads/master
Commit: b14a9ae144e1a2836a4f3c96479851469879ded7
Parents: 4444803
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jul 7 12:05:22 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jul 7 12:05:22 2015 -0600
----------------------------------------------------------------------
.../persistence/cassandra/QueryProcessor.java | 30 +++-
.../query/ir/result/GatherIterator.java | 162 ++++++++++---------
2 files changed, 119 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b14a9ae1/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
index 32fc07c..a2e0f60 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
@@ -22,6 +22,12 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -88,6 +94,11 @@ public class QueryProcessor {
private final EntityManager em;
private final ResultsLoaderFactory loaderFactory;
+ //we set up a thread pool with 100 execution threads. We purposefully use a synchronous queue and a caller runs so that we simply reject and immediately execute tasks if all 100 threads are occupied.
+
+ //we make this static, we only want 1 instance of this service in the JVM. Otherwise tests fail
+ private static final ExecutorService executorService = new ThreadPoolExecutor( 100, 100, 30, TimeUnit.SECONDS, new SynchronousQueue<Runnable>( ), new CallerRunsExecutionHandler() );
+
private Operand rootOperand;
private List<SortPredicate> sorts;
private CursorCache cursorCache;
@@ -270,7 +281,7 @@ public class QueryProcessor {
//use the gather iterator to collect all the '
final int resultSetSize = Math.min( size, Query.MAX_LIMIT );
- ResultIterator itr = new GatherIterator(resultSetSize, rootNode, searchVisitorFactory.createVisitors() );
+ ResultIterator itr = new GatherIterator(resultSetSize, rootNode, searchVisitorFactory.createVisitors(), executorService );
List<ScanColumn> entityIds = new ArrayList<ScanColumn>( );
@@ -683,4 +694,21 @@ public class QueryProcessor {
public EntityManager getEntityManager() {
return em;
}
+
+
+ private static class CallerRunsExecutionHandler implements RejectedExecutionHandler{
+
+ private static final Logger logger = LoggerFactory.getLogger( CallerRunsExecutionHandler.class );
+
+ @Override
+ public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
+ //when a task is rejected due to back pressure, we just want to run it in the caller.
+
+ logger.warn( "Concurrent shard execution rejected the task in executor {}, running it in the caller thread", executor );
+
+ r.run();
+ }
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b14a9ae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index e199d33..f4ffcd2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -19,19 +19,16 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
-import java.util.TreeMap;
import java.util.TreeSet;
-import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
-import org.apache.usergrid.persistence.ResultsIterator;
-import org.apache.usergrid.persistence.cassandra.CursorCache;
import org.apache.usergrid.persistence.query.ir.QueryNode;
import org.apache.usergrid.persistence.query.ir.SearchVisitor;
@@ -42,18 +39,27 @@ import org.apache.usergrid.persistence.query.ir.SearchVisitor;
public class GatherIterator implements ResultIterator {
- private final QueryNode rootNode;
- private final int pageSize;
+ private List<Future<Void>> iterators;
+ private final ConcurrentResultMerge merge;
+ private boolean merged;
- private Iterator<ScanColumn> mergedIterators;
- private List<ResultIterator> iterators;
+ public GatherIterator( final int pageSize, final QueryNode rootNode, final Collection<SearchVisitor> searchVisitors,
+ final ExecutorService executorService ) {
+ this.merge = new ConcurrentResultMerge( pageSize );
- public GatherIterator(final int pageSize, final QueryNode rootNode, final Collection<SearchVisitor> searchVisitors) {
- this.pageSize = pageSize;
- this.rootNode = rootNode;
- createIterators( searchVisitors );
+ this.iterators = new ArrayList<Future<Void>>( searchVisitors.size() );
+ this.merged = false;
+
+
+ /**
+ * Start our search processing
+ */
+ for ( SearchVisitor visitor : searchVisitors ) {
+ final Future<Void> result = executorService.submit( new VisitorExecutor( rootNode, merge, visitor ) );
+ iterators.add( result );
+ }
}
@@ -63,7 +69,6 @@ public class GatherIterator implements ResultIterator {
}
-
@Override
public Iterator<Set<ScanColumn>> iterator() {
return this;
@@ -72,106 +77,119 @@ public class GatherIterator implements ResultIterator {
@Override
public boolean hasNext() {
- if(mergedIterators == null || !mergedIterators.hasNext()){
- mergeIterators();
- }
+ waitForCompletion();
- return mergedIterators.hasNext();
+ return merge.results.size() > 0;
}
- @Override
- public Set<ScanColumn> next() {
- if(!hasNext()){
- throw new NoSuchElementException( "No more elements" );
+ private void waitForCompletion() {
+ if ( merged ) {
+ return;
}
- return getNextPage();
- }
-
- private void createIterators(final Collection<SearchVisitor> searchVisitors ){
-
- this.iterators = new ArrayList<ResultIterator>( searchVisitors.size() );
-
- for(SearchVisitor visitor: searchVisitors){
-
+ for ( final Future<Void> future : iterators ) {
try {
- rootNode.visit( visitor );
+ future.get();
}
catch ( Exception e ) {
- throw new RuntimeException( "Unable to process query", e );
+ throw new RuntimeException( "Unable to aggregate results" );
}
+ }
- final ResultIterator iterator = visitor.getResults();
+ merged = true;
+ }
- iterators.add( iterator );
+ @Override
+ public Set<ScanColumn> next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No more elements" );
}
+ return merge.copyAndClear();
}
/**
- * Get the next page of results
- * @return
+ * A visitor that will visit and get the first page of an set and return them.
*/
- private Set<ScanColumn> getNextPage(){
+ private final class VisitorExecutor implements Callable<Void> {
+
+ private final QueryNode rootNode;
+ private final SearchVisitor visitor;
+ private final ConcurrentResultMerge merge;
- //try to take from our PageSize
- LinkedHashSet<ScanColumn> resultSet = new LinkedHashSet<ScanColumn>( pageSize );
- for(int i = 0; i < pageSize && mergedIterators.hasNext(); i ++){
- resultSet.add( mergedIterators.next() );
+ private VisitorExecutor( final QueryNode rootNode, final ConcurrentResultMerge merge,
+ final SearchVisitor visitor ) {
+ this.rootNode = rootNode;
+ this.visitor = visitor;
+ this.merge = merge;
}
- return resultSet;
- }
+ @Override
+ public Void call() throws Exception {
- /**
- * Advance the iterator
- */
- private void mergeIterators(){
- //TODO make this concurrent
+ try {
+ rootNode.visit( visitor );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to process query", e );
+ }
- TreeSet<ScanColumn> merged = new TreeSet<ScanColumn>( );
+ final ResultIterator iterator = visitor.getResults();
- for(ResultIterator iterator: this.iterators){
- merge(merged, iterator);
- }
- mergedIterators = merged.iterator();
+ if ( iterator.hasNext() ) {
+ merge.merge( iterator.next() );
+ }
+ return null;
+ }
}
-
-
/**
- * Merge this interator into our final column results
- * @param results
- * @param iterator
+ * Class used to synchronize our treeSet access
*/
- private void merge(final TreeSet<ScanColumn> results, final ResultIterator iterator){
+ private final class ConcurrentResultMerge {
+ private final TreeSet<ScanColumn> results;
+ private final int maxSize;
- //nothing to do, return
- if( !iterator.hasNext()){
- return;
- }
- final Iterator<ScanColumn> nextPage = iterator.next().iterator();
+ private ConcurrentResultMerge( final int maxSize ) {
+ this.maxSize = maxSize;
+ results = new TreeSet<ScanColumn>();
+ }
- //only take from the iterator what we need to create a full page.
- for(int i = 0 ; i < pageSize && nextPage.hasNext(); i ++){
- final ScanColumn next = nextPage.next();
- results.add(next);
+ /**
+ * Merge this set into the existing set
+ */
+ public synchronized void merge( final Set<ScanColumn> columns ) {
+ for ( ScanColumn scanColumn : columns ) {
+ results.add( scanColumn );
+ //results are too large remove the last
+ while ( results.size() > maxSize ) {
+ results.pollLast();
+ }
+ }
}
- }
-
+ /**
+ * Get the set
+ */
+ public Set<ScanColumn> copyAndClear() {
+ //create an immutable copy
+ final Set<ScanColumn> toReturn = new LinkedHashSet<ScanColumn>( results );
+ results.clear();
+ return toReturn;
+ }
+ }
}
[03/25] incubator-usergrid git commit: Added comparator chain so that
we can merge correctly
Posted by sn...@apache.org.
Added comparator chain so that we can merge correctly
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1fe69065
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1fe69065
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1fe69065
Branch: refs/heads/master
Commit: 1fe69065b299797e1c2b40b59fa2fc567ed1f5e6
Parents: aa31768
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jun 17 17:07:34 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jun 17 17:07:34 2015 -0600
----------------------------------------------------------------------
.../persistence/cassandra/QueryProcessor.java | 7 +-
.../cassandra/RelationManagerImpl.java | 30 +++++---
.../index/DynamicCompositeComparator.java | 43 +++++++++++
.../DynamicCompositeForwardComparator.java | 40 +++++++++++
.../DynamicCompositeReverseComparator.java | 39 ++++++++++
.../index/IndexMultiBucketSetLoader.java | 55 --------------
.../cassandra/index/NoOpIndexScanner.java | 7 +-
.../persistence/geo/EntityLocationRef.java | 21 ++++--
.../persistence/query/ir/SearchVisitor.java | 2 +-
.../query/ir/result/AbstractScanColumn.java | 42 ++++++++++-
.../ir/result/ConnectionIndexSliceParser.java | 10 +--
.../result/ConnectionSearchVisitorFactory.java | 2 +-
.../query/ir/result/GatherIterator.java | 76 +++++++++++++++++++-
.../query/ir/result/GeoIterator.java | 24 +++++++
.../persistence/query/ir/result/ScanColumn.java | 23 +++++-
.../ir/result/SearchCollectionVisitor.java | 2 +-
.../ir/result/SearchConnectionVisitor.java | 2 +-
.../ir/result/SecondaryIndexSliceParser.java | 16 +++--
.../query/ir/result/SliceParser.java | 4 +-
.../query/ir/result/StaticIdIterator.java | 2 +-
.../persistence/query/ir/result/UUIDColumn.java | 50 +++++++++++++
.../query/ir/result/UUIDIndexSliceParser.java | 13 ++--
.../query/ir/result/UnionIterator.java | 42 ++---------
.../query/ir/result/AbstractScanColumnTest.java | 2 +-
24 files changed, 411 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
index 78392a4..f55aa67 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
@@ -267,11 +267,12 @@ public class QueryProcessor {
return null;
}
- //use the gather iterator to collect all the
+ //use the gather iterator to collect all the '
+ final int resultSetSize = Math.min( size, Query.MAX_LIMIT );
- ResultIterator itr = new GatherIterator(rootNode, searchVisitorFactory.createVisitors() );
+ ResultIterator itr = new GatherIterator(resultSetSize, rootNode, searchVisitorFactory.createVisitors() );
- List<ScanColumn> entityIds = new ArrayList<ScanColumn>( Math.min( size, Query.MAX_LIMIT ) );
+ List<ScanColumn> entityIds = new ArrayList<ScanColumn>( );
CursorCache resultsCursor = new CursorCache();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index 0e5011f..977dad0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -58,7 +58,9 @@ import org.apache.usergrid.persistence.geo.EntityLocationRef;
import org.apache.usergrid.persistence.hector.CountingMutator;
import org.apache.usergrid.persistence.query.ir.QuerySlice;
import org.apache.usergrid.persistence.query.ir.result.CollectionResultsLoaderFactory;
+import org.apache.usergrid.persistence.query.ir.result.CollectionSearchVisitorFactory;
import org.apache.usergrid.persistence.query.ir.result.ConnectionResultsLoaderFactory;
+import org.apache.usergrid.persistence.query.ir.result.ConnectionSearchVisitorFactory;
import org.apache.usergrid.persistence.query.ir.result.ConnectionTypesIterator;
import org.apache.usergrid.persistence.query.ir.result.SearchCollectionVisitor;
import org.apache.usergrid.persistence.query.ir.result.SearchConnectionVisitor;
@@ -1720,9 +1722,11 @@ public class RelationManagerImpl implements RelationManager {
// we have something to search with, visit our tree and evaluate the
// results
QueryProcessor qp = new QueryProcessor( query, collection, em, factory );
- SearchCollectionVisitor visitor = new SearchCollectionVisitor( this, qp );
- return qp.getResults( visitor );
+ CollectionSearchVisitorFactory collectionSearchVisitorFactory = new CollectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, collectionName );
+// SearchCollectionVisitor visitor = new SearchCollectionVisitor( this, qp );
+
+ return qp.getResults( collectionSearchVisitorFactory );
}
@@ -1914,9 +1918,12 @@ public class RelationManagerImpl implements RelationManager {
final ConnectionResultsLoaderFactory factory = new ConnectionResultsLoaderFactory( connectionRef );
QueryProcessor qp = new QueryProcessor( query, null, em, factory );
- SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, true );
- return qp.getResults( visitor );
+ ConnectionSearchVisitorFactory collectionSearchVisitorFactory = new ConnectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, true, "" );
+
+// SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, true );
+
+ return qp.getResults( collectionSearchVisitorFactory );
}
@@ -1956,9 +1963,13 @@ public class RelationManagerImpl implements RelationManager {
final ConnectionResultsLoaderFactory factory = new ConnectionResultsLoaderFactory( connectionRef );
QueryProcessor qp = new QueryProcessor( query, null, em, factory );
- SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, false );
- return qp.getResults( visitor );
+
+ ConnectionSearchVisitorFactory collectionSearchVisitorFactory = new ConnectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, false, "" );
+
+// SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, false );
+
+ return qp.getResults( collectionSearchVisitorFactory );
}
@@ -1995,9 +2006,12 @@ public class RelationManagerImpl implements RelationManager {
final ConnectionResultsLoaderFactory factory = new ConnectionResultsLoaderFactory( connectionRef );
QueryProcessor qp = new QueryProcessor( query, null, em, factory );
- SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, true );
- return qp.getResults( visitor );
+ ConnectionSearchVisitorFactory collectionSearchVisitorFactory = new ConnectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, false, "" );
+
+// SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, true );
+
+ return qp.getResults( collectionSearchVisitorFactory );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeComparator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeComparator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeComparator.java
new file mode 100644
index 0000000..436c3fd
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeComparator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.cassandra.index;
+
+
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.TypeParser;
+
+import org.apache.usergrid.persistence.cassandra.ApplicationCF;
+
+
+public abstract class DynamicCompositeComparator implements Comparator<ByteBuffer> {
+ @SuppressWarnings("rawtypes")
+ protected final AbstractType dynamicComposite;
+
+
+ protected DynamicCompositeComparator( ApplicationCF cf ) {
+ // should never happen, this will blow up during development if this fails
+ try {
+ dynamicComposite = TypeParser.parse( cf.getComparator() );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( e );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeForwardComparator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeForwardComparator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeForwardComparator.java
new file mode 100644
index 0000000..a61d422
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeForwardComparator.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.cassandra.index;
+
+
+import java.nio.ByteBuffer;
+
+import org.apache.usergrid.persistence.cassandra.ApplicationCF;
+
+
+class DynamicCompositeForwardComparator extends DynamicCompositeComparator {
+
+ /**
+ * @param cf
+ */
+ protected DynamicCompositeForwardComparator( ApplicationCF cf ) {
+ super( cf );
+ }
+
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public int compare( ByteBuffer o1, ByteBuffer o2 ) {
+ return dynamicComposite.compare( o1, o2 );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeReverseComparator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeReverseComparator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeReverseComparator.java
new file mode 100644
index 0000000..4ff333a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeReverseComparator.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.cassandra.index;
+
+
+import java.nio.ByteBuffer;
+
+import org.apache.usergrid.persistence.cassandra.ApplicationCF;
+
+
+class DynamicCompositeReverseComparator extends DynamicCompositeComparator {
+ /**
+ * @param cf
+ */
+ protected DynamicCompositeReverseComparator( ApplicationCF cf ) {
+ super( cf );
+ }
+
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public int compare( ByteBuffer o1, ByteBuffer o2 ) {
+ return dynamicComposite.compare( o2, o1 );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexMultiBucketSetLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexMultiBucketSetLoader.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexMultiBucketSetLoader.java
index 30b54ba..a90f890 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexMultiBucketSetLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexMultiBucketSetLoader.java
@@ -27,9 +27,6 @@ import java.util.UUID;
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.TypeParser;
-
import me.prettyprint.hector.api.beans.HColumn;
@@ -84,56 +81,4 @@ public class IndexMultiBucketSetLoader {
return resultsTree;
}
-
-
- private static abstract class DynamicCompositeComparator implements Comparator<ByteBuffer> {
- @SuppressWarnings("rawtypes")
- protected final AbstractType dynamicComposite;
-
-
- protected DynamicCompositeComparator( ApplicationCF cf ) {
- // should never happen, this will blow up during development if this fails
- try {
- dynamicComposite = TypeParser.parse( cf.getComparator() );
- }
- catch ( Exception e ) {
- throw new RuntimeException( e );
- }
- }
- }
-
-
- private static class DynamicCompositeForwardComparator extends DynamicCompositeComparator {
-
- /**
- * @param cf
- */
- protected DynamicCompositeForwardComparator( ApplicationCF cf ) {
- super( cf );
- }
-
-
- @SuppressWarnings("unchecked")
- @Override
- public int compare( ByteBuffer o1, ByteBuffer o2 ) {
- return dynamicComposite.compare( o1, o2 );
- }
- }
-
-
- private static class DynamicCompositeReverseComparator extends DynamicCompositeComparator {
- /**
- * @param cf
- */
- protected DynamicCompositeReverseComparator( ApplicationCF cf ) {
- super( cf );
- }
-
-
- @SuppressWarnings("unchecked")
- @Override
- public int compare( ByteBuffer o1, ByteBuffer o2 ) {
- return dynamicComposite.compare( o2, o1 );
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/NoOpIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/NoOpIndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/NoOpIndexScanner.java
index 3d1b9d7..01002d6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/NoOpIndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/NoOpIndexScanner.java
@@ -19,8 +19,7 @@ package org.apache.usergrid.persistence.cassandra.index;
import java.nio.ByteBuffer;
import java.util.Iterator;
-import java.util.NavigableSet;
-import java.util.Set;
+import java.util.List;
import me.prettyprint.hector.api.beans.HColumn;
@@ -44,7 +43,7 @@ public class NoOpIndexScanner implements IndexScanner {
* @see java.lang.Iterable#iterator()
*/
@Override
- public Iterator<Set<HColumn<ByteBuffer, ByteBuffer>>> iterator() {
+ public Iterator<List<HColumn<ByteBuffer, ByteBuffer>>> iterator() {
return this;
}
@@ -71,7 +70,7 @@ public class NoOpIndexScanner implements IndexScanner {
* @see java.util.Iterator#next()
*/
@Override
- public NavigableSet<HColumn<ByteBuffer, ByteBuffer>> next() {
+ public List<HColumn<ByteBuffer, ByteBuffer>> next() {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/geo/EntityLocationRef.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/EntityLocationRef.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/EntityLocationRef.java
index 59db1d9..05db619 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/EntityLocationRef.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/EntityLocationRef.java
@@ -30,7 +30,7 @@ import static org.apache.usergrid.utils.StringUtils.stringOrSubstringAfterLast;
import static org.apache.usergrid.utils.StringUtils.stringOrSubstringBeforeFirst;
-public class EntityLocationRef implements EntityRef {
+public class EntityLocationRef implements EntityRef, Comparable<EntityLocationRef> {
private UUID uuid;
@@ -45,10 +45,6 @@ public class EntityLocationRef implements EntityRef {
private double distance;
- public EntityLocationRef() {
- }
-
-
public EntityLocationRef( EntityRef entity, double latitude, double longitude ) {
this( entity.getType(), entity.getUuid(), latitude, longitude );
}
@@ -224,4 +220,19 @@ public class EntityLocationRef implements EntityRef {
}
return true;
}
+
+
+ /**
+ * Compares 2 locations by comparing their distance
+ * @param other
+ * @return
+ */
+ @Override
+ public int compareTo( final EntityLocationRef other ) {
+ if(other == null){
+ return 1;
+ }
+
+ return Double.compare( distance, other.distance );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
index 0512cb2..410b99f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
@@ -162,7 +162,7 @@ public abstract class SearchVisitor implements NodeVisitor {
final int nodeId = node.getId();
- UnionIterator union = new UnionIterator( queryProcessor.getPageSizeHint( node ), nodeId, queryProcessor.getCursorCache(nodeId ) );
+ UnionIterator union = new UnionIterator( queryProcessor.getPageSizeHint( node ), nodeId, queryProcessor.getCursorCache( nodeId ) );
if ( left != null ) {
union.addIterator( left );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
index fc4a1d6..e4515fb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
@@ -21,6 +21,8 @@ import java.nio.ByteBuffer;
import java.util.UUID;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
+
/**
*
@@ -31,11 +33,14 @@ public abstract class AbstractScanColumn implements ScanColumn {
private final UUID uuid;
private final ByteBuffer buffer;
+ private final DynamicCompositeComparator cfComparator;
+ private ScanColumn child;
- protected AbstractScanColumn( UUID uuid, ByteBuffer buffer ) {
+ protected AbstractScanColumn( final UUID uuid, final ByteBuffer columnNameBuffer, final DynamicCompositeComparator cfComparator ) {
this.uuid = uuid;
- this.buffer = buffer;
+ this.buffer = columnNameBuffer;
+ this.cfComparator = cfComparator;
}
@@ -80,4 +85,37 @@ public abstract class AbstractScanColumn implements ScanColumn {
", buffer=" + ByteBufferUtil.bytesToHex( buffer ) +
'}';
}
+
+
+ @Override
+ public void setChild( final ScanColumn childColumn ) {
+ this.child = childColumn;
+ }
+
+
+ @Override
+ public ScanColumn getChild() {
+ return child;
+ }
+
+
+ @Override
+ public int compareTo( final ScanColumn otherScanColumn ) {
+
+ if(otherScanColumn == null){
+ return 1;
+ }
+
+
+ final int compare = cfComparator.compare( buffer, otherScanColumn.getCursorValue() );
+
+ //equal, recurse. otherScanColumn is implicitly not null from above check
+ if(compare == 0 && child != null){
+ return child.compareTo( otherScanColumn.getChild() );
+ }
+
+
+ return 0;
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
index 33822f5..ae3b0b5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
@@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
import java.util.UUID;
import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -45,7 +46,7 @@ public class ConnectionIndexSliceParser implements SliceParser {
* @see org.apache.usergrid.persistence.query.ir.result.SliceParser#parse(java.nio.ByteBuffer)
*/
@Override
- public ScanColumn parse( ByteBuffer buff ) {
+ public ScanColumn parse( ByteBuffer buff, final DynamicCompositeComparator cfComparator ) {
DynamicComposite composite = DynamicComposite.fromByteBuffer( buff.duplicate() );
String connectedType = ( String ) composite.get( 1 );
@@ -62,19 +63,20 @@ public class ConnectionIndexSliceParser implements SliceParser {
return null;
}
- return new ConnectionColumn( ( UUID ) composite.get( 0 ), connectedType, buff );
+ return new ConnectionColumn( ( UUID ) composite.get( 0 ), connectedType, buff , cfComparator);
// return composite;
// return null;
}
+
public static class ConnectionColumn extends AbstractScanColumn {
private final String connectedType;
- public ConnectionColumn( UUID uuid, String connectedType, ByteBuffer column ) {
- super( uuid, column );
+ public ConnectionColumn( UUID uuid, String connectedType, ByteBuffer column, final DynamicCompositeComparator cfComparator ) {
+ super( uuid, column, cfComparator );
this.connectedType = connectedType;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
index c1ec724..54eeb01 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
@@ -42,7 +42,7 @@ public class ConnectionSearchVisitorFactory implements SearchVisitorFactory {
private final String[] prefix;
- private ConnectionSearchVisitorFactory( final CassandraService cassandraService,
+ public ConnectionSearchVisitorFactory( final CassandraService cassandraService,
final IndexBucketLocator indexBucketLocator,
final QueryProcessor queryProcessor, final UUID applicationId,
final EntityRef headEntity, ConnectionRefImpl connectionRef,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index 48e2301..29912cc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -3,9 +3,13 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.util.Collection;
import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.NoSuchElementException;
import java.util.Set;
+import java.util.TreeSet;
import java.util.UUID;
+import org.apache.usergrid.persistence.ResultsIterator;
import org.apache.usergrid.persistence.cassandra.CursorCache;
import org.apache.usergrid.persistence.query.ir.QueryNode;
import org.apache.usergrid.persistence.query.ir.SearchVisitor;
@@ -19,9 +23,14 @@ public class GatherIterator implements ResultIterator {
private final Collection<SearchVisitor> searchVisitors;
private final QueryNode rootNode;
+ private final int pageSize;
- public GatherIterator( final QueryNode rootNode, final Collection<SearchVisitor> searchVisitors) {
+ private Set<ScanColumn> next;
+
+
+ public GatherIterator(final int pageSize, final QueryNode rootNode, final Collection<SearchVisitor> searchVisitors) {
+ this.pageSize = pageSize;
this.rootNode = rootNode;
this.searchVisitors = searchVisitors;
}
@@ -47,12 +56,73 @@ public class GatherIterator implements ResultIterator {
@Override
public boolean hasNext() {
- return false;
+
+ if(next() == null){
+ advance();
+ }
+
+ return next != null;
}
@Override
public Set<ScanColumn> next() {
- return null;
+ if(!hasNext()){
+ throw new NoSuchElementException( "No more elements" );
+ }
+
+ final Set<ScanColumn> results = next;
+ next = null;
+ return results;
+ }
+
+
+ /**
+ * Advance the iterator
+ */
+ private void advance(){
+ //TODO make this concurrent
+
+
+ final TreeSet<ScanColumn> results = new TreeSet<ScanColumn>( );
+
+
+ for(SearchVisitor visitor: searchVisitors){
+ merge(results, visitor);
+ }
+
+ this.next = results;
+ }
+
+
+ /**
+ * Merge this interator into our final column results
+ * @param results
+ * @param visitor
+ */
+ private void merge(final TreeSet<ScanColumn> results, final SearchVisitor visitor){
+
+ final ResultIterator iterator = visitor.getResults();
+
+
+ //nothing to do, return
+ if( !iterator.hasNext()){
+ return;
+ }
+
+
+ final Iterator<ScanColumn> nextPage = iterator.next().iterator();
+
+
+ //only take from the iterator what we need to create a full page.
+ for(int i = 0 ; i < pageSize && nextPage.hasNext(); i ++){
+ results.add( nextPage.next() );
+
+ //results are too large, trim them
+ if(results.size() > pageSize){
+ results.pollLast();
+ }
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
index 92f6f03..4ecbb5a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
@@ -308,6 +308,7 @@ public class GeoIterator implements ResultIterator {
private class LocationScanColumn implements ScanColumn {
private final EntityLocationRef location;
+ private ScanColumn child;
public LocationScanColumn( EntityLocationRef location ) {
@@ -329,6 +330,18 @@ public class GeoIterator implements ResultIterator {
@Override
+ public void setChild( final ScanColumn childColumn ) {
+ this.child = childColumn;
+ }
+
+
+ @Override
+ public ScanColumn getChild() {
+ return this.child;
+ }
+
+
+ @Override
public boolean equals( Object o ) {
if ( this == o ) {
return true;
@@ -347,5 +360,16 @@ public class GeoIterator implements ResultIterator {
public int hashCode() {
return location.getUuid().hashCode();
}
+
+
+ @Override
+ public int compareTo( final ScanColumn o ) {
+
+ if(!(o instanceof LocationScanColumn)){
+ throw new UnsupportedOperationException( "Cannot compare another ScanColumn that is not an instance of LocationScanColumn" );
+ }
+
+ return this.location.compareTo( ((LocationScanColumn)o).location );
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
index 289fe86..18277e4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
@@ -20,13 +20,30 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.nio.ByteBuffer;
import java.util.UUID;
+import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
+
/** An interface that represents a column */
-public interface ScanColumn {
+public interface ScanColumn extends Comparable<ScanColumn> {
/** Get the uuid from the column */
- public UUID getUUID();
+ UUID getUUID();
/** Get the cursor value of this column */
- public ByteBuffer getCursorValue();
+ ByteBuffer getCursorValue();
+
+ /**
+ * Append the child column used in tree iterator to this column, along with the comparator used to compare them
+ *
+ * for instance, a term search of A = 1 AND B = 2 would generate a ScanColumn of A-- child -> B
+ * @param childColumn
+ */
+ void setChild( final ScanColumn childColumn );
+
+ /**
+ * Returns the childl column if present, can return null
+ * @return
+ */
+ ScanColumn getChild();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index 61315e4..20d303c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -95,7 +95,7 @@ public class SearchCollectionVisitor extends SearchVisitor {
UUID startId = null;
if ( slice.hasCursor() ) {
- startId = UUID_PARSER.parse( slice.getCursor() ).getUUID();
+ startId = UUID_PARSER.parse( slice.getCursor(), null ).getUUID();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index da99cdf..74ff19f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -82,7 +82,7 @@ public class SearchConnectionVisitor extends SearchVisitor {
// operation
queryProcessor.applyCursorAndSort( slice );
- IndexScanner columns = null;
+ final IndexScanner columns;
if ( slice.isComplete() ) {
columns = new NoOpIndexScanner();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
index ea093e6..c13ac0d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
@@ -20,6 +20,8 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.nio.ByteBuffer;
import java.util.UUID;
+import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
+
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -35,20 +37,22 @@ public class SecondaryIndexSliceParser implements SliceParser {
* @see org.apache.usergrid.persistence.query.ir.result.SliceParser#parse(java.nio.ByteBuffer)
*/
@Override
- public ScanColumn parse( ByteBuffer buff ) {
+ public ScanColumn parse( ByteBuffer buff, final DynamicCompositeComparator cfComparator ) {
DynamicComposite composite = DynamicComposite.fromByteBuffer( buff.duplicate() );
- return new SecondaryIndexColumn( ( UUID ) composite.get( 2 ), composite.get( 1 ), buff );
+ return new SecondaryIndexColumn( ( UUID ) composite.get( 2 ), composite.get( 1 ), buff, cfComparator );
}
+
+
public static class SecondaryIndexColumn extends AbstractScanColumn {
private final Object value;
- public SecondaryIndexColumn( UUID uuid, Object value, ByteBuffer buff ) {
- super( uuid, buff );
+ public SecondaryIndexColumn( final UUID uuid, final Object value, final ByteBuffer columnNameBuffer, final DynamicCompositeComparator cfComparator ) {
+ super( uuid, columnNameBuffer, cfComparator );
this.value = value;
}
@@ -57,5 +61,9 @@ public class SecondaryIndexSliceParser implements SliceParser {
public Object getValue() {
return this.value;
}
+
+
+
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceParser.java
index d1ce6a1..8e5bc53 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceParser.java
@@ -19,6 +19,8 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.nio.ByteBuffer;
+import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
+
/**
* Interface to parse and compare range slices
@@ -28,5 +30,5 @@ import java.nio.ByteBuffer;
public interface SliceParser {
/** Parse the slice and return it's parse type. If null is returned, the column should be considered discarded */
- public ScanColumn parse( ByteBuffer buff );
+ public ScanColumn parse(final ByteBuffer columnNameBytes, final DynamicCompositeComparator cfComparator );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
index e04ac6c..055839e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
@@ -38,7 +38,7 @@ public class StaticIdIterator implements ResultIterator {
*
*/
public StaticIdIterator( UUID id ) {
- final ScanColumn col = new UUIDIndexSliceParser.UUIDColumn( id, ByteBuffer.allocate( 0 ) );
+ final ScanColumn col = new UUIDColumn( id );
ids = Collections.singleton( col );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
new file mode 100644
index 0000000..ff05fe5
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
@@ -0,0 +1,50 @@
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import org.apache.usergrid.utils.UUIDUtils;
+
+
+/**
+ * Used as a comparator for columns
+ */
+class UUIDColumn implements ScanColumn{
+
+ private final UUID uuid;
+ private ScanColumn child;
+
+
+ UUIDColumn( final UUID uuid ) {this.uuid = uuid;}
+
+
+ @Override
+ public UUID getUUID() {
+ return uuid;
+ }
+
+
+ @Override
+ public ByteBuffer getCursorValue() {
+ return null;
+ }
+
+
+ @Override
+ public void setChild( final ScanColumn childColumn ) {
+ this.child = childColumn;
+ }
+
+
+ @Override
+ public ScanColumn getChild() {
+ return child;
+ }
+
+
+ @Override
+ public int compareTo( final ScanColumn other ) {
+ return UUIDUtils.compare( uuid, other.getUUID() );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
index 4b98cc7..d966cc8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
@@ -20,6 +20,8 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.nio.ByteBuffer;
import java.util.UUID;
+import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
+
import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
@@ -33,15 +35,8 @@ public class UUIDIndexSliceParser implements SliceParser {
* @see org.apache.usergrid.persistence.query.ir.result.SliceParser#parse(java.nio.ByteBuffer)
*/
@Override
- public ScanColumn parse( ByteBuffer buff ) {
- return new UUIDColumn( ue.fromByteBuffer( buff.duplicate() ), buff );
+ public ScanColumn parse( ByteBuffer buff, final DynamicCompositeComparator cfComparator ) {
+ return new UUIDColumn( ue.fromByteBuffer( buff.duplicate() ) );
}
-
- public static class UUIDColumn extends AbstractScanColumn {
-
- public UUIDColumn( UUID uuid, ByteBuffer buffer ) {
- super( uuid, buffer );
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
index ea7b3f6..916091c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -21,14 +21,12 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import org.apache.usergrid.persistence.cassandra.CursorCache;
-import org.apache.usergrid.utils.UUIDUtils;
import static org.apache.usergrid.persistence.cassandra.Serializers.*;
@@ -39,8 +37,6 @@ import static org.apache.usergrid.persistence.cassandra.Serializers.*;
*/
public class UnionIterator extends MultiIterator {
- private static final ScanColumnComparator COMP = new ScanColumnComparator();
-
private SortedColumnList list;
private final int id;
@@ -133,14 +129,11 @@ public class UnionIterator extends MultiIterator {
*/
public static final class SortedColumnList {
- private static final ScanColumnComparator COMP = new ScanColumnComparator();
-
private final int maxSize;
private final List<ScanColumn> list;
-
- private ScanColumn min;
+ private UUIDColumn min;
public SortedColumnList( final int maxSize, final UUID minUuid ) {
@@ -149,7 +142,7 @@ public class UnionIterator extends MultiIterator {
this.maxSize = maxSize;
if ( minUuid != null ) {
- min = new AbstractScanColumn( minUuid, null ) {};
+ min = new UUIDColumn( minUuid) ;
}
}
@@ -159,11 +152,11 @@ public class UnionIterator extends MultiIterator {
*/
public void add( ScanColumn col ) {
//less than our min, don't add
- if ( COMP.compare( min, col ) >= 0 ) {
+ if ( min != null && min.compareTo( col ) >= 0 ) {
return;
}
- int index = Collections.binarySearch( this.list, col, COMP );
+ int index = Collections.binarySearch( this.list, col );
//already present
if ( index > -1 ) {
@@ -221,7 +214,8 @@ public class UnionIterator extends MultiIterator {
return;
}
- min = this.list.get( size - 1 );
+ final UUID oldMin = this.list.get( size - 1 ).getUUID();
+ min = new UUIDColumn( oldMin );
}
@@ -237,28 +231,4 @@ public class UnionIterator extends MultiIterator {
this.min = null;
}
}
-
-
- /**
- * Simple comparator for comparing scan columns. Orders them by time uuid
- */
- private static class ScanColumnComparator implements Comparator<ScanColumn> {
-
- @Override
- public int compare( final ScanColumn o1, final ScanColumn o2 ) {
- if ( o1 == null ) {
- if ( o2 == null ) {
- return 0;
- }
-
- return -1;
- }
-
- else if ( o2 == null ) {
- return 1;
- }
-
- return UUIDUtils.compare( o1.getUUID(), o2.getUUID() );
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1fe69065/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
index d478942..45e7f50 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
@@ -94,7 +94,7 @@ public class AbstractScanColumnTest {
private class TestScanColumn extends AbstractScanColumn {
protected TestScanColumn( final UUID uuid, final ByteBuffer buffer ) {
- super( uuid, buffer );
+ super( uuid, buffer, cfComparator );
}
}
}
[11/25] incubator-usergrid git commit: Revert of geo per shard. Geo
has a hashing bug on write,
and therefore does not correctly allocate the same shard as the rest of the
entity on write. Aggregate + filter is the only mechanism to return correct
resul
Posted by sn...@apache.org.
Revert of geo per shard. Geo has a hashing bug on write, and therefore does not correctly allocate the same shard as the rest of the entity on write. Aggregate + filter is the only mechanism to return correct results without triggering a complete re-index.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/595aa71d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/595aa71d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/595aa71d
Branch: refs/heads/master
Commit: 595aa71d6512ed5f43f9ff252d28824094cf7223
Parents: 9971068
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jul 1 11:02:48 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jul 1 11:02:48 2015 -0600
----------------------------------------------------------------------
.../persistence/geo/CollectionGeoSearch.java | 4 +-
.../persistence/geo/ConnectionGeoSearch.java | 4 +-
.../persistence/geo/GeoIndexSearcher.java | 13 +-
.../query/ir/result/CollectionShardFilter.java | 53 ++++++
.../ir/result/SearchCollectionVisitor.java | 12 +-
.../ir/result/SearchConnectionVisitor.java | 9 +-
.../org/apache/usergrid/persistence/GeoIT.java | 161 +++++++++----------
7 files changed, 154 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
index d8635d9..c823e20 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
@@ -44,9 +44,9 @@ public class CollectionGeoSearch extends GeoIndexSearcher {
private final EntityRef headEntity;
- public CollectionGeoSearch( EntityManager entityManager, final String shard, CassandraService cass,
+ public CollectionGeoSearch( EntityManager entityManager, IndexBucketLocator locator, CassandraService cass,
EntityRef headEntity, String collectionName ) {
- super( entityManager, shard, cass );
+ super( entityManager, locator, cass );
this.collectionName = collectionName;
this.headEntity = headEntity;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
index 37b96dd..a1ad71e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
@@ -43,9 +43,9 @@ public class ConnectionGeoSearch extends GeoIndexSearcher {
private final UUID connectionId;
- public ConnectionGeoSearch( EntityManager entityManager, final String shard, CassandraService cass,
+ public ConnectionGeoSearch( EntityManager entityManager, IndexBucketLocator locator, CassandraService cass,
UUID connectionId ) {
- super( entityManager, shard, cass );
+ super( entityManager, locator, cass );
this.connectionId = connectionId;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
index e61aa64..eb20ffa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
@@ -65,12 +65,12 @@ public abstract class GeoIndexSearcher {
private static final int MAX_FETCH_SIZE = 1000;
protected final EntityManager em;
- protected final String shard;
+ protected final IndexBucketLocator locator;
protected final CassandraService cass;
- public GeoIndexSearcher(final EntityManager entityManager, final String shard, final CassandraService cass ) {
+ public GeoIndexSearcher( EntityManager entityManager, IndexBucketLocator locator, CassandraService cass ) {
this.em = entityManager;
- this.shard = shard;
+ this.locator = locator;
this.cass = cass;
}
@@ -325,7 +325,12 @@ public abstract class GeoIndexSearcher {
for ( String geoCell : curGeocellsUnique ) {
// add buckets for each geoCell
- keys.add( key( key, DICTIONARY_GEOCELL, geoCell, shard ) );
+
+ //TODO, use merge logic here
+
+ for ( String indexBucket : locator.getBuckets( ) ) {
+ keys.add( key( key, DICTIONARY_GEOCELL, geoCell, indexBucket ) );
+ }
}
DynamicComposite start = null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java
new file mode 100644
index 0000000..cc8b7c7
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+
+
+/**
+ * Class that performs validation on an entity to ensure it's in the shard we expect
+ */
+public final class CollectionShardFilter implements ShardFilter {
+ private final IndexBucketLocator indexBucketLocator;
+ private final String expectedBucket;
+
+
+ public CollectionShardFilter( final IndexBucketLocator indexBucketLocator, final String expectedBucket ) {
+ this.indexBucketLocator = indexBucketLocator;
+ this.expectedBucket = expectedBucket;
+ }
+
+
+
+
+
+ public boolean isInShard( final ScanColumn scanColumn ) {
+
+ final UUID entityId = scanColumn.getUUID();
+
+ //not for our current processing shard, discard
+ final String shard = indexBucketLocator.getBucket( entityId );
+
+ return expectedBucket.equals( shard );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index 73eb148..32c3a6e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -128,16 +128,16 @@ public class SearchCollectionVisitor extends SearchVisitor {
final int size = queryProcessor.getPageSizeHint( node );
GeoIterator itr = new GeoIterator(
- new CollectionGeoSearch( em, bucket, cassandraService, headEntity, collection.getName() ),
+ new CollectionGeoSearch( em, indexBucketLocator, cassandraService, headEntity, collection.getName() ),
size, slice, node.getPropertyName(), new Point( node.getLattitude(), node.getLongitude() ),
node.getDistance() );
- this.results.push( itr );
+// this.results.push( itr );
-// final CollectionShardFilter
-// validator = new CollectionShardFilter(indexBucketLocator, bucket );
-//
-// this.results.push( new ShardFilterIterator( validator, itr, size));
+ final CollectionShardFilter
+ validator = new CollectionShardFilter(indexBucketLocator, bucket );
+
+ this.results.push( new ShardFilterIterator( validator, itr, size));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 404420a..5505914 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -115,17 +115,16 @@ public class SearchConnectionVisitor extends SearchVisitor {
//TODO, make search take a shard
GeoIterator itr =
- new GeoIterator( new ConnectionGeoSearch( em, bucket, cassandraService, connection.getIndexId() ),
+ new GeoIterator( new ConnectionGeoSearch( em, indexBucketLocator, cassandraService, connection.getIndexId() ),
size, slice, node.getPropertyName(),
new Point( node.getLattitude(), node.getLongitude() ), node.getDistance() );
- this.results.push( itr );
-// final CollectionShardFilter
-// validator = new CollectionShardFilter(indexBucketLocator, bucket );
+ final ConnectionShardFilter
+ validator = new ConnectionShardFilter(indexBucketLocator, bucket, connection );
-// this.results.push( new ShardFilterIterator( validator, itr, size ) );
+ this.results.push( new ShardFilterIterator( validator, itr, size ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
index 8e6fb80..8c9e9b7 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
@@ -76,10 +76,8 @@ public class GeoIT extends AbstractCoreIT {
Point center = new Point( 37.774277, -122.404744 );
- final String shard = setup.getIbl().getBucket( user.getUuid() );
-
CollectionGeoSearch connSearch =
- new CollectionGeoSearch( em, shard, setup.getCassSvc(), em.getApplicationRef(), "users" );
+ new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(), "users" );
List<EntityLocationRef> listResults =
@@ -421,86 +419,83 @@ public class GeoIT extends AbstractCoreIT {
}
-// @Test
-// public void testDenseSearch() throws Exception {
-//
-// UUID applicationId = setup.createApplication( "testOrganization", "testDenseSearch" );
-// assertNotNull( applicationId );
-//
-// EntityManager em = setup.getEmf().getEntityManager( applicationId );
-// assertNotNull( em );
-//
-// // save objects in a diagonal line from -90 -180 to 90 180
-//
-// int numEntities = 500;
-//
-// float minLattitude = 48.32455f;
-// float maxLattitude = 48.46481f;
-// float minLongitude = 9.89561f;
-// float maxLongitude = 10.0471f;
-//
-// float lattitudeDelta = ( maxLattitude - minLattitude ) / numEntities;
-//
-// float longitudeDelta = ( maxLongitude - minLongitude ) / numEntities;
-//
-// for ( int i = 0; i < numEntities; i++ ) {
-// float lattitude = minLattitude + lattitudeDelta * i;
-// float longitude = minLongitude + longitudeDelta * i;
-//
-// Map<String, Float> location = MapUtils.hashMap( "latitude", lattitude ).map( "longitude", longitude );
-//
-// Map<String, Object> data = new HashMap<String, Object>( 2 );
-// data.put( "name", String.valueOf( i ) );
-// data.put( "location", location );
-//
-// em.create( "store", data );
-// }
-//
-// //do a direct geo iterator test. We need to make sure that we short circuit on the correct tile.
-//
-// float lattitude = 48.38626f;
-// float longtitude = 9.94175f;
-// int distance = 1000;
-// int limit = 8;
-//
-//
-// QuerySlice slice = new QuerySlice( "location", 0 );
-//
-// //TODO test this
-// final String shard = setup.getIbl().getBucket( user.getUuid() );
-//
-// GeoIterator itr = new GeoIterator(
-// new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(), "stores" ),
-// limit, slice, "location", new Point( lattitude, longtitude ), distance );
-//
-//
-// // check we got back all 500 entities
-// assertFalse( itr.hasNext() );
-//
-// List<String> cells = itr.getLastCellsSearched();
-//
-// assertEquals( 1, cells.size() );
-//
-// assertEquals( 4, cells.get( 0 ).length() );
-//
-//
-// long startTime = System.currentTimeMillis();
-//
-// //now test at the EM level, there should be 0 results.
-// Query query = new Query();
-//
-// query.addFilter( "location within 1000 of 48.38626, 9.94175" );
-// query.setLimit( 8 );
-//
-//
-// Results results = em.searchCollection( em.getApplicationRef(), "stores", query );
-//
-// assertEquals( 0, results.size() );
-//
-// long endTime = System.currentTimeMillis();
-//
-// LOG.info( "Runtime took {} milliseconds to search", endTime - startTime );
-// }
+ @Test
+ public void testDenseSearch() throws Exception {
+
+ UUID applicationId = setup.createApplication( "testOrganization", "testDenseSearch" );
+ assertNotNull( applicationId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ assertNotNull( em );
+
+ // save objects in a diagonal line from -90 -180 to 90 180
+
+ int numEntities = 500;
+
+ float minLattitude = 48.32455f;
+ float maxLattitude = 48.46481f;
+ float minLongitude = 9.89561f;
+ float maxLongitude = 10.0471f;
+
+ float lattitudeDelta = ( maxLattitude - minLattitude ) / numEntities;
+
+ float longitudeDelta = ( maxLongitude - minLongitude ) / numEntities;
+
+ for ( int i = 0; i < numEntities; i++ ) {
+ float lattitude = minLattitude + lattitudeDelta * i;
+ float longitude = minLongitude + longitudeDelta * i;
+
+ Map<String, Float> location = MapUtils.hashMap( "latitude", lattitude ).map( "longitude", longitude );
+
+ Map<String, Object> data = new HashMap<String, Object>( 2 );
+ data.put( "name", String.valueOf( i ) );
+ data.put( "location", location );
+
+ em.create( "store", data );
+ }
+
+ //do a direct geo iterator test. We need to make sure that we short circuit on the correct tile.
+
+ float lattitude = 48.38626f;
+ float longtitude = 9.94175f;
+ int distance = 1000;
+ int limit = 8;
+
+
+ QuerySlice slice = new QuerySlice( "location", 0 );
+
+ GeoIterator itr = new GeoIterator(
+ new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(), "stores" ),
+ limit, slice, "location", new Point( lattitude, longtitude ), distance );
+
+
+ // check we got back all 500 entities
+ assertFalse( itr.hasNext() );
+
+ List<String> cells = itr.getLastCellsSearched();
+
+ assertEquals( 1, cells.size() );
+
+ assertEquals( 4, cells.get( 0 ).length() );
+
+
+ long startTime = System.currentTimeMillis();
+
+ //now test at the EM level, there should be 0 results.
+ Query query = new Query();
+
+ query.addFilter( "location within 1000 of 48.38626, 9.94175" );
+ query.setLimit( 8 );
+
+
+ Results results = em.searchCollection( em.getApplicationRef(), "stores", query );
+
+ assertEquals( 0, results.size() );
+
+ long endTime = System.currentTimeMillis();
+
+ LOG.info( "Runtime took {} milliseconds to search", endTime - startTime );
+ }
public Map<String, Object> getLocation( double latitude, double longitude ) throws Exception {
[04/25] incubator-usergrid git commit: Pushed comparator down into
each column implementation.
Posted by sn...@apache.org.
Pushed comparator down into each column implementation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bb2837ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bb2837ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bb2837ca
Branch: refs/heads/master
Commit: bb2837caaa52d5e2b4977aa5b76533ed5240c3ca
Parents: 1fe6906
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jun 17 17:36:16 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jun 17 17:36:16 2015 -0600
----------------------------------------------------------------------
.../cassandra/index/ConnectedIndexScanner.java | 6 ++++
.../cassandra/index/IndexBucketScanner.java | 6 ++++
.../cassandra/index/IndexScanner.java | 15 +++++++--
.../cassandra/index/NoOpIndexScanner.java | 6 ++++
.../query/ir/result/AbstractScanColumn.java | 34 +++++++++-----------
.../ir/result/ConnectionIndexSliceParser.java | 19 ++++++++---
.../ir/result/SearchCollectionVisitor.java | 2 +-
.../ir/result/SecondaryIndexSliceParser.java | 34 +++++++++++++-------
.../query/ir/result/SliceIterator.java | 5 ++-
.../query/ir/result/SliceParser.java | 2 +-
.../query/ir/result/StaticIdIterator.java | 3 +-
.../persistence/query/ir/result/UUIDColumn.java | 10 ++++--
.../query/ir/result/UUIDIndexSliceParser.java | 14 +++-----
.../query/ir/result/UnionIterator.java | 4 +--
.../query/ir/result/AbstractScanColumnTest.java | 8 ++++-
.../query/ir/result/InOrderIterator.java | 2 +-
.../query/ir/result/IteratorHelper.java | 2 +-
17 files changed, 115 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
index 27689ce..87726cb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
@@ -259,4 +259,10 @@ public class ConnectedIndexScanner implements IndexScanner {
public int getPageSize() {
return pageSize;
}
+
+
+ @Override
+ public boolean isReversed() {
+ return this.reversed;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
index 668ab02..e568575 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
@@ -229,4 +229,10 @@ public class IndexBucketScanner implements IndexScanner {
public int getPageSize() {
return pageSize;
}
+
+
+ @Override
+ public boolean isReversed() {
+ return this.reversed;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexScanner.java
index 1307361..e5e9adb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexScanner.java
@@ -33,8 +33,19 @@ public interface IndexScanner
extends Iterable<List<HColumn<ByteBuffer, ByteBuffer>>>, Iterator<List<HColumn<ByteBuffer, ByteBuffer>>> {
/** Reset the scanner back to the start */
- public void reset();
+ void reset();
+
+ /**
+ * Return the page size of the index
+ * @return
+ */
+ int getPageSize();
+
+ /**
+ * Return true if this iterator iterates in reverse
+ * @return
+ */
+ boolean isReversed();
- public int getPageSize();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/NoOpIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/NoOpIndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/NoOpIndexScanner.java
index 01002d6..d6fb113 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/NoOpIndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/NoOpIndexScanner.java
@@ -91,4 +91,10 @@ public class NoOpIndexScanner implements IndexScanner {
public int getPageSize() {
return 0;
}
+
+
+ @Override
+ public boolean isReversed() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
index e4515fb..383ead7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
@@ -19,9 +19,8 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.nio.ByteBuffer;
import java.util.UUID;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
+import org.apache.cassandra.utils.ByteBufferUtil;
/**
@@ -33,14 +32,12 @@ public abstract class AbstractScanColumn implements ScanColumn {
private final UUID uuid;
private final ByteBuffer buffer;
- private final DynamicCompositeComparator cfComparator;
private ScanColumn child;
- protected AbstractScanColumn( final UUID uuid, final ByteBuffer columnNameBuffer, final DynamicCompositeComparator cfComparator ) {
+ protected AbstractScanColumn( final UUID uuid, final ByteBuffer columnNameBuffer ) {
this.uuid = uuid;
this.buffer = columnNameBuffer;
- this.cfComparator = cfComparator;
}
@@ -52,7 +49,7 @@ public abstract class AbstractScanColumn implements ScanColumn {
@Override
public ByteBuffer getCursorValue() {
- return buffer == null ? null :buffer.duplicate();
+ return buffer == null ? null : buffer.duplicate();
}
@@ -67,8 +64,7 @@ public abstract class AbstractScanColumn implements ScanColumn {
AbstractScanColumn that = ( AbstractScanColumn ) o;
- return uuid.equals(that.uuid);
-
+ return uuid.equals( that.uuid );
}
@@ -89,7 +85,7 @@ public abstract class AbstractScanColumn implements ScanColumn {
@Override
public void setChild( final ScanColumn childColumn ) {
- this.child = childColumn;
+ this.child = childColumn;
}
@@ -99,23 +95,23 @@ public abstract class AbstractScanColumn implements ScanColumn {
}
- @Override
- public int compareTo( final ScanColumn otherScanColumn ) {
+ /**
+ * Comparator for comparing children. A null safe call
+ * @param otherScanColumn
+ * @return
+ */
+ protected int compareChildren( final ScanColumn otherScanColumn ) {
- if(otherScanColumn == null){
+ if ( otherScanColumn == null ) {
return 1;
}
+ final ScanColumn otherChild = otherScanColumn.getChild();
- final int compare = cfComparator.compare( buffer, otherScanColumn.getCursorValue() );
-
- //equal, recurse. otherScanColumn is implicitly not null from above check
- if(compare == 0 && child != null){
- return child.compareTo( otherScanColumn.getChild() );
+ if ( child != null && otherChild != null ) {
+ return child.compareTo( otherChild );
}
-
return 0;
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
index ae3b0b5..2bb83b4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
@@ -46,7 +46,7 @@ public class ConnectionIndexSliceParser implements SliceParser {
* @see org.apache.usergrid.persistence.query.ir.result.SliceParser#parse(java.nio.ByteBuffer)
*/
@Override
- public ScanColumn parse( ByteBuffer buff, final DynamicCompositeComparator cfComparator ) {
+ public ScanColumn parse( ByteBuffer buff, final boolean isReversed ) {
DynamicComposite composite = DynamicComposite.fromByteBuffer( buff.duplicate() );
String connectedType = ( String ) composite.get( 1 );
@@ -63,20 +63,21 @@ public class ConnectionIndexSliceParser implements SliceParser {
return null;
}
- return new ConnectionColumn( ( UUID ) composite.get( 0 ), connectedType, buff , cfComparator);
+ return new ConnectionColumn( ( UUID ) composite.get( 0 ), connectedType, buff );
// return composite;
// return null;
}
+
public static class ConnectionColumn extends AbstractScanColumn {
private final String connectedType;
- public ConnectionColumn( UUID uuid, String connectedType, ByteBuffer column, final DynamicCompositeComparator cfComparator ) {
- super( uuid, column, cfComparator );
+ public ConnectionColumn( UUID uuid, String connectedType, ByteBuffer column) {
+ super( uuid, column );
this.connectedType = connectedType;
}
@@ -85,5 +86,15 @@ public class ConnectionIndexSliceParser implements SliceParser {
public String getTargetType() {
return connectedType;
}
+
+
+ @Override
+ public int compareTo( final ScanColumn o ) {
+ if(o == null){
+ return 1;
+ }
+
+ return connectedType.compareTo( ((ConnectionColumn)o).connectedType );
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index 20d303c..78abbca 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -95,7 +95,7 @@ public class SearchCollectionVisitor extends SearchVisitor {
UUID startId = null;
if ( slice.hasCursor() ) {
- startId = UUID_PARSER.parse( slice.getCursor(), null ).getUUID();
+ startId = UUID_PARSER.parse( slice.getCursor(), false ).getUUID();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
index c13ac0d..e27d99c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
@@ -18,10 +18,9 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.nio.ByteBuffer;
+import java.util.Comparator;
import java.util.UUID;
-import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
-
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -37,23 +36,34 @@ public class SecondaryIndexSliceParser implements SliceParser {
* @see org.apache.usergrid.persistence.query.ir.result.SliceParser#parse(java.nio.ByteBuffer)
*/
@Override
- public ScanColumn parse( ByteBuffer buff, final DynamicCompositeComparator cfComparator ) {
+ public ScanColumn parse( ByteBuffer buff, final boolean isReversed) {
DynamicComposite composite = DynamicComposite.fromByteBuffer( buff.duplicate() );
- return new SecondaryIndexColumn( ( UUID ) composite.get( 2 ), composite.get( 1 ), buff, cfComparator );
- }
+ throw new UnsupportedOperationException( "Implement me with static comparators" );
+// return new SecondaryIndexColumn( ( UUID ) composite.get( 2 ), composite.get( 1 ), buff, null );
+ }
public static class SecondaryIndexColumn extends AbstractScanColumn {
private final Object value;
-
-
- public SecondaryIndexColumn( final UUID uuid, final Object value, final ByteBuffer columnNameBuffer, final DynamicCompositeComparator cfComparator ) {
- super( uuid, columnNameBuffer, cfComparator );
+ private final Comparator<Object> valueComparator;
+
+
+ /**
+ * Create the secondary index column
+ * @param uuid
+ * @param value
+ * @param columnNameBuffer
+ * @param valueComparator The comparator for the values
+ */
+ public SecondaryIndexColumn( final UUID uuid, final Object value, final ByteBuffer columnNameBuffer,
+ final Comparator<Object> valueComparator ) {
+ super( uuid, columnNameBuffer );
this.value = value;
+ this.valueComparator = valueComparator;
}
@@ -63,7 +73,9 @@ public class SecondaryIndexSliceParser implements SliceParser {
}
-
-
+ @Override
+ public int compareTo( final ScanColumn o ) {
+ throw new UnsupportedOperationException( "Impelment me" );
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
index 2a4c3cf..3f052da 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -48,6 +48,8 @@ public class SliceIterator implements ResultIterator {
private final SliceParser parser;
private final IndexScanner scanner;
private final int pageSize;
+ private final boolean isReversed;
+
/**
* Pointer to the uuid set until it's returned
@@ -82,6 +84,7 @@ public class SliceIterator implements ResultIterator {
this.pageSize = scanner.getPageSize();
this.cols = new LinkedHashMap<UUID, ScanColumn>( this.pageSize );
this.parsedCols = new LinkedHashSet<ScanColumn>( this.pageSize );
+ this.isReversed = scanner.isReversed();
}
@@ -126,7 +129,7 @@ public class SliceIterator implements ResultIterator {
ByteBuffer colName = results.next().getName().duplicate();
- ScanColumn parsed = parser.parse( colName );
+ ScanColumn parsed = parser.parse( colName, isReversed );
//skip this value, the parser has discarded it
if ( parsed == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceParser.java
index 8e5bc53..d0f9c7d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceParser.java
@@ -30,5 +30,5 @@ import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparato
public interface SliceParser {
/** Parse the slice and return it's parse type. If null is returned, the column should be considered discarded */
- public ScanColumn parse(final ByteBuffer columnNameBytes, final DynamicCompositeComparator cfComparator );
+ ScanColumn parse( final ByteBuffer columnNameBytes, final boolean isReversed );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
index 055839e..1ed6949 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
@@ -17,7 +17,6 @@
package org.apache.usergrid.persistence.query.ir.result;
-import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.Iterator;
import java.util.Set;
@@ -38,7 +37,7 @@ public class StaticIdIterator implements ResultIterator {
*
*/
public StaticIdIterator( UUID id ) {
- final ScanColumn col = new UUIDColumn( id );
+ final ScanColumn col = new UUIDColumn( id, 1 );
ids = Collections.singleton( col );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
index ff05fe5..a5e09c1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
@@ -13,10 +13,14 @@ import org.apache.usergrid.utils.UUIDUtils;
class UUIDColumn implements ScanColumn{
private final UUID uuid;
+ private final int compareReversed;
private ScanColumn child;
- UUIDColumn( final UUID uuid ) {this.uuid = uuid;}
+ UUIDColumn( final UUID uuid, final int compareReversed ) {
+ this.uuid = uuid;
+ this.compareReversed = compareReversed;
+ }
@Override
@@ -45,6 +49,8 @@ class UUIDColumn implements ScanColumn{
@Override
public int compareTo( final ScanColumn other ) {
- return UUIDUtils.compare( uuid, other.getUUID() );
+
+ return UUIDUtils.compare( uuid, other.getUUID() ) * compareReversed;
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
index d966cc8..91644ce 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
@@ -18,9 +18,6 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.nio.ByteBuffer;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
import static org.apache.usergrid.persistence.cassandra.Serializers.*;
@@ -31,12 +28,11 @@ import static org.apache.usergrid.persistence.cassandra.Serializers.*;
*/
public class UUIDIndexSliceParser implements SliceParser {
- /* (non-Javadoc)
- * @see org.apache.usergrid.persistence.query.ir.result.SliceParser#parse(java.nio.ByteBuffer)
- */
+
+
@Override
- public ScanColumn parse( ByteBuffer buff, final DynamicCompositeComparator cfComparator ) {
- return new UUIDColumn( ue.fromByteBuffer( buff.duplicate() ) );
+ public ScanColumn parse( final ByteBuffer columnNameBytes, final boolean isReversed ) {
+ final int compare = isReversed? -1: 1;
+ return new UUIDColumn( ue.fromByteBuffer( columnNameBytes.duplicate() ), compare );
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
index 916091c..635ca97 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -142,7 +142,7 @@ public class UnionIterator extends MultiIterator {
this.maxSize = maxSize;
if ( minUuid != null ) {
- min = new UUIDColumn( minUuid) ;
+ min = new UUIDColumn( minUuid, 1 ) ;
}
}
@@ -215,7 +215,7 @@ public class UnionIterator extends MultiIterator {
}
final UUID oldMin = this.list.get( size - 1 ).getUUID();
- min = new UUIDColumn( oldMin );
+ min = new UUIDColumn( oldMin, 1 );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
index 45e7f50..de46836 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
@@ -94,7 +94,13 @@ public class AbstractScanColumnTest {
private class TestScanColumn extends AbstractScanColumn {
protected TestScanColumn( final UUID uuid, final ByteBuffer buffer ) {
- super( uuid, buffer, cfComparator );
+ super( uuid, buffer );
+ }
+
+
+ @Override
+ public int compareTo( final ScanColumn o ) {
+ return 0;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
index e937162..b4cd906 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
@@ -51,7 +51,7 @@ public class InOrderIterator implements ResultIterator {
/** Add a uuid to the list */
public void add( UUID... ids ) {
for ( UUID current : ids ) {
- uuids.add( new UUIDIndexSliceParser.UUIDColumn( current, ByteBuffer.allocate( 0 ) ) );
+ uuids.add( new UUIDColumn( current, 1 ) );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bb2837ca/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
index 049296e..13ad542 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
@@ -29,6 +29,6 @@ import java.util.UUID;
public class IteratorHelper {
public static ScanColumn uuidColumn( UUID value ) {
- return new UUIDIndexSliceParser.UUIDColumn( value, ByteBuffer.allocate( 0 ) );
+ return new UUIDColumn( value, 1 );
}
}
[10/25] incubator-usergrid git commit: Refactor of Geo to execute per
shard for faster querying rather than post filter
Posted by sn...@apache.org.
Refactor of Geo to execute per shard for faster querying rather than post filter
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9971068a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9971068a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9971068a
Branch: refs/heads/master
Commit: 9971068a04db30237350742d5cb7e0c42962ba8c
Parents: efab0b9
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jun 30 17:41:27 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jun 30 17:41:27 2015 -0600
----------------------------------------------------------------------
.../persistence/IndexBucketLocator.java | 32 +---
.../persistence/cassandra/CassandraService.java | 9 +-
.../cassandra/ConnectionRefImpl.java | 10 ++
.../cassandra/EntityManagerImpl.java | 10 +-
.../persistence/cassandra/GeoIndexManager.java | 33 ++--
.../cassandra/RelationManagerImpl.java | 44 ++---
.../cassandra/SimpleIndexBucketLocatorImpl.java | 4 +-
.../persistence/geo/CollectionGeoSearch.java | 4 +-
.../persistence/geo/ConnectionGeoSearch.java | 4 +-
.../persistence/geo/GeoIndexSearcher.java | 27 ++-
.../persistence/query/ir/SearchVisitor.java | 6 +-
.../result/CollectionSearchVisitorFactory.java | 4 +-
.../result/ConnectionSearchVisitorFactory.java | 2 +-
.../query/ir/result/ConnectionShardFilter.java | 62 +++++++
.../ir/result/SearchCollectionVisitor.java | 12 +-
.../ir/result/SearchConnectionVisitor.java | 18 +-
.../query/ir/result/ShardFilter.java | 29 +++
.../query/ir/result/ShardFilterIterator.java | 124 +++++++++++++
.../query/ir/result/SliceIterator.java | 11 +-
.../ir/result/SliceShardFilterIterator.java | 160 -----------------
.../org/apache/usergrid/persistence/GeoIT.java | 161 ++++++++---------
.../SimpleIndexBucketLocatorImplTest.java | 22 +--
.../query/AbstractIteratingQueryIT.java | 3 +-
.../ir/result/ShardFilterIteratorTest.java | 175 +++++++++++++++++++
.../ir/result/SliceShardFilterIteratorTest.java | 155 ----------------
25 files changed, 578 insertions(+), 543 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/IndexBucketLocator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/IndexBucketLocator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/IndexBucketLocator.java
index 3b45f8f..64e7b5d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/IndexBucketLocator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/IndexBucketLocator.java
@@ -29,48 +29,22 @@ import java.util.UUID;
*/
public interface IndexBucketLocator {
- public enum IndexType {
- COLLECTION( "collection" ), CONNECTION( "connection" ), GEO( "geo" ), UNIQUE( "unique" );
-
- private final String type;
-
-
- private IndexType( String type ) {
- this.type = type;
- }
-
-
- public String getType() {
- return type;
- }
-
- }
/**
* Return the bucket to use for indexing this entity
- *
- * @param applicationId The application id
- * @param type The type of the index. This way indexing on the same property value for different types of indexes
- * does not cause collisions on partitioning and lookups
* @param entityId The entity id to be indexed
- * @param components The strings and uniquely identify the path to this index. I.E entityType and propName,
- * collection name etc This string must remain the same for all reads and writes
*
* @return A bucket to use. Note that ALL properties for the given entity should be in the same bucket. This
* allows us to shard and execute queries in parallel. Generally speaking, sharding on entityId is the best
* strategy, since this is an immutable value
*/
- public String getBucket( UUID applicationId, IndexType type, UUID entityId, String... components );
+ String getBucket( UUID entityId );
/**
* Get all buckets that exist for this application with the given entity type, and property name
- *
- * @param applicationId The application id
- * @param type The type of index
- * @param components The strings and uniquely identify the path to this index. I.E entityType and propName,
- * collection name etc
+
*
* @return All buckets for this application at the given component path
*/
- public List<String> getBuckets( UUID applicationId, IndexType type, String... components );
+ List<String> getBuckets();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
index 12ea338..85dc1f4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
@@ -28,15 +28,11 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.locking.LockManager;
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.UUIDStartToBytes;
@@ -68,7 +64,6 @@ import me.prettyprint.hector.api.ddl.KeyspaceDefinition;
import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.ColumnQuery;
-import me.prettyprint.hector.api.query.CountQuery;
import me.prettyprint.hector.api.query.MultigetSliceQuery;
import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.RangeSlicesQuery;
@@ -77,7 +72,6 @@ import me.prettyprint.hector.api.query.SliceQuery;
import static me.prettyprint.cassandra.service.FailoverPolicy.ON_FAIL_TRY_ALL_AVAILABLE;
import static me.prettyprint.hector.api.factory.HFactory.createColumn;
import static me.prettyprint.hector.api.factory.HFactory.createMultigetSliceQuery;
-
import static me.prettyprint.hector.api.factory.HFactory.createRangeSlicesQuery;
import static me.prettyprint.hector.api.factory.HFactory.createSliceQuery;
import static me.prettyprint.hector.api.factory.HFactory.createVirtualKeyspace;
@@ -85,7 +79,6 @@ import static org.apache.commons.collections.MapUtils.getIntValue;
import static org.apache.commons.collections.MapUtils.getString;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_ID_SETS;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.batchExecute;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.buildSetIdListMutator;
import static org.apache.usergrid.utils.ConversionUtils.bytebuffer;
import static org.apache.usergrid.utils.ConversionUtils.bytebuffers;
import static org.apache.usergrid.utils.JsonUtils.mapToFormattedJsonString;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/ConnectionRefImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/ConnectionRefImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/ConnectionRefImpl.java
index 1aa5f6d..fb4c5d2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/ConnectionRefImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/ConnectionRefImpl.java
@@ -355,6 +355,16 @@ public class ConnectionRefImpl implements ConnectionRef {
}
+ /**
+ * Return the shard UUID to be used for hashing and filters on connections
+ * @return
+ */
+ public UUID getConnectionSearchShardId(){
+ return getIndexId( ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE, getConnectedEntity(),
+ getConnectionType(), getConnectedEntityType(), new ConnectedEntityRef[0] );
+ }
+
+
public ConnectionRefImpl getConnectionToConnectionEntity() {
return new ConnectionRefImpl( getConnectingEntity(),
new ConnectedEntityRefImpl( CONNECTION_ENTITY_CONNECTION_TYPE, CONNECTION_ENTITY_TYPE, getUuid() ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 8f35f17..51383c2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -40,6 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.util.Assert;
+
import org.apache.usergrid.locking.Lock;
import org.apache.usergrid.mq.Message;
import org.apache.usergrid.mq.QueueManager;
@@ -57,7 +58,6 @@ import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.Identifier;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Query.CounterFilterPredicate;
import org.apache.usergrid.persistence.Results;
@@ -109,7 +109,6 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static java.util.Arrays.asList;
import static me.prettyprint.hector.api.factory.HFactory.createCounterSliceQuery;
-
import static org.apache.commons.lang.StringUtils.capitalize;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.usergrid.locking.LockHelper.getUniqueUpdateLock;
@@ -157,6 +156,10 @@ import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtil
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.toStorableBinaryValue;
import static org.apache.usergrid.persistence.cassandra.CassandraService.ALL_COUNT;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.le;
+import static org.apache.usergrid.persistence.cassandra.Serializers.se;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
import static org.apache.usergrid.utils.ClassUtils.cast;
import static org.apache.usergrid.utils.ConversionUtils.bytebuffer;
import static org.apache.usergrid.utils.ConversionUtils.getLong;
@@ -168,7 +171,6 @@ import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMillis;
import static org.apache.usergrid.utils.UUIDUtils.isTimeBased;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
@@ -808,7 +810,7 @@ public class EntityManagerImpl implements EntityManager {
// Create collection name based on entity: i.e. "users"
String collection_name = Schema.defaultCollectionName( eType );
// Create collection key based collection name
- String bucketId = indexBucketLocator.getBucket( applicationId, IndexType.COLLECTION, itemId, collection_name );
+ String bucketId = indexBucketLocator.getBucket(itemId );
Object collection_key = key( applicationId, Schema.DICTIONARY_COLLECTIONS, collection_name, bucketId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/GeoIndexManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/GeoIndexManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/GeoIndexManager.java
index 2f3beb1..ce5fab8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/GeoIndexManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/GeoIndexManager.java
@@ -23,9 +23,9 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.geo.EntityLocationRef;
import org.apache.usergrid.persistence.geo.GeocellManager;
import org.apache.usergrid.persistence.geo.model.Point;
@@ -38,7 +38,6 @@ import me.prettyprint.hector.api.beans.HColumn;
import me.prettyprint.hector.api.mutation.Mutator;
import static me.prettyprint.hector.api.factory.HFactory.createColumn;
-
import static org.apache.usergrid.persistence.Schema.DICTIONARY_GEOCELL;
import static org.apache.usergrid.persistence.Schema.INDEX_CONNECTIONS;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
@@ -104,26 +103,25 @@ public class GeoIndexManager {
// entity_id,prop_name
Object property_index_key =
key( index_keys[ConnectionRefImpl.ALL], INDEX_CONNECTIONS, propertyName, DICTIONARY_GEOCELL, geoCell,
- locator.getBucket( appId, IndexType.CONNECTION, index_keys[ConnectionRefImpl.ALL], geoCell ) );
+ locator.getBucket(index_keys[ConnectionRefImpl.ALL] ) );
// entity_id,entity_type,prop_name
Object entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], INDEX_CONNECTIONS, propertyName, DICTIONARY_GEOCELL,
geoCell,
- locator.getBucket( appId, IndexType.CONNECTION, index_keys[ConnectionRefImpl.BY_ENTITY_TYPE],
- geoCell ) );
+ locator.getBucket( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE] ) );
// entity_id,connection_type,prop_name
Object connection_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], INDEX_CONNECTIONS, propertyName,
- DICTIONARY_GEOCELL, geoCell, locator.getBucket( appId, IndexType.CONNECTION,
- index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], geoCell ) );
+ DICTIONARY_GEOCELL, geoCell, locator.getBucket(
+ index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE] ) );
// entity_id,connection_type,entity_type,prop_name
Object connection_type_and_entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], INDEX_CONNECTIONS, propertyName,
- DICTIONARY_GEOCELL, geoCell, locator.getBucket( appId, IndexType.CONNECTION,
- index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], geoCell ) );
+ DICTIONARY_GEOCELL, geoCell, locator.getBucket(
+ index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE] ) );
// composite(property_value,connected_entity_id,connection_type,entity_type,entry_timestamp)
addInsertToMutator( m, ENTITY_INDEX, property_index_key, columnName, columnValue, timestamp );
@@ -187,26 +185,25 @@ public class GeoIndexManager {
// entity_id,prop_name
Object property_index_key =
key( index_keys[ConnectionRefImpl.ALL], INDEX_CONNECTIONS, propertyName, DICTIONARY_GEOCELL, geoCell,
- locator.getBucket( appId, IndexType.CONNECTION, index_keys[ConnectionRefImpl.ALL], geoCell ) );
+ locator.getBucket( index_keys[ConnectionRefImpl.ALL] ) );
// entity_id,entity_type,prop_name
Object entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], INDEX_CONNECTIONS, propertyName, DICTIONARY_GEOCELL,
geoCell,
- locator.getBucket( appId, IndexType.CONNECTION, index_keys[ConnectionRefImpl.BY_ENTITY_TYPE],
- geoCell ) );
+ locator.getBucket( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE]) );
// entity_id,connection_type,prop_name
Object connection_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], INDEX_CONNECTIONS, propertyName,
- DICTIONARY_GEOCELL, geoCell, locator.getBucket( appId, IndexType.CONNECTION,
- index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], geoCell ) );
+ DICTIONARY_GEOCELL, geoCell, locator.getBucket(
+ index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE] ) );
// entity_id,connection_type,entity_type,prop_name
Object connection_type_and_entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], INDEX_CONNECTIONS, propertyName,
- DICTIONARY_GEOCELL, geoCell, locator.getBucket( appId, IndexType.CONNECTION,
- index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], geoCell ) );
+ DICTIONARY_GEOCELL, geoCell, locator.getBucket(
+ index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE] ) );
// composite(property_value,connected_entity_id,connection_type,entity_type,entry_timestamp)
m.addDeletion( bytebuffer( property_index_key ), ENTITY_INDEX.toString(), columnName,
@@ -260,7 +257,7 @@ public class GeoIndexManager {
for ( int i = 0; i < MAX_RESOLUTION; i++ ) {
String cell = cells.get( i );
- String indexBucket = locator.getBucket( appId, IndexType.GEO, entityId, cell );
+ String indexBucket = locator.getBucket( entityId );
addLocationEntryInsertionToMutator( m, key( key, DICTIONARY_GEOCELL, cell, indexBucket ), location );
}
@@ -297,7 +294,7 @@ public class GeoIndexManager {
String cell = cells.get( i );
- for ( String indexBucket : locator.getBuckets( appId, IndexType.GEO, cell ) ) {
+ for ( String indexBucket : locator.getBuckets() ) {
addLocationEntryDeletionToMutator( m, key( key, DICTIONARY_GEOCELL, cell, indexBucket ), location );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index ebe2aec..74d1d60 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -39,7 +39,6 @@ import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.PagingResultsIterator;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.RelationManager;
@@ -51,20 +50,14 @@ import org.apache.usergrid.persistence.SimpleCollectionRef;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.SimpleRoleRef;
import org.apache.usergrid.persistence.cassandra.IndexUpdate.IndexEntry;
-import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
-import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.geo.EntityLocationRef;
import org.apache.usergrid.persistence.hector.CountingMutator;
-import org.apache.usergrid.persistence.query.ir.QuerySlice;
import org.apache.usergrid.persistence.query.ir.result.CollectionResultsLoaderFactory;
import org.apache.usergrid.persistence.query.ir.result.CollectionSearchVisitorFactory;
import org.apache.usergrid.persistence.query.ir.result.ConnectionResultsLoaderFactory;
import org.apache.usergrid.persistence.query.ir.result.ConnectionSearchVisitorFactory;
import org.apache.usergrid.persistence.query.ir.result.ConnectionTypesIterator;
-import org.apache.usergrid.persistence.query.ir.result.SearchCollectionVisitor;
-import org.apache.usergrid.persistence.query.ir.result.SearchConnectionVisitor;
-import org.apache.usergrid.persistence.query.ir.result.UUIDIndexSliceParser;
import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.apache.usergrid.utils.IndexUtils;
import org.apache.usergrid.utils.MapUtils;
@@ -202,8 +195,7 @@ public class RelationManagerImpl implements RelationManager {
Entity indexedEntity = indexUpdate.getEntity();
String bucketId = indexBucketLocator
- .getBucket( applicationId, IndexType.COLLECTION, indexedEntity.getUuid(), indexedEntity.getType(),
- indexUpdate.getEntryName() );
+ .getBucket(indexedEntity.getUuid() );
// the root name without the bucket
// entity_id,collection_name,prop_name,
@@ -397,7 +389,7 @@ public class RelationManagerImpl implements RelationManager {
// get the bucket this entityId needs to be inserted into
String bucketId = indexBucketLocator
- .getBucket( applicationId, IndexType.COLLECTION, entity.getUuid(), collectionName );
+ .getBucket(entity.getUuid());
Object collections_key = key( ownerId, Schema.DICTIONARY_COLLECTIONS, collectionName, bucketId );
@@ -502,7 +494,7 @@ public class RelationManagerImpl implements RelationManager {
}
Object collections_key = key( headEntity.getUuid(), Schema.DICTIONARY_COLLECTIONS, collectionName,
- indexBucketLocator.getBucket( applicationId, IndexType.COLLECTION, entity.getUuid(), collectionName ) );
+ indexBucketLocator.getBucket( entity.getUuid()) );
// Remove property indexes
@@ -563,26 +555,25 @@ public class RelationManagerImpl implements RelationManager {
// entity_id,prop_name
Object property_index_key = key( index_keys[ConnectionRefImpl.ALL], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket( applicationId, IndexType.CONNECTION, index_keys[ConnectionRefImpl.ALL],
- entry.getPath() ) );
+ indexBucketLocator.getBucket( index_keys[ConnectionRefImpl.ALL] ) );
// entity_id,entity_type,prop_name
Object entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket( applicationId, IndexType.CONNECTION,
- index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], entry.getPath() ) );
+ indexBucketLocator.getBucket(
+ index_keys[ConnectionRefImpl.BY_ENTITY_TYPE] ) );
// entity_id,connection_type,prop_name
Object connection_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket( applicationId, IndexType.CONNECTION,
- index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], entry.getPath() ) );
+ indexBucketLocator.getBucket(
+ index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE]) );
// entity_id,connection_type,entity_type,prop_name
Object connection_type_and_entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket( applicationId, IndexType.CONNECTION,
- index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], entry.getPath() ) );
+ indexBucketLocator.getBucket(
+ index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE]) );
// composite(property_value,connected_entity_id,connection_type,entity_type,entry_timestamp)
addDeleteToMutator( indexUpdate.getBatch(), ENTITY_INDEX, property_index_key,
@@ -613,26 +604,25 @@ public class RelationManagerImpl implements RelationManager {
// entity_id,prop_name
Object property_index_key = key( index_keys[ConnectionRefImpl.ALL], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket( applicationId, IndexType.CONNECTION, index_keys[ConnectionRefImpl.ALL],
- entry.getPath() ) );
+ indexBucketLocator.getBucket( index_keys[ConnectionRefImpl.ALL] ) );
// entity_id,entity_type,prop_name
Object entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket( applicationId, IndexType.CONNECTION,
- index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], entry.getPath() ) );
+ indexBucketLocator.getBucket(
+ index_keys[ConnectionRefImpl.BY_ENTITY_TYPE]) );
// entity_id,connection_type,prop_name
Object connection_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket( applicationId, IndexType.CONNECTION,
- index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], entry.getPath() ) );
+ indexBucketLocator.getBucket(
+ index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE] ) );
// entity_id,connection_type,entity_type,prop_name
Object connection_type_and_entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket( applicationId, IndexType.CONNECTION,
- index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], entry.getPath() ) );
+ indexBucketLocator.getBucket(
+ index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE]) );
// composite(property_value,connected_entity_id,connection_type,entity_type,entry_timestamp)
addInsertToMutator( indexUpdate.getBatch(), ENTITY_INDEX, property_index_key,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/SimpleIndexBucketLocatorImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/SimpleIndexBucketLocatorImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/SimpleIndexBucketLocatorImpl.java
index 16405d9..4a202ff 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/SimpleIndexBucketLocatorImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/SimpleIndexBucketLocatorImpl.java
@@ -102,7 +102,7 @@ public class SimpleIndexBucketLocatorImpl implements IndexBucketLocator {
* java.lang.String[])
*/
@Override
- public String getBucket( UUID applicationId, IndexType type, UUID entityId, String... components ) {
+ public String getBucket( UUID entityId ) {
return getClosestToken( entityId );
}
@@ -116,7 +116,7 @@ public class SimpleIndexBucketLocatorImpl implements IndexBucketLocator {
* java.lang.String[])
*/
@Override
- public List<String> getBuckets( UUID applicationId, IndexType type, String... components ) {
+ public List<String> getBuckets( ) {
return bucketsString;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
index c823e20..d8635d9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
@@ -44,9 +44,9 @@ public class CollectionGeoSearch extends GeoIndexSearcher {
private final EntityRef headEntity;
- public CollectionGeoSearch( EntityManager entityManager, IndexBucketLocator locator, CassandraService cass,
+ public CollectionGeoSearch( EntityManager entityManager, final String shard, CassandraService cass,
EntityRef headEntity, String collectionName ) {
- super( entityManager, locator, cass );
+ super( entityManager, shard, cass );
this.collectionName = collectionName;
this.headEntity = headEntity;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
index a1ad71e..37b96dd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
@@ -43,9 +43,9 @@ public class ConnectionGeoSearch extends GeoIndexSearcher {
private final UUID connectionId;
- public ConnectionGeoSearch( EntityManager entityManager, IndexBucketLocator locator, CassandraService cass,
+ public ConnectionGeoSearch( EntityManager entityManager, final String shard, CassandraService cass,
UUID connectionId ) {
- super( entityManager, locator, cass );
+ super( entityManager, shard, cass );
this.connectionId = connectionId;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
index 0bbace0..e61aa64 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
@@ -29,17 +29,17 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.GeoIndexManager;
import org.apache.usergrid.persistence.cassandra.index.IndexMultiBucketSetLoader;
import org.apache.usergrid.persistence.geo.model.Point;
import org.apache.usergrid.persistence.geo.model.Tuple;
-import org.apache.commons.lang.StringUtils;
-
import me.prettyprint.hector.api.beans.AbstractComposite.ComponentEquality;
import me.prettyprint.hector.api.beans.DynamicComposite;
import me.prettyprint.hector.api.beans.HColumn;
@@ -47,8 +47,10 @@ import me.prettyprint.hector.api.beans.HColumn;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_GEOCELL;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.Serializers.de;
+import static org.apache.usergrid.persistence.cassandra.Serializers.se;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
import static org.apache.usergrid.utils.CompositeUtils.setEqualityFlag;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
public abstract class GeoIndexSearcher {
@@ -63,12 +65,12 @@ public abstract class GeoIndexSearcher {
private static final int MAX_FETCH_SIZE = 1000;
protected final EntityManager em;
- protected final IndexBucketLocator locator;
+ protected final String shard;
protected final CassandraService cass;
- public GeoIndexSearcher( EntityManager entityManager, IndexBucketLocator locator, CassandraService cass ) {
+ public GeoIndexSearcher(final EntityManager entityManager, final String shard, final CassandraService cass ) {
this.em = entityManager;
- this.locator = locator;
+ this.shard = shard;
this.cass = cass;
}
@@ -80,10 +82,6 @@ public abstract class GeoIndexSearcher {
* @param maxResults The maximum number of results to include
* @param minDistance The minimum distance (inclusive)
* @param maxDistance The maximum distance (exclusive)
- * @param entityClass The entity class
- * @param baseQuery The base query
- * @param queryEngine The query engine to use
- * @param maxGeocellResolution The max resolution to use when searching
*/
public final SearchResults proximitySearch( final EntityLocationRef minMatch, final List<String> geoCells,
Point searchPoint, String propertyName, double minDistance,
@@ -327,12 +325,7 @@ public abstract class GeoIndexSearcher {
for ( String geoCell : curGeocellsUnique ) {
// add buckets for each geoCell
-
- //TODO, use merge logic here
-
- for ( String indexBucket : locator.getBuckets( appId, IndexType.GEO, geoCell ) ) {
- keys.add( key( key, DICTIONARY_GEOCELL, geoCell, indexBucket ) );
- }
+ keys.add( key( key, DICTIONARY_GEOCELL, geoCell, shard ) );
}
DynamicComposite start = null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
index fdeca6d..2fe2e37 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
@@ -213,7 +213,7 @@ public abstract class SearchVisitor implements NodeVisitor {
final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( firstFieldSlice );
subResults =
- new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), new SecondaryIndexSliceParser( sliceCursorGenerator ) );
+ new SliceIterator( secondaryIndexScan( orderByNode, firstFieldSlice ), new SecondaryIndexSliceParser( sliceCursorGenerator ) );
}
orderIterator = new OrderByIterator( slice, orderByNode.getSecondarySorts(), subResults, em,
@@ -234,7 +234,7 @@ public abstract class SearchVisitor implements NodeVisitor {
final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( slice );
- SliceIterator joinSlice = new SliceIterator( slice, scanner, new SecondaryIndexSliceParser(
+ SliceIterator joinSlice = new SliceIterator( scanner, new SecondaryIndexSliceParser(
sliceCursorGenerator ));
IntersectionIterator union = new IntersectionIterator( queryProcessor.getPageSizeHint( orderByNode ) );
@@ -268,7 +268,7 @@ public abstract class SearchVisitor implements NodeVisitor {
final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( slice );
- intersections.addIterator( new SliceIterator( slice, scanner, new SecondaryIndexSliceParser(
+ intersections.addIterator( new SliceIterator( scanner, new SecondaryIndexSliceParser(
sliceCursorGenerator )) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
index 1637f2d..fd2eb1e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
@@ -39,7 +39,6 @@ public class CollectionSearchVisitorFactory implements SearchVisitorFactory {
private final QueryProcessor queryProcessor;
private final UUID applicationId;
private final EntityRef headEntity;
- private final String collectionName;
public CollectionSearchVisitorFactory( final CassandraService cassandraService,
@@ -51,7 +50,6 @@ public class CollectionSearchVisitorFactory implements SearchVisitorFactory {
this.queryProcessor = queryProcessor;
this.applicationId = applicationId;
this.headEntity = headEntity;
- this.collectionName = collectionName;
}
@@ -59,7 +57,7 @@ public class CollectionSearchVisitorFactory implements SearchVisitorFactory {
public Collection<SearchVisitor> createVisitors() {
final List<String> buckets =
- indexBucketLocator.getBuckets( applicationId, IndexBucketLocator.IndexType.CONNECTION, collectionName );
+ indexBucketLocator.getBuckets( );
final List<SearchVisitor> visitors = new ArrayList<SearchVisitor>( buckets.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
index 6a340b9..1a5b2bd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
@@ -62,7 +62,7 @@ public class ConnectionSearchVisitorFactory implements SearchVisitorFactory {
public Collection<SearchVisitor> createVisitors() {
final List<String> buckets =
- indexBucketLocator.getBuckets( applicationId, IndexBucketLocator.IndexType.CONNECTION, prefix );
+ indexBucketLocator.getBuckets( );
final List<SearchVisitor> visitors = new ArrayList<SearchVisitor>( buckets.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
new file mode 100644
index 0000000..39f3683
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+
+
+/**
+ * Class that performs validation on an entity to ensure it's in the shard we expect
+ */
+public final class ConnectionShardFilter implements ShardFilter {
+ private final IndexBucketLocator indexBucketLocator;
+ private final String expectedBucket;
+ private final ConnectionRefImpl searchConnection;
+
+
+ public ConnectionShardFilter( final IndexBucketLocator indexBucketLocator, final String expectedBucket,
+ final ConnectionRefImpl connection ) {
+ this.indexBucketLocator = indexBucketLocator;
+ this.expectedBucket = expectedBucket;
+ this.searchConnection = connection;
+
+
+ }
+
+
+
+
+
+ public boolean isInShard( final ScanColumn scanColumn ) {
+
+ final UUID entityId = scanColumn.getUUID();
+
+ final ConnectionRefImpl hashRef = new ConnectionRefImpl( searchConnection.getConnectingEntityType(), searchConnection.getConnectedEntityId(), searchConnection.getConnectionType(), searchConnection.getConnectingEntityType(), entityId );
+
+ final UUID hashId = hashRef.getConnectionSearchShardId();
+
+ //not for our current processing shard, discard
+ final String shard = indexBucketLocator.getBucket( hashId );
+
+ return expectedBucket.equals( shard );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index 3dfce2c..73eb148 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -108,7 +108,7 @@ public class SearchCollectionVisitor extends SearchVisitor {
queryProcessor.getPageSizeHint( node ), query.isReversed(), bucket, applicationId,
node.isForceKeepFirst() );
- this.results.push( new SliceIterator( slice, indexScanner, uuidIndexSliceParser ) );
+ this.results.push( new SliceIterator( indexScanner, uuidIndexSliceParser ) );
}
@@ -128,14 +128,16 @@ public class SearchCollectionVisitor extends SearchVisitor {
final int size = queryProcessor.getPageSizeHint( node );
GeoIterator itr = new GeoIterator(
- new CollectionGeoSearch( em, indexBucketLocator, cassandraService, headEntity, collection.getName() ),
+ new CollectionGeoSearch( em, bucket, cassandraService, headEntity, collection.getName() ),
size, slice, node.getPropertyName(), new Point( node.getLattitude(), node.getLongitude() ),
node.getDistance() );
+ this.results.push( itr );
- final SliceShardFilterIterator.ShardBucketValidator validator = new SliceShardFilterIterator.ShardBucketValidator(indexBucketLocator, bucket, applicationId, IndexBucketLocator.IndexType.COLLECTION, collection.getName() );
-
- this.results.push( new SliceShardFilterIterator( validator, itr, size));
+// final CollectionShardFilter
+// validator = new CollectionShardFilter(indexBucketLocator, bucket );
+//
+// this.results.push( new ShardFilterIterator( validator, itr, size));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 3923bf4..404420a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -113,15 +113,19 @@ public class SearchConnectionVisitor extends SearchVisitor {
+ //TODO, make search take a shard
GeoIterator itr =
- new GeoIterator( new ConnectionGeoSearch( em, indexBucketLocator, cassandraService, connection.getIndexId() ),
+ new GeoIterator( new ConnectionGeoSearch( em, bucket, cassandraService, connection.getIndexId() ),
size, slice, node.getPropertyName(),
new Point( node.getLattitude(), node.getLongitude() ), node.getDistance() );
- final SliceShardFilterIterator.ShardBucketValidator validator = new SliceShardFilterIterator.ShardBucketValidator(indexBucketLocator, bucket, applicationId, IndexBucketLocator.IndexType.CONNECTION, connection.getSearchIndexName() );
+ this.results.push( itr );
+// final CollectionShardFilter
+// validator = new CollectionShardFilter(indexBucketLocator, bucket );
- this.results.push( new SliceShardFilterIterator( validator, itr, size ) );
+
+// this.results.push( new ShardFilterIterator( validator, itr, size ) );
}
@@ -193,13 +197,15 @@ public class SearchConnectionVisitor extends SearchVisitor {
start, slice.isReversed(), size, skipFirst );
//we have to create our wrapper so validate the data we read is correct for our shard
- final SliceShardFilterIterator.ShardBucketValidator validator = new SliceShardFilterIterator.ShardBucketValidator(indexBucketLocator, bucket, applicationId, IndexBucketLocator.IndexType.CONNECTION, "" );
- final SliceIterator sliceIterator = new SliceIterator( slice, connectionScanner, connectionParser );
+ final ConnectionShardFilter connectionShardFilter = new ConnectionShardFilter( indexBucketLocator, bucket, connection);
+
+
+ final SliceIterator sliceIterator = new SliceIterator( connectionScanner, connectionParser );
- this.results.push( new SliceShardFilterIterator( validator, sliceIterator, size));
+ this.results.push( new ShardFilterIterator( connectionShardFilter, sliceIterator, size));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilter.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilter.java
new file mode 100644
index 0000000..25383bd
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilter.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+public interface ShardFilter {
+
+ /**
+ * Return true if the column should be retained, false otherwise
+ * @param scanColumn
+ * @return
+ */
+ boolean isInShard( final ScanColumn scanColumn );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIterator.java
new file mode 100644
index 0000000..3a97fb7
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIterator.java
@@ -0,0 +1,124 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * An iterator that will check if the parsed column is part of this shard. This is required due to a legacy storage
+ * format in both connection pointers, as well as geo points.
+ *
+ * Some formats are not sharded by target entity, as a result, we get data partition mismatches when performing
+ * intersections and seeks. This is meant to discard target entities that are not part of the current shard
+ *
+ * @author tnine
+ */
+public class ShardFilterIterator implements ResultIterator {
+
+ private static final Logger logger = LoggerFactory.getLogger( ShardFilterIterator.class );
+
+ private final ShardFilter shardFilter;
+ private final ResultIterator resultsIterator;
+ private final int pageSize;
+
+ private Set<ScanColumn> current;
+
+
+ /**
+ * @param shardFilter The validator to use when validating results belong to a shard
+ * @param resultsIterator The iterator to filter results from
+ * @param pageSize
+ */
+ public ShardFilterIterator( final ShardFilter shardFilter, final ResultIterator resultsIterator,
+ final int pageSize ) {
+ this.shardFilter = shardFilter;
+ this.resultsIterator = resultsIterator;
+ this.pageSize = pageSize;
+ }
+
+
+
+ @Override
+ public void reset() {
+ current = null;
+ resultsIterator.reset();
+ }
+
+
+
+ @Override
+ public Iterator<Set<ScanColumn>> iterator() {
+ return this;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+ if(current == null){
+ advance();
+ }
+
+ return current != null && current.size() > 0;
+ }
+
+
+ @Override
+ public Set<ScanColumn> next() {
+
+ final Set<ScanColumn> toReturn = current;
+
+ current = null;
+
+ return toReturn;
+ }
+
+
+ /**
+ * Advance the column pointers
+ */
+ private void advance(){
+
+ final Set<ScanColumn> results = new LinkedHashSet<ScanColumn>( );
+
+ while(resultsIterator.hasNext()){
+
+ final Iterator<ScanColumn> scanColumns = resultsIterator.next().iterator();
+
+
+ while(results.size() < pageSize && scanColumns.hasNext()){
+ final ScanColumn scanColumn = scanColumns.next();
+
+ if( shardFilter.isInShard( scanColumn )){
+ results.add( scanColumn );
+ }
+ }
+ }
+
+ current = results;
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
index 73cecd7..f71e617 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -42,9 +42,7 @@ import me.prettyprint.hector.api.beans.HColumn;
*/
public class SliceIterator implements ResultIterator {
- private static final Logger logger = LoggerFactory.getLogger( SliceIterator.class );
- private final QuerySlice slice;
protected final SliceParser parser;
protected final IndexScanner scanner;
private final int pageSize;
@@ -66,19 +64,13 @@ public class SliceIterator implements ResultIterator {
*/
private int pagesLoaded = 0;
- /**
- * Pointer to the last column we parsed
- */
- private ScanColumn last;
/**
* @param scanner The scanner to use to read the cols
- * @param slice The slice used in the scanner
* @param parser The parser for the scanner results
*/
- public SliceIterator( QuerySlice slice, IndexScanner scanner, SliceParser parser ) {
- this.slice = slice;
+ public SliceIterator( IndexScanner scanner, SliceParser parser ) {
this.parser = parser;
this.scanner = scanner;
this.pageSize = scanner.getPageSize();
@@ -136,7 +128,6 @@ public class SliceIterator implements ResultIterator {
continue;
}
- last = parsed;
parsedCols.add( parsed );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
deleted file mode 100644
index 1664627..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.usergrid.persistence.query.ir.result;
-
-
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.Set;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.cassandra.CursorCache;
-
-
-/**
- * An iterator that will check if the parsed column is part of this shard. This is required due to a legacy storage
- * format in both connection pointers, as well as geo points.
- *
- * Some formats are not sharded by target entity, as a result, we get data partition mismatches when performing
- * intersections and seeks. This is meant to discard target entities that are not part of the current shard
- *
- * @author tnine
- */
-public class SliceShardFilterIterator implements ResultIterator {
-
- private static final Logger logger = LoggerFactory.getLogger( SliceShardFilterIterator.class );
-
- private final ShardBucketValidator shardBucketValidator;
- private final ResultIterator resultsIterator;
- private final int pageSize;
-
- private Set<ScanColumn> current;
-
-
- /**
- * @param shardBucketValidator The validator to use when validating results belong to a shard
- * @param resultsIterator The iterator to filter results from
- * @param pageSize
- */
- public SliceShardFilterIterator( final ShardBucketValidator shardBucketValidator,
- final ResultIterator resultsIterator, final int pageSize ) {
- this.shardBucketValidator = shardBucketValidator;
- this.resultsIterator = resultsIterator;
- this.pageSize = pageSize;
- }
-
-
-
- @Override
- public void reset() {
- current = null;
- resultsIterator.reset();
- }
-
-
-
- @Override
- public Iterator<Set<ScanColumn>> iterator() {
- return this;
- }
-
-
- @Override
- public boolean hasNext() {
- if(current == null){
- advance();
- }
-
- return current != null && current.size() > 0;
- }
-
-
- @Override
- public Set<ScanColumn> next() {
-
- final Set<ScanColumn> toReturn = current;
-
- current = null;
-
- return toReturn;
- }
-
-
- /**
- * Advance the column pointers
- */
- private void advance(){
-
- final Set<ScanColumn> results = new LinkedHashSet<ScanColumn>( );
-
- while(resultsIterator.hasNext()){
-
- final Iterator<ScanColumn> scanColumns = resultsIterator.next().iterator();
-
-
- while(results.size() < pageSize && scanColumns.hasNext()){
- final ScanColumn scanColumn = scanColumns.next();
-
- if(shardBucketValidator.isInShard( scanColumn.getUUID() )){
- results.add( scanColumn );
- }
- }
- }
-
- current = results;
-
-
- }
-
-
-
- /**
- * Class that performs validation on an entity to ensure it's in the shard we expecte
- */
- public static final class ShardBucketValidator {
- private final IndexBucketLocator indexBucketLocator;
- private final String expectedBucket;
- private final UUID applicationId;
- private final IndexBucketLocator.IndexType type;
- private final String[] components;
-
-
- public ShardBucketValidator( final IndexBucketLocator indexBucketLocator, final String expectedBucket,
- final UUID applicationId, final IndexBucketLocator.IndexType type,
- final String... components ) {
- this.indexBucketLocator = indexBucketLocator;
- this.expectedBucket = expectedBucket;
- this.applicationId = applicationId;
- this.type = type;
- this.components = components;
- }
-
-
- public boolean isInShard( final UUID entityId ) {
- //not for our current processing shard, discard
- final String shard = indexBucketLocator.getBucket( applicationId, type, entityId, components );
-
- return expectedBucket.equals( shard );
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
index 8c9e9b7..8e6fb80 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
@@ -76,8 +76,10 @@ public class GeoIT extends AbstractCoreIT {
Point center = new Point( 37.774277, -122.404744 );
+ final String shard = setup.getIbl().getBucket( user.getUuid() );
+
CollectionGeoSearch connSearch =
- new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(), "users" );
+ new CollectionGeoSearch( em, shard, setup.getCassSvc(), em.getApplicationRef(), "users" );
List<EntityLocationRef> listResults =
@@ -419,83 +421,86 @@ public class GeoIT extends AbstractCoreIT {
}
- @Test
- public void testDenseSearch() throws Exception {
-
- UUID applicationId = setup.createApplication( "testOrganization", "testDenseSearch" );
- assertNotNull( applicationId );
-
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
- assertNotNull( em );
-
- // save objects in a diagonal line from -90 -180 to 90 180
-
- int numEntities = 500;
-
- float minLattitude = 48.32455f;
- float maxLattitude = 48.46481f;
- float minLongitude = 9.89561f;
- float maxLongitude = 10.0471f;
-
- float lattitudeDelta = ( maxLattitude - minLattitude ) / numEntities;
-
- float longitudeDelta = ( maxLongitude - minLongitude ) / numEntities;
-
- for ( int i = 0; i < numEntities; i++ ) {
- float lattitude = minLattitude + lattitudeDelta * i;
- float longitude = minLongitude + longitudeDelta * i;
-
- Map<String, Float> location = MapUtils.hashMap( "latitude", lattitude ).map( "longitude", longitude );
-
- Map<String, Object> data = new HashMap<String, Object>( 2 );
- data.put( "name", String.valueOf( i ) );
- data.put( "location", location );
-
- em.create( "store", data );
- }
-
- //do a direct geo iterator test. We need to make sure that we short circuit on the correct tile.
-
- float lattitude = 48.38626f;
- float longtitude = 9.94175f;
- int distance = 1000;
- int limit = 8;
-
-
- QuerySlice slice = new QuerySlice( "location", 0 );
-
- GeoIterator itr = new GeoIterator(
- new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(), "stores" ),
- limit, slice, "location", new Point( lattitude, longtitude ), distance );
-
-
- // check we got back all 500 entities
- assertFalse( itr.hasNext() );
-
- List<String> cells = itr.getLastCellsSearched();
-
- assertEquals( 1, cells.size() );
-
- assertEquals( 4, cells.get( 0 ).length() );
-
-
- long startTime = System.currentTimeMillis();
-
- //now test at the EM level, there should be 0 results.
- Query query = new Query();
-
- query.addFilter( "location within 1000 of 48.38626, 9.94175" );
- query.setLimit( 8 );
-
-
- Results results = em.searchCollection( em.getApplicationRef(), "stores", query );
-
- assertEquals( 0, results.size() );
-
- long endTime = System.currentTimeMillis();
-
- LOG.info( "Runtime took {} milliseconds to search", endTime - startTime );
- }
+// @Test
+// public void testDenseSearch() throws Exception {
+//
+// UUID applicationId = setup.createApplication( "testOrganization", "testDenseSearch" );
+// assertNotNull( applicationId );
+//
+// EntityManager em = setup.getEmf().getEntityManager( applicationId );
+// assertNotNull( em );
+//
+// // save objects in a diagonal line from -90 -180 to 90 180
+//
+// int numEntities = 500;
+//
+// float minLattitude = 48.32455f;
+// float maxLattitude = 48.46481f;
+// float minLongitude = 9.89561f;
+// float maxLongitude = 10.0471f;
+//
+// float lattitudeDelta = ( maxLattitude - minLattitude ) / numEntities;
+//
+// float longitudeDelta = ( maxLongitude - minLongitude ) / numEntities;
+//
+// for ( int i = 0; i < numEntities; i++ ) {
+// float lattitude = minLattitude + lattitudeDelta * i;
+// float longitude = minLongitude + longitudeDelta * i;
+//
+// Map<String, Float> location = MapUtils.hashMap( "latitude", lattitude ).map( "longitude", longitude );
+//
+// Map<String, Object> data = new HashMap<String, Object>( 2 );
+// data.put( "name", String.valueOf( i ) );
+// data.put( "location", location );
+//
+// em.create( "store", data );
+// }
+//
+// //do a direct geo iterator test. We need to make sure that we short circuit on the correct tile.
+//
+// float lattitude = 48.38626f;
+// float longtitude = 9.94175f;
+// int distance = 1000;
+// int limit = 8;
+//
+//
+// QuerySlice slice = new QuerySlice( "location", 0 );
+//
+// //TODO test this
+// final String shard = setup.getIbl().getBucket( user.getUuid() );
+//
+// GeoIterator itr = new GeoIterator(
+// new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(), "stores" ),
+// limit, slice, "location", new Point( lattitude, longtitude ), distance );
+//
+//
+// // check we got back all 500 entities
+// assertFalse( itr.hasNext() );
+//
+// List<String> cells = itr.getLastCellsSearched();
+//
+// assertEquals( 1, cells.size() );
+//
+// assertEquals( 4, cells.get( 0 ).length() );
+//
+//
+// long startTime = System.currentTimeMillis();
+//
+// //now test at the EM level, there should be 0 results.
+// Query query = new Query();
+//
+// query.addFilter( "location within 1000 of 48.38626, 9.94175" );
+// query.setLimit( 8 );
+//
+//
+// Results results = em.searchCollection( em.getApplicationRef(), "stores", query );
+//
+// assertEquals( 0, results.size() );
+//
+// long endTime = System.currentTimeMillis();
+//
+// LOG.info( "Runtime took {} milliseconds to search", endTime - startTime );
+// }
public Map<String, Object> getLocation( double latitude, double longitude ) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/SimpleIndexBucketLocatorImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/SimpleIndexBucketLocatorImplTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/SimpleIndexBucketLocatorImplTest.java
index 2ef3ea1..81a33d2 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/SimpleIndexBucketLocatorImplTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/SimpleIndexBucketLocatorImplTest.java
@@ -25,8 +25,8 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.junit.Test;
+
import org.apache.usergrid.cassandra.Concurrent;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.utils.UUIDUtils;
import com.yammer.metrics.Metrics;
@@ -48,7 +48,7 @@ public class SimpleIndexBucketLocatorImplTest {
SimpleIndexBucketLocatorImpl locator = new SimpleIndexBucketLocatorImpl( 1 );
- List<String> buckets = locator.getBuckets( appId, IndexType.COLLECTION, entityType, propName );
+ List<String> buckets = locator.getBuckets( );
assertEquals( 1, buckets.size() );
@@ -58,11 +58,11 @@ public class SimpleIndexBucketLocatorImplTest {
UUID testId3 = UUIDUtils.minTimeUUID( Long.MAX_VALUE );
- String bucket1 = locator.getBucket( appId, IndexType.COLLECTION, testId1, entityType, propName );
+ String bucket1 = locator.getBucket( testId1);
- String bucket2 = locator.getBucket( appId, IndexType.COLLECTION, testId2, entityType, propName );
+ String bucket2 = locator.getBucket( testId2);
- String bucket3 = locator.getBucket( appId, IndexType.COLLECTION, testId3, entityType, propName );
+ String bucket3 = locator.getBucket( testId3 );
assertEquals( bucket1, "000000000000000000000000000000000000000" );
assertEquals( bucket2, "000000000000000000000000000000000000000" );
@@ -79,7 +79,7 @@ public class SimpleIndexBucketLocatorImplTest {
SimpleIndexBucketLocatorImpl locator = new SimpleIndexBucketLocatorImpl( 2 );
- List<String> buckets = locator.getBuckets( appId, IndexType.COLLECTION, entityType, propName );
+ List<String> buckets = locator.getBuckets( );
assertEquals( 2, buckets.size() );
@@ -89,11 +89,11 @@ public class SimpleIndexBucketLocatorImplTest {
UUID testId3 = UUIDUtils.minTimeUUID( Long.MAX_VALUE );
- String bucket1 = locator.getBucket( appId, IndexType.COLLECTION, testId1, entityType, propName );
+ String bucket1 = locator.getBucket( testId1 );
- String bucket2 = locator.getBucket( appId, IndexType.COLLECTION, testId2, entityType, propName );
+ String bucket2 = locator.getBucket( testId2);
- String bucket3 = locator.getBucket( appId, IndexType.COLLECTION, testId3, entityType, propName );
+ String bucket3 = locator.getBucket( testId3);
assertEquals( bucket1, "000000000000000000000000000000000000000" );
assertEquals( bucket2, "085070591730234615865843651857942052863" );
@@ -114,7 +114,7 @@ public class SimpleIndexBucketLocatorImplTest {
// test 100 elements
SimpleIndexBucketLocatorImpl locator = new SimpleIndexBucketLocatorImpl( bucketSize );
- List<String> buckets = locator.getBuckets( appId, IndexType.COLLECTION, entityType, propName );
+ List<String> buckets = locator.getBuckets( );
assertEquals( bucketSize, buckets.size() );
@@ -136,7 +136,7 @@ public class SimpleIndexBucketLocatorImplTest {
final TimerContext context = hashes.time();
- String bucket = locator.getBucket( appId, IndexType.COLLECTION, id, entityType, propName );
+ String bucket = locator.getBucket( id );
context.stop();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
index 1d0d0cb..f6b2661 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
@@ -371,8 +371,7 @@ public abstract class AbstractIteratingQueryIT {
io.doSetup();
-// int size = 2000;
- int size = 2;
+ int size = 2000;
int queryLimit = Query.MAX_LIMIT;
// the number of entities that should be written including an intersection
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIteratorTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIteratorTest.java
new file mode 100644
index 0000000..6fa686f
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIteratorTest.java
@@ -0,0 +1,175 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.SimpleIndexBucketLocatorImpl;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Simple test to test UUID
+ */
+public class ShardFilterIteratorTest {
+
+ @Test
+ public void testIndexValues() {
+
+ int size = 100;
+
+ final Multimap<String, ScanColumn> shards = HashMultimap.create();
+
+ final IndexBucketLocator indexBucketLocator = new SimpleIndexBucketLocatorImpl( 20 );
+
+
+ final UUID applicationId = UUIDUtils.newTimeUUID();
+
+
+ final String components = "things";
+
+
+ final Set<ScanColumn> allColumns = new LinkedHashSet<ScanColumn>( size );
+
+
+ final UUIDCursorGenerator uuidCursorGenerator = new UUIDCursorGenerator( 1 );
+
+ for ( int i = 0; i < size; i++ ) {
+ final UUID entityId = UUIDUtils.newTimeUUID();
+
+ final String shard = indexBucketLocator.getBucket(entityId );
+
+ final UUIDColumn uuidColumn = new UUIDColumn( entityId, 1, uuidCursorGenerator );
+
+ //add the shard to our assertion set
+ shards.put( shard, uuidColumn );
+
+ allColumns.add( uuidColumn );
+ }
+
+ //now create an iterator with all the uuid sand verity they're correct.
+
+
+ for ( final String shard : shards.keySet() ) {
+ //create a copy of our expected uuids
+ final Set<ScanColumn> expected = new HashSet<ScanColumn>( shards.get( shard ) );
+
+
+ final TestIterator testIterator = new TestIterator( new HashSet<ScanColumn>( shards.get( shard ) ) );
+
+
+ final TestEntityFilter collectionSliceShardFilter =
+ new TestEntityFilter( indexBucketLocator, shard );
+
+
+ //now iterate over everything and remove it from expected
+ final ShardFilterIterator shardFilterIterator = new ShardFilterIterator(
+ collectionSliceShardFilter, testIterator, 10 );
+
+ //keep removing
+ while( shardFilterIterator.hasNext()){
+
+ //check each scan column from our results
+ for(final ScanColumn column : shardFilterIterator.next()){
+
+ final boolean contained = expected.remove( column );
+
+ assertTrue( "Column should be present", contained );
+
+ }
+
+
+ }
+
+ assertTrue("expected should be empty", expected.isEmpty());
+ }
+
+ }
+
+ private static final class TestEntityFilter implements ShardFilter{
+
+ private final IndexBucketLocator indexBucketLocator;
+ private final String expectedShard;
+
+
+ private TestEntityFilter( final IndexBucketLocator indexBucketLocator, final String expectedShard ) {
+ this.indexBucketLocator = indexBucketLocator;
+ this.expectedShard = expectedShard;
+ }
+
+
+ @Override
+ public boolean isInShard( final ScanColumn scanColumn ) {
+
+ final String shard = indexBucketLocator.getBucket( scanColumn.getUUID() );
+
+ return expectedShard.equals( shard );
+ }
+ }
+
+
+ private static final class TestIterator implements ResultIterator {
+
+
+ private final Set<ScanColumn> scanColumns;
+ private boolean completed;
+
+
+ private TestIterator( final Set<ScanColumn> scanColumns ) {this.scanColumns = scanColumns;}
+
+
+ @Override
+ public void reset() {
+ //no op
+ }
+
+
+ @Override
+ public Iterator<Set<ScanColumn>> iterator() {
+ return this;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+ return !completed;
+ }
+
+
+ @Override
+ public Set<ScanColumn> next() {
+ completed = true;
+ return scanColumns;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9971068a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java
deleted file mode 100644
index 9079582..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.usergrid.persistence.query.ir.result;
-
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.Set;
-import java.util.UUID;
-
-import org.junit.Test;
-
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.cassandra.SimpleIndexBucketLocatorImpl;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-
-import static org.junit.Assert.assertTrue;
-
-
-/**
- * Simple test to test UUID
- */
-public class SliceShardFilterIteratorTest {
-
- @Test
- public void testIndexValues() {
-
- int size = 100;
-
- final Multimap<String, ScanColumn> shards = HashMultimap.create();
-
- final IndexBucketLocator indexBucketLocator = new SimpleIndexBucketLocatorImpl( 20 );
-
-
- final UUID applicationId = UUIDUtils.newTimeUUID();
-
- final IndexBucketLocator.IndexType indexType = IndexBucketLocator.IndexType.COLLECTION;
-
- final String components = "things";
-
-
- final Set<ScanColumn> allColumns = new LinkedHashSet<ScanColumn>( size );
-
-
- final UUIDCursorGenerator uuidCursorGenerator = new UUIDCursorGenerator( 1 );
-
- for ( int i = 0; i < size; i++ ) {
- final UUID entityId = UUIDUtils.newTimeUUID();
-
- final String shard = indexBucketLocator.getBucket( applicationId, indexType, entityId, components );
-
- final UUIDColumn uuidColumn = new UUIDColumn( entityId, 1, uuidCursorGenerator );
-
- //add the shard to our assertion set
- shards.put( shard, uuidColumn );
-
- allColumns.add( uuidColumn );
- }
-
- //now create an iterator with all the uuid sand verity they're correct.
-
-
- for ( final String shard : shards.keySet() ) {
- //create a copy of our expected uuids
- final Set<ScanColumn> expected = new HashSet<ScanColumn>( shards.get( shard ) );
-
-
- final TestIterator testIterator = new TestIterator( new HashSet<ScanColumn>( shards.get( shard ) ) );
-
-
- final SliceShardFilterIterator.ShardBucketValidator shardBucketValidator =
- new SliceShardFilterIterator.ShardBucketValidator( indexBucketLocator, shard, applicationId,
- indexType, components );
-
-
- //now iterate over everything and remove it from expected
- final SliceShardFilterIterator sliceShardFilterIterator = new SliceShardFilterIterator( shardBucketValidator, testIterator, 10 );
-
- //keep removing
- while(sliceShardFilterIterator.hasNext()){
-
- //check each scan column from our results
- for(final ScanColumn column : sliceShardFilterIterator.next()){
-
- final boolean contained = expected.remove( column );
-
- assertTrue("Column should be present", contained);
-
- }
-
-
- }
-
- assertTrue("expected should be empty", expected.isEmpty());
- }
-
- }
-
-
- private static final class TestIterator implements ResultIterator {
-
-
- private final Set<ScanColumn> scanColumns;
- private boolean completed;
-
-
- private TestIterator( final Set<ScanColumn> scanColumns ) {this.scanColumns = scanColumns;}
-
-
- @Override
- public void reset() {
- //no op
- }
-
-
- @Override
- public Iterator<Set<ScanColumn>> iterator() {
- return this;
- }
-
-
- @Override
- public boolean hasNext() {
- return !completed;
- }
-
-
- @Override
- public Set<ScanColumn> next() {
- completed = true;
- return scanColumns;
- }
- }
-}
[23/25] incubator-usergrid git commit: Fixes shard allocation. Keeps
old allocation logic for deletion.
Posted by sn...@apache.org.
Fixes shard allocation. Keeps old allocation logic for deletion.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d1ca419f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d1ca419f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d1ca419f
Branch: refs/heads/master
Commit: d1ca419f3def8e92cefec8c84508185823d3d4ac
Parents: a22c996
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Jul 9 13:44:50 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Jul 9 13:44:50 2015 -0600
----------------------------------------------------------------------
.../persistence/cassandra/GeoIndexManager.java | 78 ++++++++++++++----
.../cassandra/QueryExecutorServiceImpl.java | 25 +++++-
.../cassandra/RelationManagerImpl.java | 86 ++++++++++++++++----
.../query/ir/result/ConnectionShardFilter.java | 26 ++----
.../ir/result/SearchConnectionVisitor.java | 4 +-
5 files changed, 165 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1ca419f/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/GeoIndexManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/GeoIndexManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/GeoIndexManager.java
index ce5fab8..109881f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/GeoIndexManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/GeoIndexManager.java
@@ -95,33 +95,34 @@ public class GeoIndexManager {
private static Mutator<ByteBuffer> batchAddConnectionIndexEntries( Mutator<ByteBuffer> m,
- IndexBucketLocator locator, UUID appId,
+ IndexBucketLocator locator, UUID entityId,
String propertyName, String geoCell,
UUID[] index_keys, ByteBuffer columnName,
ByteBuffer columnValue, long timestamp ) {
+ final String bucket = locator.getBucket( entityId );
+
+
// entity_id,prop_name
Object property_index_key =
key( index_keys[ConnectionRefImpl.ALL], INDEX_CONNECTIONS, propertyName, DICTIONARY_GEOCELL, geoCell,
- locator.getBucket(index_keys[ConnectionRefImpl.ALL] ) );
+ bucket );
// entity_id,entity_type,prop_name
Object entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], INDEX_CONNECTIONS, propertyName, DICTIONARY_GEOCELL,
geoCell,
- locator.getBucket( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE] ) );
+ bucket );
// entity_id,connection_type,prop_name
Object connection_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], INDEX_CONNECTIONS, propertyName,
- DICTIONARY_GEOCELL, geoCell, locator.getBucket(
- index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE] ) );
+ DICTIONARY_GEOCELL, geoCell, bucket );
// entity_id,connection_type,entity_type,prop_name
Object connection_type_and_entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], INDEX_CONNECTIONS, propertyName,
- DICTIONARY_GEOCELL, geoCell, locator.getBucket(
- index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE] ) );
+ DICTIONARY_GEOCELL, geoCell, bucket );
// composite(property_value,connected_entity_id,connection_type,entity_type,entry_timestamp)
addInsertToMutator( m, ENTITY_INDEX, property_index_key, columnName, columnValue, timestamp );
@@ -151,7 +152,7 @@ public class GeoIndexManager {
ByteBuffer columnValue = location.getColumnValue().serialize();
long ts = location.getTimestampInMicros();
for ( String cell : cells ) {
- batchAddConnectionIndexEntries( m, locator, appId, propertyName, cell, index_keys, columnName, columnValue,
+ batchAddConnectionIndexEntries( m, locator, location.getUuid(), propertyName, cell, index_keys, columnName, columnValue,
ts );
}
@@ -177,11 +178,15 @@ public class GeoIndexManager {
private static Mutator<ByteBuffer> batchDeleteConnectionIndexEntries( Mutator<ByteBuffer> m,
- IndexBucketLocator locator, UUID appId,
+ IndexBucketLocator locator, UUID entityId,
String propertyName, String geoCell,
UUID[] index_keys, ByteBuffer columnName,
long timestamp ) {
+ /**
+ * Legacy key scheme
+ */
+
// entity_id,prop_name
Object property_index_key =
key( index_keys[ConnectionRefImpl.ALL], INDEX_CONNECTIONS, propertyName, DICTIONARY_GEOCELL, geoCell,
@@ -221,6 +226,49 @@ public class GeoIndexManager {
m.addDeletion( bytebuffer( connection_type_and_entity_type_prop_index_key ), ENTITY_INDEX.toString(),
columnName, ByteBufferSerializer.get(), timestamp );
+
+ /**
+ * New key scheme
+ */
+
+ final String bucket = locator.getBucket( entityId );
+
+ // entity_id,prop_name
+ Object property_index_key_new =
+ key( index_keys[ConnectionRefImpl.ALL], INDEX_CONNECTIONS, propertyName, DICTIONARY_GEOCELL, geoCell,
+ bucket );
+
+ // entity_id,entity_type,prop_name
+ Object entity_type_prop_index_key_new =
+ key( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], INDEX_CONNECTIONS, propertyName, DICTIONARY_GEOCELL,
+ geoCell, bucket );
+
+ // entity_id,connection_type,prop_name
+ Object connection_type_prop_index_key_new =
+ key( index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], INDEX_CONNECTIONS, propertyName,
+ DICTIONARY_GEOCELL, geoCell, bucket );
+
+ // entity_id,connection_type,entity_type,prop_name
+ Object connection_type_and_entity_type_prop_index_key_new =
+ key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], INDEX_CONNECTIONS, propertyName,
+ DICTIONARY_GEOCELL, geoCell, bucket );
+
+ // composite(property_value,connected_entity_id,connection_type,entity_type,entry_timestamp)
+ m.addDeletion( bytebuffer( property_index_key_new ), ENTITY_INDEX.toString(), columnName,
+ ByteBufferSerializer.get(), timestamp );
+
+ // composite(property_value,connected_entity_id,connection_type,entry_timestamp)
+ m.addDeletion( bytebuffer( entity_type_prop_index_key_new ), ENTITY_INDEX.toString(), columnName,
+ ByteBufferSerializer.get(), timestamp );
+
+ // composite(property_value,connected_entity_id,entity_type,entry_timestamp)
+ m.addDeletion( bytebuffer( connection_type_prop_index_key_new ), ENTITY_INDEX.toString(), columnName,
+ ByteBufferSerializer.get(), timestamp );
+
+ // composite(property_value,connected_entity_id,entry_timestamp)
+ m.addDeletion( bytebuffer( connection_type_and_entity_type_prop_index_key_new ), ENTITY_INDEX.toString(),
+ columnName, ByteBufferSerializer.get(), timestamp );
+
return m;
}
@@ -238,7 +286,7 @@ public class GeoIndexManager {
for ( String cell : cells ) {
- batchDeleteConnectionIndexEntries( m, locator, appId, propertyName, cell, index_keys, columnName, ts );
+ batchDeleteConnectionIndexEntries( m, locator, location.getUuid(), propertyName, cell, index_keys, columnName, ts );
}
logger.info( "Geocells to be saved for Point({} , {} ) are: {}", new Object[] {
@@ -247,8 +295,7 @@ public class GeoIndexManager {
}
- public static void batchStoreLocationInCollectionIndex( Mutator<ByteBuffer> m, IndexBucketLocator locator,
- UUID appId, Object key, UUID entityId,
+ public static void batchStoreLocationInCollectionIndex( Mutator<ByteBuffer> m, IndexBucketLocator locator, Object key, UUID entityId,
EntityLocationRef location ) {
Point p = location.getPoint();
@@ -276,15 +323,14 @@ public class GeoIndexManager {
Keyspace ko = cass.getApplicationKeyspace( em.getApplicationId() );
Mutator<ByteBuffer> m = CountingMutator.createFlushingMutator( ko, ByteBufferSerializer.get() );
- batchStoreLocationInCollectionIndex( m, em.getIndexBucketLocator(), em.getApplicationId(),
+ batchStoreLocationInCollectionIndex( m, em.getIndexBucketLocator(),
key( owner.getUuid(), collectionName, propertyName ), owner.getUuid(), location );
batchExecute( m, CassandraService.RETRY_COUNT );
}
- public static void batchRemoveLocationFromCollectionIndex( Mutator<ByteBuffer> m, IndexBucketLocator locator,
- UUID appId, Object key, EntityLocationRef location ) {
+ public static void batchRemoveLocationFromCollectionIndex( Mutator<ByteBuffer> m, IndexBucketLocator locator, Object key, EntityLocationRef location ) {
Point p = location.getPoint();
List<String> cells = GeocellManager.generateGeoCell( p );
@@ -314,7 +360,7 @@ public class GeoIndexManager {
Keyspace ko = cass.getApplicationKeyspace( em.getApplicationId() );
Mutator<ByteBuffer> m = CountingMutator.createFlushingMutator( ko, ByteBufferSerializer.get() );
- batchRemoveLocationFromCollectionIndex( m, em.getIndexBucketLocator(), em.getApplicationId(),
+ batchRemoveLocationFromCollectionIndex( m, em.getIndexBucketLocator(),
key( owner.getUuid(), collectionName, propertyName ), location );
batchExecute( m, CassandraService.RETRY_COUNT );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1ca419f/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorServiceImpl.java
index 7376641..3504a3f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorServiceImpl.java
@@ -21,8 +21,10 @@ package org.apache.usergrid.persistence.cassandra;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,7 +72,7 @@ public class QueryExecutorServiceImpl implements QueryExecutorService {
}
- executorService = new ThreadPoolExecutor( threadCount, threadCount, 30, TimeUnit.SECONDS, new SynchronousQueue<Runnable>( ), new CallerRunsExecutionHandler() );
+ executorService = new ThreadPoolExecutor( threadCount, threadCount, 30, TimeUnit.SECONDS, new SynchronousQueue<Runnable>( ), new QueryThreadFactory(), new CallerRunsExecutionHandler() );
return executorService;
}
@@ -92,4 +94,25 @@ public class QueryExecutorServiceImpl implements QueryExecutorService {
}
}
+ /**
+ * Simple factory for labeling job worker threads for easier debugging
+ */
+ private static final class QueryThreadFactory implements ThreadFactory {
+
+ public static final QueryThreadFactory INSTANCE = new QueryThreadFactory();
+
+ private static final String NAME = "query-";
+ private final AtomicLong counter = new AtomicLong();
+
+
+ @Override
+ public Thread newThread( final Runnable r ) {
+
+ Thread newThread = new Thread( r, NAME + counter.incrementAndGet() );
+ newThread.setDaemon( true );
+
+ return newThread;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1ca419f/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index 86e1690..587f17d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -250,7 +250,7 @@ public class RelationManagerImpl implements RelationManager {
EntityLocationRef loc =
new EntityLocationRef( indexUpdate.getEntity(), indexEntry.getTimestampUuid(),
indexEntry.getValue().toString() );
- batchStoreLocationInCollectionIndex( indexUpdate.getBatch(), indexBucketLocator, applicationId,
+ batchStoreLocationInCollectionIndex( indexUpdate.getBatch(), indexBucketLocator,
index_name, indexedEntity.getUuid(), loc );
}
@@ -555,6 +555,12 @@ public class RelationManagerImpl implements RelationManager {
ConnectionRefImpl connection, UUID[] index_keys )
throws Exception {
+
+ /**
+ * Original bucket scheme. Incorrect and legacy, but we need to keep it b/c we're not sure if the original write was the
+ * incorrect legacy system
+ */
+
// entity_id,prop_name
Object property_index_key = key( index_keys[ConnectionRefImpl.ALL], INDEX_CONNECTIONS, entry.getPath(),
indexBucketLocator.getBucket( index_keys[ConnectionRefImpl.ALL] ) );
@@ -562,20 +568,19 @@ public class RelationManagerImpl implements RelationManager {
// entity_id,entity_type,prop_name
Object entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket(
- index_keys[ConnectionRefImpl.BY_ENTITY_TYPE] ) );
+ indexBucketLocator.getBucket( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE] ) );
// entity_id,connection_type,prop_name
Object connection_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket(
- index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE]) );
+ indexBucketLocator.getBucket( index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE] ) );
// entity_id,connection_type,entity_type,prop_name
Object connection_type_and_entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket(
- index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE]) );
+ indexBucketLocator.getBucket( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE] ) );
+
+
// composite(property_value,connected_entity_id,connection_type,entity_type,entry_timestamp)
addDeleteToMutator( indexUpdate.getBatch(), ENTITY_INDEX, property_index_key,
@@ -596,35 +601,85 @@ public class RelationManagerImpl implements RelationManager {
addDeleteToMutator( indexUpdate.getBatch(), ENTITY_INDEX, connection_type_and_entity_type_prop_index_key,
entry.getIndexComposite( connection.getConnectedEntityId() ), indexUpdate.getTimestamp() );
+
+ /**
+ * New bucket scheme for deletes
+ */
+
+ final UUID entityId = connection.getConnectedEntityId();
+ final String bucket = indexBucketLocator.getBucket( entityId );
+
+
+ // entity_id,prop_name
+ Object property_index_key_new =
+ key( index_keys[ConnectionRefImpl.ALL], INDEX_CONNECTIONS, entry.getPath(), bucket );
+
+ // entity_id,entity_type,prop_name
+ Object entity_type_prop_index_key_new =
+ key( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], INDEX_CONNECTIONS, entry.getPath(), bucket );
+
+ // entity_id,connection_type,prop_name
+ Object connection_type_prop_index_key_new =
+ key( index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], INDEX_CONNECTIONS, entry.getPath(), bucket );
+
+ // entity_id,connection_type,entity_type,prop_name
+ Object connection_type_and_entity_type_prop_index_key_new =
+ key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], INDEX_CONNECTIONS, entry.getPath(),
+ bucket );
+
+
+ // composite(property_value,connected_entity_id,connection_type,entity_type,entry_timestamp)
+ addDeleteToMutator( indexUpdate.getBatch(), ENTITY_INDEX, property_index_key_new,
+ entry.getIndexComposite( connection.getConnectedEntityId(), connection.getConnectionType(),
+ connection.getConnectedEntityType() ), indexUpdate.getTimestamp() );
+
+ // composite(property_value,connected_entity_id,connection_type,entry_timestamp)
+ addDeleteToMutator( indexUpdate.getBatch(), ENTITY_INDEX, entity_type_prop_index_key_new,
+ entry.getIndexComposite( connection.getConnectedEntityId(), connection.getConnectionType() ),
+ indexUpdate.getTimestamp() );
+
+ // composite(property_value,connected_entity_id,entity_type,entry_timestamp)
+ addDeleteToMutator( indexUpdate.getBatch(), ENTITY_INDEX, connection_type_prop_index_key_new,
+ entry.getIndexComposite( connection.getConnectedEntityId(), connection.getConnectedEntityType() ),
+ indexUpdate.getTimestamp() );
+
+ // composite(property_value,connected_entity_id,entry_timestamp)
+ addDeleteToMutator( indexUpdate.getBatch(), ENTITY_INDEX, connection_type_and_entity_type_prop_index_key_new,
+ entry.getIndexComposite( connection.getConnectedEntityId() ), indexUpdate.getTimestamp() );
+
+
return indexUpdate.getBatch();
}
+
+
@Metered(group = "core", name = "RelationManager_batchAddConnectionIndexEntries")
public Mutator<ByteBuffer> batchAddConnectionIndexEntries( IndexUpdate indexUpdate, IndexEntry entry,
ConnectionRefImpl connection, UUID[] index_keys ) {
+ final UUID entityId = connection.getConnectedEntityId();
+
+ final String bucket = indexBucketLocator.getBucket( entityId );
+
// entity_id,prop_name
Object property_index_key = key( index_keys[ConnectionRefImpl.ALL], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket( index_keys[ConnectionRefImpl.ALL] ) );
+ bucket );
// entity_id,entity_type,prop_name
Object entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_ENTITY_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket(
- index_keys[ConnectionRefImpl.BY_ENTITY_TYPE]) );
+ bucket );
// entity_id,connection_type,prop_name
Object connection_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket(
- index_keys[ConnectionRefImpl.BY_CONNECTION_TYPE] ) );
+ bucket );
// entity_id,connection_type,entity_type,prop_name
Object connection_type_and_entity_type_prop_index_key =
key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], INDEX_CONNECTIONS, entry.getPath(),
- indexBucketLocator.getBucket(
- index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE]) );
+ bucket );
// composite(property_value,connected_entity_id,connection_type,entity_type,entry_timestamp)
addInsertToMutator( indexUpdate.getBatch(), ENTITY_INDEX, property_index_key,
@@ -725,7 +780,8 @@ public class RelationManagerImpl implements RelationManager {
public Set<String> getConnectionIndexes( ConnectionRefImpl connection ) throws Exception {
List<HColumn<String, String>> results =
cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_DICTIONARIES,
- key( connection.getConnectingIndexId(), Schema.DICTIONARY_INDEXES ), Serializers.se, Serializers.se );
+ key( connection.getConnectingIndexId(), Schema.DICTIONARY_INDEXES ), Serializers.se,
+ Serializers.se );
Set<String> indexes = new TreeSet<String>();
if ( results != null ) {
for ( HColumn<String, String> column : results ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1ca419f/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
index f56dc2b..2fec250 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.util.UUID;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
/**
@@ -30,35 +29,22 @@ import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
public final class ConnectionShardFilter implements ShardFilter {
private final IndexBucketLocator indexBucketLocator;
private final String expectedBucket;
- private final ConnectionRefImpl searchConnection;
- public ConnectionShardFilter( final IndexBucketLocator indexBucketLocator, final String expectedBucket,
- final ConnectionRefImpl connection ) {
+ public ConnectionShardFilter( final IndexBucketLocator indexBucketLocator, final String expectedBucket ) {
this.indexBucketLocator = indexBucketLocator;
this.expectedBucket = expectedBucket;
- this.searchConnection = connection;
-
-
}
-
-
-
public boolean isInShard( final ScanColumn scanColumn ) {
- //shard hashing is currently based on source. this is a placeholder for when this is fixed.
-// UUID[] indexIds = searchConnection.getIndexIds();
-//
-// final String shard = indexBucketLocator.getBucket(indexIds[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE] );
-//
-// return expectedBucket.equals( shard );
-
- return true;
-//
- }
+ final UUID entityId = scanColumn.getUUID();
+ //not for our current processing shard, discard
+ final String shard = indexBucketLocator.getBucket( entityId );
+ return expectedBucket.equals( shard );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1ca419f/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index f518297..909ae5d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -136,7 +136,7 @@ public class SearchConnectionVisitor extends SearchVisitor {
final ConnectionShardFilter
- validator = new ConnectionShardFilter(indexBucketLocator, bucket, connection );
+ validator = new ConnectionShardFilter(indexBucketLocator, bucket );
this.results.push( new ShardFilterIterator( validator, itr, size ) );
@@ -213,7 +213,7 @@ public class SearchConnectionVisitor extends SearchVisitor {
//we have to create our wrapper so validate the data we read is correct for our shard
- final ConnectionShardFilter connectionShardFilter = new ConnectionShardFilter( indexBucketLocator, bucket, connection);
+ final ConnectionShardFilter connectionShardFilter = new ConnectionShardFilter( indexBucketLocator, bucket);
final SliceIterator sliceIterator = new SliceIterator( connectionScanner, connectionParser );
[02/25] incubator-usergrid git commit: Second pass with visitor
factory generation
Posted by sn...@apache.org.
Second pass with visitor factory generation
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/aa31768c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/aa31768c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/aa31768c
Branch: refs/heads/master
Commit: aa31768cb8f6826692e4bb06f3459e1c7af4827f
Parents: 9b21332
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jun 16 18:47:34 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jun 16 18:47:34 2015 -0600
----------------------------------------------------------------------
.../cassandra/RelationManagerImpl.java | 336 +------------------
.../cassandra/index/IndexBucketScanner.java | 4 +-
.../persistence/query/ir/SearchVisitor.java | 24 +-
.../result/CollectionSearchVisitorFactory.java | 47 ++-
.../result/ConnectionSearchVisitorFactory.java | 43 ++-
.../ir/result/SearchCollectionVisitor.java | 168 ++++++++++
.../ir/result/SearchConnectionVisitor.java | 213 ++++++++++++
7 files changed, 492 insertions(+), 343 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa31768c/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index 456add3..0e5011f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -20,8 +20,6 @@ package org.apache.usergrid.persistence.cassandra;
import java.nio.ByteBuffer;
import java.util.AbstractMap;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -34,6 +32,7 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+
import org.apache.usergrid.persistence.CollectionRef;
import org.apache.usergrid.persistence.ConnectedEntityRef;
import org.apache.usergrid.persistence.ConnectionRef;
@@ -52,30 +51,17 @@ import org.apache.usergrid.persistence.SimpleCollectionRef;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.SimpleRoleRef;
import org.apache.usergrid.persistence.cassandra.IndexUpdate.IndexEntry;
-import org.apache.usergrid.persistence.cassandra.index.ConnectedIndexScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
-import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
import org.apache.usergrid.persistence.entities.Group;
-import org.apache.usergrid.persistence.geo.CollectionGeoSearch;
-import org.apache.usergrid.persistence.geo.ConnectionGeoSearch;
import org.apache.usergrid.persistence.geo.EntityLocationRef;
-import org.apache.usergrid.persistence.geo.model.Point;
import org.apache.usergrid.persistence.hector.CountingMutator;
-import org.apache.usergrid.persistence.query.ir.AllNode;
-import org.apache.usergrid.persistence.query.ir.NameIdentifierNode;
-import org.apache.usergrid.persistence.query.ir.QueryNode;
import org.apache.usergrid.persistence.query.ir.QuerySlice;
-import org.apache.usergrid.persistence.query.ir.SearchVisitor;
-import org.apache.usergrid.persistence.query.ir.WithinNode;
import org.apache.usergrid.persistence.query.ir.result.CollectionResultsLoaderFactory;
-import org.apache.usergrid.persistence.query.ir.result.ConnectionIndexSliceParser;
import org.apache.usergrid.persistence.query.ir.result.ConnectionResultsLoaderFactory;
import org.apache.usergrid.persistence.query.ir.result.ConnectionTypesIterator;
-import org.apache.usergrid.persistence.query.ir.result.EmptyIterator;
-import org.apache.usergrid.persistence.query.ir.result.GeoIterator;
-import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
-import org.apache.usergrid.persistence.query.ir.result.StaticIdIterator;
+import org.apache.usergrid.persistence.query.ir.result.SearchCollectionVisitor;
+import org.apache.usergrid.persistence.query.ir.result.SearchConnectionVisitor;
import org.apache.usergrid.persistence.query.ir.result.UUIDIndexSliceParser;
import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.apache.usergrid.utils.IndexUtils;
@@ -83,9 +69,6 @@ import org.apache.usergrid.utils.MapUtils;
import com.yammer.metrics.annotation.Metered;
-import me.prettyprint.cassandra.serializers.ByteBufferSerializer;
-import me.prettyprint.cassandra.serializers.StringSerializer;
-import me.prettyprint.cassandra.serializers.UUIDSerializer;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.DynamicComposite;
import me.prettyprint.hector.api.beans.HColumn;
@@ -94,9 +77,7 @@ import me.prettyprint.hector.api.mutation.Mutator;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static java.util.Arrays.asList;
-
import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_COLLECTIONS;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_ENTITIES;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_TYPES;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_ENTITIES;
@@ -1374,41 +1355,8 @@ public class RelationManagerImpl implements RelationManager {
}
- private IndexScanner searchIndex( Object indexKey, QuerySlice slice, int pageSize ) throws Exception {
-
- DynamicComposite[] range = slice.getRange();
- Object keyPrefix = key( indexKey, slice.getPropertyName() );
- IndexScanner scanner =
- new IndexBucketScanner( cass, indexBucketLocator, ENTITY_INDEX, applicationId, IndexType.CONNECTION,
- keyPrefix, range[0], range[1], slice.isReversed(), pageSize, slice.hasCursor(), slice.getPropertyName() );
-
- return scanner;
- }
-
-
- /**
- * Search the collection index using all the buckets for the given collection
- *
- * @param indexKey The index key to read
- * @param slice Slice set in the query
- * @param collectionName The name of the collection to search
- * @param pageSize The page size to load when iterating
- */
- private IndexScanner searchIndexBuckets( Object indexKey, QuerySlice slice, String collectionName, int pageSize )
- throws Exception {
-
- DynamicComposite[] range = slice.getRange();
-
- Object keyPrefix = key( indexKey, slice.getPropertyName() );
-
- IndexScanner scanner =
- new IndexBucketScanner( cass, indexBucketLocator, ENTITY_INDEX, applicationId, IndexType.COLLECTION,
- keyPrefix, range[0], range[1], slice.isReversed(), pageSize, slice.hasCursor(), collectionName );
-
- return scanner;
- }
@SuppressWarnings("unchecked")
@@ -1772,7 +1720,7 @@ public class RelationManagerImpl implements RelationManager {
// we have something to search with, visit our tree and evaluate the
// results
QueryProcessor qp = new QueryProcessor( query, collection, em, factory );
- SearchCollectionVisitor visitor = new SearchCollectionVisitor( qp );
+ SearchCollectionVisitor visitor = new SearchCollectionVisitor( this, qp );
return qp.getResults( visitor );
}
@@ -1966,7 +1914,7 @@ public class RelationManagerImpl implements RelationManager {
final ConnectionResultsLoaderFactory factory = new ConnectionResultsLoaderFactory( connectionRef );
QueryProcessor qp = new QueryProcessor( query, null, em, factory );
- SearchConnectionVisitor visitor = new SearchConnectionVisitor( qp, connectionRef, true );
+ SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, true );
return qp.getResults( visitor );
}
@@ -2008,7 +1956,7 @@ public class RelationManagerImpl implements RelationManager {
final ConnectionResultsLoaderFactory factory = new ConnectionResultsLoaderFactory( connectionRef );
QueryProcessor qp = new QueryProcessor( query, null, em, factory );
- SearchConnectionVisitor visitor = new SearchConnectionVisitor( qp, connectionRef, false );
+ SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, false );
return qp.getResults( visitor );
}
@@ -2047,7 +1995,7 @@ public class RelationManagerImpl implements RelationManager {
final ConnectionResultsLoaderFactory factory = new ConnectionResultsLoaderFactory( connectionRef );
QueryProcessor qp = new QueryProcessor( query, null, em, factory );
- SearchConnectionVisitor visitor = new SearchConnectionVisitor( qp, connectionRef, true );
+ SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, true );
return qp.getResults( visitor );
}
@@ -2059,274 +2007,4 @@ public class RelationManagerImpl implements RelationManager {
}
- private static final UUIDIndexSliceParser UUID_PARSER = new UUIDIndexSliceParser();
-
-
- /**
- * Simple search visitor that performs all the joining
- *
- * @author tnine
- */
- private class SearchCollectionVisitor extends SearchVisitor {
-
- private final CollectionInfo collection;
-
-
- /**
- * @param queryProcessor
- */
- public SearchCollectionVisitor( QueryProcessor queryProcessor ) {
- super( queryProcessor );
- this.collection = queryProcessor.getCollectionInfo();
- }
-
-
- /* (non-Javadoc)
- * @see org.apache.usergrid.persistence.query.ir.SearchVisitor#secondaryIndexScan(org.apache.usergrid.persistence.query.ir
- * .QueryNode, org.apache.usergrid.persistence.query.ir.QuerySlice)
- */
- @Override
- protected IndexScanner secondaryIndexScan( QueryNode node, QuerySlice slice ) throws Exception {
- // NOTE we explicitly do not append the slice value here. This
- // is done in the searchIndex method below
- Object indexKey = key( headEntity.getUuid(), collection.getName() );
-
- // update the cursor and order before we perform the slice
- // operation. Should be done after subkeying since this can
- // change the hash value of the slice
- queryProcessor.applyCursorAndSort( slice );
-
- IndexScanner columns = null;
-
- // nothing left to search for this range
- if ( slice.isComplete() ) {
- columns = new NoOpIndexScanner();
- }
- // perform the search
- else {
- columns = searchIndexBuckets( indexKey, slice, collection.getName(),
- queryProcessor.getPageSizeHint( node ) );
- }
-
- return columns;
- }
-
-
- public void visit( AllNode node ) throws Exception {
-
- String collectionName = collection.getName();
-
- QuerySlice slice = node.getSlice();
-
- queryProcessor.applyCursorAndSort( slice );
-
- UUID startId = null;
-
- if ( slice.hasCursor() ) {
- startId = UUID_PARSER.parse( slice.getCursor() ).getUUID();
- }
-
-
- IndexScanner indexScanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
- key( headEntity.getUuid(), DICTIONARY_COLLECTIONS, collectionName ), startId, null,
- queryProcessor.getPageSizeHint( node ), query.isReversed(), indexBucketLocator, applicationId,
- collectionName, node.isForceKeepFirst() );
-
- this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER ) );
- }
-
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.usergrid.persistence.query.ir.NodeVisitor#visit(org.apache.usergrid.
- * persistence.query.ir.WithinNode)
- */
- @Override
- public void visit( WithinNode node ) throws Exception {
-
- QuerySlice slice = node.getSlice();
-
- queryProcessor.applyCursorAndSort( slice );
-
- GeoIterator itr = new GeoIterator(
- new CollectionGeoSearch( em, indexBucketLocator, cass, headEntity, collection.getName() ),
- query.getLimit(), slice, node.getPropertyName(),
- new Point( node.getLattitude(), node.getLongitude() ), node.getDistance() );
-
- results.push( itr );
- }
-
-
- @Override
- public void visit( NameIdentifierNode nameIdentifierNode ) throws Exception {
- EntityRef ref = em.getAlias( headEntity.getUuid(), collection.getType(), nameIdentifierNode.getName() );
-
- if ( ref == null ) {
- this.results.push( new EmptyIterator() );
- return;
- }
-
- this.results.push( new StaticIdIterator( ref.getUuid() ) );
- }
- }
-
-
- /**
- * Simple search visitor that performs all the joining
- *
- * @author tnine
- */
- private class SearchConnectionVisitor extends SearchVisitor {
-
- private final ConnectionRefImpl connection;
-
- /** True if we should search from source->target edges. False if we should search from target<-source edges */
- private final boolean outgoing;
-
-
- /**
- * @param queryProcessor They query processor to use
- * @param connection The connection refernce
- * @param outgoing The direction to search. True if we should search from source->target edges. False if we
- * should search from target<-source edges
- */
- public SearchConnectionVisitor( QueryProcessor queryProcessor, ConnectionRefImpl connection,
- boolean outgoing ) {
- super( queryProcessor );
- this.connection = connection;
- this.outgoing = outgoing;
- }
-
-
- /* (non-Javadoc)
- * @see org.apache.usergrid.persistence.query.ir.SearchVisitor#secondaryIndexScan(org.apache.usergrid.persistence.query.ir
- * .QueryNode, org.apache.usergrid.persistence.query.ir.QuerySlice)
- */
- @Override
- protected IndexScanner secondaryIndexScan( QueryNode node, QuerySlice slice ) throws Exception {
-
- UUID id = ConnectionRefImpl.getIndexId( ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE, headEntity,
- connection.getConnectionType(), connection.getConnectedEntityType(), new ConnectedEntityRef[0] );
-
- Object key = key( id, INDEX_CONNECTIONS );
-
- // update the cursor and order before we perform the slice
- // operation
- queryProcessor.applyCursorAndSort( slice );
-
- IndexScanner columns = null;
-
- if ( slice.isComplete() ) {
- columns = new NoOpIndexScanner();
- }
- else {
- columns = searchIndex( key, slice, queryProcessor.getPageSizeHint( node ) );
- }
-
- return columns;
- }
-
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.usergrid.persistence.query.ir.NodeVisitor#visit(org.apache.usergrid.
- * persistence.query.ir.WithinNode)
- */
- @Override
- public void visit( WithinNode node ) throws Exception {
-
- QuerySlice slice = node.getSlice();
-
- queryProcessor.applyCursorAndSort( slice );
-
- GeoIterator itr =
- new GeoIterator( new ConnectionGeoSearch( em, indexBucketLocator, cass, connection.getIndexId() ),
- query.getLimit(), slice, node.getPropertyName(),
- new Point( node.getLattitude(), node.getLongitude() ), node.getDistance() );
-
- results.push( itr );
- }
-
-
- @Override
- public void visit( AllNode node ) throws Exception {
- QuerySlice slice = node.getSlice();
-
- queryProcessor.applyCursorAndSort( slice );
-
- int size = queryProcessor.getPageSizeHint( node );
-
- ByteBuffer start = null;
-
- if ( slice.hasCursor() ) {
- start = slice.getCursor();
- }
-
-
- boolean skipFirst = node.isForceKeepFirst() ? false : slice.hasCursor();
-
- UUID entityIdToUse;
-
- //change our type depending on which direction we're loading
- String dictionaryType;
-
- //the target type
- String targetType;
-
- //this is on the "source" side of the edge
- if ( outgoing ) {
- entityIdToUse = connection.getConnectingEntityId();
- dictionaryType = DICTIONARY_CONNECTED_ENTITIES;
- targetType = connection.getConnectedEntityType();
- }
-
- //we're on the target side of the edge
- else {
- entityIdToUse = connection.getConnectedEntityId();
- dictionaryType = DICTIONARY_CONNECTING_ENTITIES;
- targetType = connection.getConnectingEntityType();
- }
-
- final String connectionType = connection.getConnectionType();
-
-
- final ConnectionIndexSliceParser connectionParser = new ConnectionIndexSliceParser( targetType );
-
-
- final Iterator<String> connectionTypes;
-
- //use the provided connection type
- if ( connectionType != null ) {
- connectionTypes = Collections.singleton( connectionType ).iterator();
- }
-
- //we need to iterate all connection types
- else {
- connectionTypes = new ConnectionTypesIterator( cass, applicationId, entityIdToUse, outgoing, size );
- }
-
- IndexScanner connectionScanner =
- new ConnectedIndexScanner( cass, dictionaryType, applicationId, entityIdToUse, connectionTypes,
- start, slice.isReversed(), size, skipFirst );
-
- this.results.push( new SliceIterator( slice, connectionScanner, connectionParser ) );
- }
-
-
- @Override
- public void visit( NameIdentifierNode nameIdentifierNode ) throws Exception {
- //TODO T.N. USERGRID-1919 actually validate this is connected
- EntityRef ref =
- em.getAlias( applicationId, connection.getConnectedEntityType(), nameIdentifierNode.getName() );
-
- if ( ref == null ) {
- this.results.push( new EmptyIterator() );
- return;
- }
-
- this.results.push( new StaticIdIterator( ref.getUuid() ) );
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa31768c/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
index 8a9b709..668ab02 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
@@ -71,8 +71,8 @@ public class IndexBucketScanner implements IndexScanner {
public IndexBucketScanner( CassandraService cass, ApplicationCF columnFamily,
- UUID applicationId, Object keyPrefix, Object start, Object finish,
- boolean reversed, int pageSize, boolean skipFirst, String bucket) {
+ UUID applicationId, Object keyPrefix, String bucket, Object start, Object finish,
+ boolean reversed, int pageSize, boolean skipFirst) {
this.cass = cass;
this.applicationId = applicationId;
this.keyPrefix = keyPrefix;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa31768c/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
index 0038689..0512cb2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
@@ -18,10 +18,13 @@ package org.apache.usergrid.persistence.query.ir;
import java.util.Stack;
+import java.util.UUID;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.QueryProcessor;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
@@ -58,11 +61,27 @@ public abstract class SearchVisitor implements NodeVisitor {
protected final String bucket;
+ protected final EntityRef headEntity;
+ protected final CassandraService cassandraService;
+ protected final IndexBucketLocator indexBucketLocator;
+ protected final UUID applicationId;
+
/**
+ * @param cassandraService
+ * @param indexBucketLocator
+ * @param applicationId
+ * @param headEntity
* @param queryProcessor
*/
- public SearchVisitor( QueryProcessor queryProcessor, final String bucket ) {
+ public SearchVisitor( final CassandraService cassandraService, final IndexBucketLocator indexBucketLocator,
+ final UUID applicationId, final EntityRef headEntity, QueryProcessor queryProcessor, final String bucket ) {
+
+
+ this.cassandraService = cassandraService;
+ this.indexBucketLocator = indexBucketLocator;
+ this.applicationId = applicationId;
+ this.headEntity = headEntity;
this.query = queryProcessor.getQuery();
this.queryProcessor = queryProcessor;
this.em = queryProcessor.getEntityManager();
@@ -70,6 +89,9 @@ public abstract class SearchVisitor implements NodeVisitor {
}
+
+
+
/** Return the results if they exist, null otherwise */
public ResultIterator getResults() {
return results.isEmpty() ? null : results.pop();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa31768c/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
index 1fd1604..1637f2d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionSearchVisitorFactory.java
@@ -17,19 +17,62 @@
package org.apache.usergrid.persistence.query.ir.result;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.QueryProcessor;
import org.apache.usergrid.persistence.query.ir.SearchVisitor;
+/**
+ * Creates collection visitors per shard
+ */
public class CollectionSearchVisitorFactory implements SearchVisitorFactory {
+ private final CassandraService cassandraService;
+ private final IndexBucketLocator indexBucketLocator;
+ private final QueryProcessor queryProcessor;
+ private final UUID applicationId;
+ private final EntityRef headEntity;
+ private final String collectionName;
+
+
+ public CollectionSearchVisitorFactory( final CassandraService cassandraService,
+ final IndexBucketLocator indexBucketLocator,
+ final QueryProcessor queryProcessor, final UUID applicationId,
+ final EntityRef headEntity, final String collectionName ) {
+ this.cassandraService = cassandraService;
+ this.indexBucketLocator = indexBucketLocator;
+ this.queryProcessor = queryProcessor;
+ this.applicationId = applicationId;
+ this.headEntity = headEntity;
+ this.collectionName = collectionName;
+ }
+
@Override
public Collection<SearchVisitor> createVisitors() {
-// SearchConnectionVisitor visitor = new SearchConnectionVisitor( qp, connectionRef, true );
- return null;
+ final List<String> buckets =
+ indexBucketLocator.getBuckets( applicationId, IndexBucketLocator.IndexType.CONNECTION, collectionName );
+
+
+ final List<SearchVisitor> visitors = new ArrayList<SearchVisitor>( buckets.size() );
+
+ for ( final String bucket : buckets ) {
+
+ final SearchVisitor searchVisitor =
+ new SearchCollectionVisitor( cassandraService, indexBucketLocator, queryProcessor, applicationId,
+ headEntity, bucket );
+ visitors.add( searchVisitor );
+ }
+
+
+ return visitors;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa31768c/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
index 371f79b..c1ec724 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
@@ -17,40 +17,65 @@
package org.apache.usergrid.persistence.query.ir.result;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
-import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
import org.apache.usergrid.persistence.cassandra.QueryProcessor;
import org.apache.usergrid.persistence.query.ir.SearchVisitor;
public class ConnectionSearchVisitorFactory implements SearchVisitorFactory {
- private final
+ private final CassandraService cassandraService;
private final IndexBucketLocator indexBucketLocator;
private final QueryProcessor queryProcessor;
- private final ConnectionRef connectionRef;
+ private final UUID applicationId;
+ private final EntityRef headEntity;
+ private final ConnectionRefImpl connectionRef;
private final boolean outgoing;
+ private final String[] prefix;
- private ConnectionSearchVisitorFactory( final IndexBucketLocator indexBucketLocator,
- final QueryProcessor queryProcessor, final ConnectionRef connectionRef,
- final boolean outgoing ){
-// SearchConnectionVisitor visitor = new SearchConnectionVisitor( indexBucketLocator, qp, connectionRef, true );
+ private ConnectionSearchVisitorFactory( final CassandraService cassandraService,
+ final IndexBucketLocator indexBucketLocator,
+ final QueryProcessor queryProcessor, final UUID applicationId,
+ final EntityRef headEntity, ConnectionRefImpl connectionRef,
+ boolean outgoing, final String... prefix ) {
+ this.applicationId = applicationId;
+ this.cassandraService = cassandraService;
this.indexBucketLocator = indexBucketLocator;
this.queryProcessor = queryProcessor;
+ this.headEntity = headEntity;
this.connectionRef = connectionRef;
this.outgoing = outgoing;
+ this.prefix = prefix;
}
@Override
public Collection<SearchVisitor> createVisitors() {
- indexBucketLocator.getBuckets( )
+ final List<String> buckets =
+ indexBucketLocator.getBuckets( applicationId, IndexBucketLocator.IndexType.CONNECTION, prefix );
- return null;
+ final List<SearchVisitor> visitors = new ArrayList<SearchVisitor>( buckets.size() );
+
+ for ( final String bucket : buckets ) {
+
+ final SearchVisitor searchVisitor =
+ new SearchConnectionVisitor( cassandraService, indexBucketLocator, queryProcessor, applicationId,
+ headEntity, connectionRef, outgoing, bucket );
+ visitors.add( searchVisitor );
+ }
+
+
+ return visitors;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa31768c/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
new file mode 100644
index 0000000..61315e4
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -0,0 +1,168 @@
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.cassandra.QueryProcessor;
+import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
+import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
+import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
+import org.apache.usergrid.persistence.geo.CollectionGeoSearch;
+import org.apache.usergrid.persistence.geo.model.Point;
+import org.apache.usergrid.persistence.query.ir.AllNode;
+import org.apache.usergrid.persistence.query.ir.NameIdentifierNode;
+import org.apache.usergrid.persistence.query.ir.QueryNode;
+import org.apache.usergrid.persistence.query.ir.QuerySlice;
+import org.apache.usergrid.persistence.query.ir.SearchVisitor;
+import org.apache.usergrid.persistence.query.ir.WithinNode;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+
+import me.prettyprint.hector.api.beans.DynamicComposite;
+
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_COLLECTIONS;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+
+
+/**
+ * Simple search visitor that performs all the joining
+ *
+ * @author tnine
+ */
+public class SearchCollectionVisitor extends SearchVisitor {
+
+
+ private static final UUIDIndexSliceParser UUID_PARSER = new UUIDIndexSliceParser();
+
+
+ private final CollectionInfo collection;
+
+
+ /**
+ * @param queryProcessor
+ */
+ public SearchCollectionVisitor( final CassandraService cassandraService,
+ final IndexBucketLocator indexBucketLocator, final QueryProcessor queryProcessor,
+ final UUID applicationId, final EntityRef headEntity, final String bucket ) {
+ super( cassandraService, indexBucketLocator, applicationId, headEntity, queryProcessor, bucket );
+ this.collection = queryProcessor.getCollectionInfo();
+ }
+
+
+ /* (non-Javadoc)
+ * @see org.apache.usergrid.persistence.query.ir.SearchVisitor#secondaryIndexScan(org.apache.usergrid.persistence
+ * .query.ir
+ * .QueryNode, org.apache.usergrid.persistence.query.ir.QuerySlice)
+ */
+ @Override
+ protected IndexScanner secondaryIndexScan( QueryNode node, QuerySlice slice ) throws Exception {
+ // NOTE we explicitly do not append the slice value here. This
+ // is done in the searchIndex method below
+ Object indexKey = key( headEntity.getUuid(), collection.getName() );
+
+ // update the cursor and order before we perform the slice
+ // operation. Should be done after subkeying since this can
+ // change the hash value of the slice
+ queryProcessor.applyCursorAndSort( slice );
+
+ IndexScanner columns = null;
+
+ // nothing left to search for this range
+ if ( slice.isComplete() ) {
+ columns = new NoOpIndexScanner();
+ }
+ // perform the search
+ else {
+ columns =
+ searchIndexBuckets( indexKey, slice, collection.getName(), queryProcessor.getPageSizeHint( node ) );
+ }
+
+ return columns;
+ }
+
+
+ public void visit( AllNode node ) throws Exception {
+
+ String collectionName = collection.getName();
+
+ QuerySlice slice = node.getSlice();
+
+ queryProcessor.applyCursorAndSort( slice );
+
+ UUID startId = null;
+
+ if ( slice.hasCursor() ) {
+ startId = UUID_PARSER.parse( slice.getCursor() ).getUUID();
+ }
+
+
+ IndexScanner indexScanner = cassandraService
+ .getIdList( cassandraService.getApplicationKeyspace( applicationId ),
+ key( headEntity.getUuid(), DICTIONARY_COLLECTIONS, collectionName ), startId, null,
+ queryProcessor.getPageSizeHint( node ), query.isReversed(), indexBucketLocator, applicationId,
+ collectionName, node.isForceKeepFirst() );
+
+ this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER ) );
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.usergrid.persistence.query.ir.NodeVisitor#visit(org.apache.usergrid.
+ * persistence.query.ir.WithinNode)
+ */
+ @Override
+ public void visit( WithinNode node ) throws Exception {
+
+ QuerySlice slice = node.getSlice();
+
+ queryProcessor.applyCursorAndSort( slice );
+
+ GeoIterator itr = new GeoIterator(
+ new CollectionGeoSearch( em, indexBucketLocator, cassandraService, headEntity, collection.getName() ),
+ query.getLimit(), slice, node.getPropertyName(), new Point( node.getLattitude(), node.getLongitude() ),
+ node.getDistance() );
+
+ results.push( itr );
+ }
+
+
+ @Override
+ public void visit( NameIdentifierNode nameIdentifierNode ) throws Exception {
+ EntityRef ref = em.getAlias( headEntity.getUuid(), collection.getType(), nameIdentifierNode.getName() );
+
+ if ( ref == null ) {
+ this.results.push( new EmptyIterator() );
+ return;
+ }
+
+ this.results.push( new StaticIdIterator( ref.getUuid() ) );
+ }
+
+
+ /**
+ * Search the collection index using all the buckets for the given collection
+ *
+ * @param indexKey The index key to read
+ * @param slice Slice set in the query
+ * @param collectionName The name of the collection to search
+ * @param pageSize The page size to load when iterating
+ */
+ private IndexScanner searchIndexBuckets( Object indexKey, QuerySlice slice, String collectionName, int pageSize )
+ throws Exception {
+
+ DynamicComposite[] range = slice.getRange();
+
+ Object keyPrefix = key( indexKey, slice.getPropertyName() );
+
+ IndexScanner scanner =
+ new IndexBucketScanner( cassandraService, ENTITY_INDEX, applicationId, keyPrefix, bucket, range[0],
+ range[1], slice.isReversed(), pageSize, slice.hasCursor() );
+
+ return scanner;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa31768c/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
new file mode 100644
index 0000000..da99cdf
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -0,0 +1,213 @@
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.ConnectedEntityRef;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+import org.apache.usergrid.persistence.cassandra.QueryProcessor;
+import org.apache.usergrid.persistence.cassandra.index.ConnectedIndexScanner;
+import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
+import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
+import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
+import org.apache.usergrid.persistence.geo.ConnectionGeoSearch;
+import org.apache.usergrid.persistence.geo.model.Point;
+import org.apache.usergrid.persistence.query.ir.AllNode;
+import org.apache.usergrid.persistence.query.ir.NameIdentifierNode;
+import org.apache.usergrid.persistence.query.ir.QueryNode;
+import org.apache.usergrid.persistence.query.ir.QuerySlice;
+import org.apache.usergrid.persistence.query.ir.SearchVisitor;
+import org.apache.usergrid.persistence.query.ir.WithinNode;
+
+import me.prettyprint.hector.api.beans.DynamicComposite;
+
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_ENTITIES;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_ENTITIES;
+import static org.apache.usergrid.persistence.Schema.INDEX_CONNECTIONS;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+
+
+/**
+ * Simple search visitor that performs all the joining
+ *
+ * @author tnine
+ */
+public class SearchConnectionVisitor extends SearchVisitor {
+
+ private final ConnectionRefImpl connection;
+
+ /** True if we should search from source->target edges. False if we should search from target<-source edges */
+ private final boolean outgoing;
+
+
+
+ /**
+ * @param queryProcessor They query processor to use
+ * @param applicationId
+ * @param connection The connection refernce
+ * @param outgoing The direction to search. True if we should search from source->target edges. False if we
+ */
+ public SearchConnectionVisitor( final CassandraService cassandraService,
+ final IndexBucketLocator indexBucketLocator, final QueryProcessor queryProcessor,
+ final UUID applicationId, final EntityRef headEntity, ConnectionRefImpl connection,
+ boolean outgoing, final String bucket ) {
+ super( cassandraService, indexBucketLocator, applicationId, headEntity, queryProcessor, bucket );
+ this.connection = connection;
+ this.outgoing = outgoing;
+ }
+
+
+ /* (non-Javadoc)
+ * @see org.apache.usergrid.persistence.query.ir.SearchVisitor#secondaryIndexScan(org.apache.usergrid.persistence
+ * .query.ir
+ * .QueryNode, org.apache.usergrid.persistence.query.ir.QuerySlice)
+ */
+ @Override
+ protected IndexScanner secondaryIndexScan( QueryNode node, QuerySlice slice ) throws Exception {
+
+ UUID
+ id = ConnectionRefImpl.getIndexId( ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE, headEntity,
+ connection.getConnectionType(), connection.getConnectedEntityType(), new ConnectedEntityRef[0] );
+
+ Object key = key( id, INDEX_CONNECTIONS );
+
+ // update the cursor and order before we perform the slice
+ // operation
+ queryProcessor.applyCursorAndSort( slice );
+
+ IndexScanner columns = null;
+
+ if ( slice.isComplete() ) {
+ columns = new NoOpIndexScanner();
+ }
+ else {
+ columns = searchIndex( key, slice, queryProcessor.getPageSizeHint( node ), bucket );
+ }
+
+ return columns;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.usergrid.persistence.query.ir.NodeVisitor#visit(org.apache.usergrid.
+ * persistence.query.ir.WithinNode)
+ */
+ @Override
+ public void visit( WithinNode node ) throws Exception {
+
+ QuerySlice slice = node.getSlice();
+
+ queryProcessor.applyCursorAndSort( slice );
+
+ GeoIterator itr =
+ new GeoIterator( new ConnectionGeoSearch( em, indexBucketLocator, cassandraService, connection.getIndexId() ),
+ query.getLimit(), slice, node.getPropertyName(),
+ new Point( node.getLattitude(), node.getLongitude() ), node.getDistance() );
+
+ results.push( itr );
+ }
+
+
+ @Override
+ public void visit( AllNode node ) throws Exception {
+ QuerySlice slice = node.getSlice();
+
+ queryProcessor.applyCursorAndSort( slice );
+
+ int size = queryProcessor.getPageSizeHint( node );
+
+ ByteBuffer start = null;
+
+ if ( slice.hasCursor() ) {
+ start = slice.getCursor();
+ }
+
+
+ boolean skipFirst = node.isForceKeepFirst() ? false : slice.hasCursor();
+
+ UUID entityIdToUse;
+
+ //change our type depending on which direction we're loading
+ String dictionaryType;
+
+ //the target type
+ String targetType;
+
+ //this is on the "source" side of the edge
+ if ( outgoing ) {
+ entityIdToUse = connection.getConnectingEntityId();
+ dictionaryType = DICTIONARY_CONNECTED_ENTITIES;
+ targetType = connection.getConnectedEntityType();
+ }
+
+ //we're on the target side of the edge
+ else {
+ entityIdToUse = connection.getConnectedEntityId();
+ dictionaryType = DICTIONARY_CONNECTING_ENTITIES;
+ targetType = connection.getConnectingEntityType();
+ }
+
+ final String connectionType = connection.getConnectionType();
+
+
+ final ConnectionIndexSliceParser connectionParser = new ConnectionIndexSliceParser( targetType );
+
+
+ final Iterator<String> connectionTypes;
+
+ //use the provided connection type
+ if ( connectionType != null ) {
+ connectionTypes = Collections.singleton( connectionType ).iterator();
+ }
+
+ //we need to iterate all connection types
+ else {
+ connectionTypes = new ConnectionTypesIterator( cassandraService, applicationId, entityIdToUse, outgoing, size );
+ }
+
+ IndexScanner connectionScanner =
+ new ConnectedIndexScanner( cassandraService, dictionaryType, applicationId, entityIdToUse, connectionTypes,
+ start, slice.isReversed(), size, skipFirst );
+
+ this.results.push( new SliceIterator( slice, connectionScanner, connectionParser ) );
+ }
+
+
+ @Override
+ public void visit( NameIdentifierNode nameIdentifierNode ) throws Exception {
+ //TODO T.N. USERGRID-1919 actually validate this is connected
+ EntityRef ref =
+ em.getAlias( applicationId, connection.getConnectedEntityType(), nameIdentifierNode.getName() );
+
+ if ( ref == null ) {
+ this.results.push( new EmptyIterator() );
+ return;
+ }
+
+ this.results.push( new StaticIdIterator( ref.getUuid() ) );
+ }
+
+ private IndexScanner searchIndex( Object indexKey, QuerySlice slice, int pageSize, final String shardBucket ) throws Exception {
+
+ DynamicComposite[] range = slice.getRange();
+
+ Object keyPrefix = key( indexKey, slice.getPropertyName() );
+
+
+
+ IndexScanner scanner =
+ new IndexBucketScanner( cassandraService, ENTITY_INDEX, applicationId,
+ keyPrefix, shardBucket, range[0], range[1], slice.isReversed(), pageSize, slice.hasCursor());
+
+ return scanner;
+ }
+}
[08/25] incubator-usergrid git commit: Finished refactor of shard
slice iterator. Need to refactor cursor generation to be encapsulated within
the ScanColumn.
Posted by sn...@apache.org.
Finished refactor of shard slice iterator. Need to refactor cursor generation to be encapsulated within the ScanColumn.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f087924d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f087924d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f087924d
Branch: refs/heads/master
Commit: f087924dd0533de5ce320f815180cf8ae25a8415
Parents: c115744
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Jun 25 17:43:16 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Jun 25 17:43:16 2015 -0600
----------------------------------------------------------------------
.../query/ir/result/GeoIterator.java | 17 +-
.../ir/result/SearchCollectionVisitor.java | 16 +-
.../ir/result/SearchConnectionVisitor.java | 21 ++-
.../query/ir/result/SliceIterator.java | 18 +-
.../ir/result/SliceShardFilterIterator.java | 165 +++++++++++++++++++
.../query/ir/result/SliceShardIterator.java | 118 -------------
6 files changed, 211 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f087924d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
index 4ecbb5a..587bb49 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
@@ -35,6 +35,8 @@ import org.apache.usergrid.persistence.geo.GeoIndexSearcher.SearchResults;
import org.apache.usergrid.persistence.geo.model.Point;
import org.apache.usergrid.persistence.query.ir.QuerySlice;
+import com.fasterxml.uuid.UUIDComparator;
+
import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
@@ -223,7 +225,7 @@ public class GeoIterator implements ResultIterator {
LocationScanColumn col = idOrder.get( uuid );
if ( col == null ) {
- return;
+ throw new IllegalArgumentException( "Could not generate cursor for column because column could not be found" );
}
final EntityLocationRef location = col.location;
@@ -369,7 +371,18 @@ public class GeoIterator implements ResultIterator {
throw new UnsupportedOperationException( "Cannot compare another ScanColumn that is not an instance of LocationScanColumn" );
}
- return this.location.compareTo( ((LocationScanColumn)o).location );
+ //sort by location (closest first) if that's the same compare uuids
+ final int locationCompare = this.location.compareTo( ( ( LocationScanColumn ) o ).location ) ;
+
+ if(locationCompare == 0) {
+ //same distance, return compare by uuid
+ final int uuidCompare = UUIDComparator.staticCompare( getUUID(), o.getUUID() );
+
+
+ return uuidCompare;
+ }
+
+ return locationCompare;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f087924d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index c20baf4..a3fe116 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -69,7 +69,7 @@ public class SearchCollectionVisitor extends SearchVisitor {
// change the hash value of the slice
queryProcessor.applyCursorAndSort( slice );
- IndexScanner columns = null;
+ IndexScanner columns;
// nothing left to search for this range
if ( slice.isComplete() ) {
@@ -101,9 +101,8 @@ public class SearchCollectionVisitor extends SearchVisitor {
IndexScanner indexScanner = cassandraService
- .getIdList(
- key( headEntity.getUuid(), DICTIONARY_COLLECTIONS, collectionName ), startId, null,
- queryProcessor.getPageSizeHint( node ), query.isReversed(), bucket, applicationId,
+ .getIdList( key( headEntity.getUuid(), DICTIONARY_COLLECTIONS, collectionName ), startId, null,
+ queryProcessor.getPageSizeHint( node ), query.isReversed(), bucket, applicationId,
node.isForceKeepFirst() );
this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER ) );
@@ -123,12 +122,17 @@ public class SearchCollectionVisitor extends SearchVisitor {
queryProcessor.applyCursorAndSort( slice );
+ final int size = queryProcessor.getPageSizeHint( node );
+
GeoIterator itr = new GeoIterator(
new CollectionGeoSearch( em, indexBucketLocator, cassandraService, headEntity, collection.getName() ),
- query.getLimit(), slice, node.getPropertyName(), new Point( node.getLattitude(), node.getLongitude() ),
+ size, slice, node.getPropertyName(), new Point( node.getLattitude(), node.getLongitude() ),
node.getDistance() );
- results.push( itr );
+
+ final SliceShardFilterIterator.ShardBucketValidator validator = new SliceShardFilterIterator.ShardBucketValidator(indexBucketLocator, bucket, applicationId, IndexBucketLocator.IndexType.COLLECTION, collection.getName() );
+
+ this.results.push( new SliceShardFilterIterator( validator, itr, size));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f087924d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index ff08245..7e7ddf6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -17,6 +17,7 @@ import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeStartToBy
import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
+import org.apache.usergrid.persistence.geo.CollectionGeoSearch;
import org.apache.usergrid.persistence.geo.ConnectionGeoSearch;
import org.apache.usergrid.persistence.geo.model.Point;
import org.apache.usergrid.persistence.query.ir.AllNode;
@@ -109,12 +110,20 @@ public class SearchConnectionVisitor extends SearchVisitor {
queryProcessor.applyCursorAndSort( slice );
+ final int size = queryProcessor.getPageSizeHint( node );
+
+
+
GeoIterator itr =
new GeoIterator( new ConnectionGeoSearch( em, indexBucketLocator, cassandraService, connection.getIndexId() ),
- query.getLimit(), slice, node.getPropertyName(),
+ size, slice, node.getPropertyName(),
new Point( node.getLattitude(), node.getLongitude() ), node.getDistance() );
- results.push( itr );
+ final SliceShardFilterIterator.ShardBucketValidator validator = new SliceShardFilterIterator.ShardBucketValidator(indexBucketLocator, bucket, applicationId, IndexBucketLocator.IndexType.CONNECTION, connection.getSearchIndexName() );
+
+
+ this.results.push( new SliceShardFilterIterator( validator, itr, size ) );
+
}
@@ -183,9 +192,13 @@ public class SearchConnectionVisitor extends SearchVisitor {
start, slice.isReversed(), size, skipFirst );
//we have to create our wrapper so validate the data we read is correct for our shard
- final SliceShardIterator.ShardBucketValidator validator = new SliceShardIterator.ShardBucketValidator(indexBucketLocator, bucket, applicationId, IndexBucketLocator.IndexType.CONNECTION, "" );
+ final SliceShardFilterIterator.ShardBucketValidator validator = new SliceShardFilterIterator.ShardBucketValidator(indexBucketLocator, bucket, applicationId, IndexBucketLocator.IndexType.CONNECTION, "" );
+
+
+ final SliceIterator sliceIterator = new SliceIterator( slice, connectionScanner, connectionParser );
+
- this.results.push( new SliceShardIterator( validator, slice, connectionScanner, connectionParser ));
+ this.results.push( new SliceShardFilterIterator( validator, sliceIterator, size));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f087924d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
index 4f5bff5..3f9e86c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -130,7 +130,10 @@ public class SliceIterator implements ResultIterator {
final HColumn<ByteBuffer, ByteBuffer> column = results.next();
- final ScanColumn parsed = parse(column);
+ final ByteBuffer colName = column.getName().duplicate();
+
+ final ScanColumn parsed = parser.parse( colName, isReversed );
+
//skip this value, the parser has discarded it
if ( parsed == null ) {
@@ -151,19 +154,6 @@ public class SliceIterator implements ResultIterator {
}
- /**
- * Parses the column. If the column should be discarded, null should be returned
- * @param column
- * @return
- */
- protected ScanColumn parse( HColumn<ByteBuffer, ByteBuffer> column){
-
- final ByteBuffer colName = column.getName().duplicate();
-
- final ScanColumn parsed = parser.parse( colName, isReversed );
-
- return parsed;
- }
/*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f087924d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
new file mode 100644
index 0000000..fb0d295
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
@@ -0,0 +1,165 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.CursorCache;
+
+
+/**
+ * An iterator that will check if the parsed column is part of this shard. This is required due to a legacy storage
+ * format in both connection pointers, as well as geo points.
+ *
+ * Some formats are not sharded by target entity, as a result, we get data partition mismatches when performing
+ * intersections and seeks. This is meant to discard target entities that are not part of the current shard
+ *
+ * @author tnine
+ */
+public class SliceShardFilterIterator implements ResultIterator {
+
+ private static final Logger logger = LoggerFactory.getLogger( SliceShardFilterIterator.class );
+
+ private final ShardBucketValidator shardBucketValidator;
+ private final ResultIterator resultsIterator;
+ private final int pageSize;
+
+ private Set<ScanColumn> current;
+
+
+ /**
+ * @param shardBucketValidator The validator to use when validating results belong to a shard
+ * @param resultsIterator The iterator to filter results from
+ * @param pageSize
+ */
+ public SliceShardFilterIterator( final ShardBucketValidator shardBucketValidator,
+ final ResultIterator resultsIterator, final int pageSize ) {
+ this.shardBucketValidator = shardBucketValidator;
+ this.resultsIterator = resultsIterator;
+ this.pageSize = pageSize;
+ }
+
+
+
+ @Override
+ public void reset() {
+ current = null;
+ resultsIterator.reset();
+ }
+
+
+ @Override
+ public void finalizeCursor( final CursorCache cache, final UUID lastValue ) {
+ resultsIterator.finalizeCursor( cache, lastValue );
+ }
+
+
+ @Override
+ public Iterator<Set<ScanColumn>> iterator() {
+ return this;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+ if(current == null){
+ advance();
+ }
+
+ return current != null && current.size() > 0;
+ }
+
+
+ @Override
+ public Set<ScanColumn> next() {
+
+ final Set<ScanColumn> toReturn = current;
+
+ current = null;
+
+ return toReturn;
+ }
+
+
+ /**
+ * Advance the column pointers
+ */
+ private void advance(){
+
+ final Set<ScanColumn> results = new LinkedHashSet<ScanColumn>( );
+
+ while(resultsIterator.hasNext()){
+
+ final Iterator<ScanColumn> scanColumns = resultsIterator.next().iterator();
+
+
+ while(results.size() < pageSize && scanColumns.hasNext()){
+ final ScanColumn scanColumn = scanColumns.next();
+
+ if(shardBucketValidator.isInShard( scanColumn.getUUID() )){
+ results.add( scanColumn );
+ }
+ }
+ }
+
+ current = results;
+
+
+ }
+
+
+
+ /**
+ * Class that performs validation on an entity to ensure it's in the shard we expecte
+ */
+ public static final class ShardBucketValidator {
+ private final IndexBucketLocator indexBucketLocator;
+ private final String expectedBucket;
+ private final UUID applicationId;
+ private final IndexBucketLocator.IndexType type;
+ private final String[] components;
+
+
+ public ShardBucketValidator( final IndexBucketLocator indexBucketLocator, final String expectedBucket,
+ final UUID applicationId, final IndexBucketLocator.IndexType type,
+ final String... components ) {
+ this.indexBucketLocator = indexBucketLocator;
+ this.expectedBucket = expectedBucket;
+ this.applicationId = applicationId;
+ this.type = type;
+ this.components = components;
+ }
+
+
+ public boolean isInShard( final UUID entityId ) {
+ //not for our current processing shard, discard
+ final String shard = indexBucketLocator.getBucket( applicationId, type, entityId, components );
+
+ return expectedBucket.equals( shard );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f087924d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardIterator.java
deleted file mode 100644
index 279ba2c..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardIterator.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.usergrid.persistence.query.ir.result;
-
-
-import java.nio.ByteBuffer;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
-import org.apache.usergrid.persistence.query.ir.QuerySlice;
-
-import me.prettyprint.hector.api.beans.HColumn;
-
-
-/**
- * An iterator that will check if the parsed column is part of this shard. This is required due to a legacy storage
- * format
- *
- * Connections are not sharded by target entity, as a result, we get data partition mismatches when performing
- * intersections and seeks. This is meant to discard target entities that are not part of the current shard
- *
- * @author tnine
- */
-public class SliceShardIterator extends SliceIterator {
-
- private static final Logger logger = LoggerFactory.getLogger( SliceShardIterator.class );
-
- private final ShardBucketValidator shardBucketValidator;
-
-
- /**
- * @param slice The slice used in the scanner
- * @param scanner The scanner to use to read the cols
- * @param parser The parser for the scanner results
- */
- public SliceShardIterator( final ShardBucketValidator shardBucketValidator, final QuerySlice slice,
- final IndexScanner scanner, final SliceParser parser ) {
- super( slice, scanner, parser );
-
- this.shardBucketValidator = shardBucketValidator;
- }
-
-
- /**
- * Parses the column. If the column should be discarded, null should be returned
- */
- protected ScanColumn parse( HColumn<ByteBuffer, ByteBuffer> column ) {
-
- final ByteBuffer colName = column.getName().duplicate();
-
- final ScanColumn parsed = parser.parse( colName, isReversed );
-
- if(parsed == null){
- return null;
- }
-
- final UUID entityId = parsed.getUUID();
-
-
- //not for our current processing shard, discard
- if(!shardBucketValidator.isInShard( entityId )){
- return null;
- }
-
- return parsed;
- }
-
-
- /**
- * Class that performs validation on an entity to ensure it's in the shard we expecte
- */
- public static final class ShardBucketValidator {
- private final IndexBucketLocator indexBucketLocator;
- private final String expectedBucket;
- private final UUID applicationId;
- private final IndexBucketLocator.IndexType type;
- private final String[] components;
-
-
- public ShardBucketValidator( final IndexBucketLocator indexBucketLocator, final String expectedBucket,
- final UUID applicationId, final IndexBucketLocator.IndexType type,
- final String... components ) {
- this.indexBucketLocator = indexBucketLocator;
- this.expectedBucket = expectedBucket;
- this.applicationId = applicationId;
- this.type = type;
- this.components = components;
- }
-
-
- public boolean isInShard( final UUID entityId ) {
- //not for our current processing shard, discard
- final String shard = indexBucketLocator.getBucket( applicationId, type, entityId, components );
-
- return expectedBucket.equals( shard );
- }
- }
-}
[06/25] incubator-usergrid git commit: Fixes resumption and off by 1
bugs
Posted by sn...@apache.org.
Fixes resumption and off by 1 bugs
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c1157449
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c1157449
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c1157449
Branch: refs/heads/master
Commit: c1157449c9c2a32662423c0e14a1f73ef64042b4
Parents: aa79488
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Jun 25 10:16:07 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Jun 25 11:14:52 2015 -0600
----------------------------------------------------------------------
.../cassandra/index/CassandraColumnUtils.java | 72 ++++++++++++++++++
.../cassandra/index/ConnectedIndexScanner.java | 31 ++++----
.../cassandra/index/IndexBucketScanner.java | 77 ++++++--------------
.../ir/result/ConnectionIndexSliceParser.java | 10 ++-
4 files changed, 120 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c1157449/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/CassandraColumnUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/CassandraColumnUtils.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/CassandraColumnUtils.java
new file mode 100644
index 0000000..8cc1ccd
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/CassandraColumnUtils.java
@@ -0,0 +1,72 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.cassandra.index;
+
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.cassandra.utils.FBUtilities;
+
+import me.prettyprint.hector.api.beans.HColumn;
+
+
+/**
+ * Utils for dealing with Pagination in cassandra
+ */
+public class CassandraColumnUtils {
+
+ /**
+ * Returns true if the 2 byte buffers contain the same bytes, false otherwise
+ */
+ public static boolean equals( final ByteBuffer first, final ByteBuffer second ) {
+ int firstLength = first.remaining();
+ int firstPosition = first.position();
+
+ int secondLength = second.remaining();
+ int secondPosition = second.position();
+
+ final int compare = FBUtilities
+ .compareUnsigned( first.array(), second.array(), firstPosition, secondPosition, firstLength,
+ secondLength );
+
+ return compare == 0;
+ }
+
+
+ /**
+ * Maybe remove the first if the byte buffers match
+ * @param columns
+ * @param startScan
+ */
+ public static void maybeRemoveFirst(final List<HColumn<ByteBuffer, ByteBuffer>> columns, final ByteBuffer startScan){
+ //remove the first element since it needs to be skipped AFTER the size check. Otherwise it will fail
+ //we only want to skip if our byte value are the same as our expected start. Since we aren't stateful you can't
+ //be sure your start even comes back, and you don't want to erroneously remove columns
+ if ( columns != null && columns.size() > 0 && startScan != null) {
+ final ByteBuffer returnedBuffer = columns.get( 0 ).getName();
+
+ //the byte buffers are the same as our seek (which may or may not be the case in the first seek)
+ if( CassandraColumnUtils.equals( startScan, returnedBuffer ) ){
+ columns.remove( 0 );
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c1157449/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
index 4b1a049..aa3ddfb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
@@ -18,14 +18,12 @@ package org.apache.usergrid.persistence.cassandra.index;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Iterator;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Set;
import java.util.UUID;
import org.springframework.util.Assert;
+
import org.apache.usergrid.persistence.cassandra.CassandraService;
import com.yammer.metrics.annotation.Metered;
@@ -64,7 +62,7 @@ public class ConnectedIndexScanner implements IndexScanner {
/**
* Iterator for our results from the last page load
*/
- private List<HColumn<ByteBuffer, ByteBuffer>> lastResults;
+ private List<HColumn<ByteBuffer, ByteBuffer>> results;
/**
* True if our last load loaded a full page size.
@@ -122,12 +120,12 @@ public class ConnectedIndexScanner implements IndexScanner {
return false;
}
- boolean skipFirst = this.skipFirst && start == scanStart;
+ boolean shouldDropFirst = skipFirst && start == scanStart;
int totalSelectSize = pageSize + 1;
//we're discarding the first, so increase our total size by 1 since this value will be inclusive in the seek
- if ( skipFirst ) {
+ if ( shouldDropFirst ) {
totalSelectSize++;
}
@@ -135,7 +133,7 @@ public class ConnectedIndexScanner implements IndexScanner {
//go through each connection type until we exhaust the result sets
while ( currentConnectionType != null ) {
- final int lastResultsSize = lastResults == null? 0: lastResults.size();
+ final int lastResultsSize = results == null ? 0 : results.size();
//only load a delta size to get this next page
final int selectSize = totalSelectSize - lastResultsSize;
@@ -150,7 +148,7 @@ public class ConnectedIndexScanner implements IndexScanner {
final int resultSize = results.size();
- lastResults = results;
+ this.results = results;
// we loaded a full page, there might be more
@@ -177,16 +175,17 @@ public class ConnectedIndexScanner implements IndexScanner {
}
//remove the first element, we need to skip it
- if ( skipFirst && lastResults != null && lastResults.size() > 0) {
- lastResults.remove( 0 );
+ if ( shouldDropFirst ) {
+
+ results.remove( 0 );
}
- if ( hasMore && lastResults !=null && lastResults.size() > 0 ) {
+ if ( hasMore && results != null && results.size() > 0 ) {
// set the bytebuffer for the next pass
- start = lastResults.remove( lastResults.size() - 1 ).getName();
+ start = results.remove( results.size() - 1 ).getName();
}
- return lastResults != null && lastResults.size() > 0;
+ return results != null && results.size() > 0;
}
@@ -213,7 +212,7 @@ public class ConnectedIndexScanner implements IndexScanner {
// "next page" pointer
// Our currently buffered results don't exist or don't have a next. Try to
// load them again if they're less than the page size
- if ( lastResults == null && hasMore ) {
+ if ( results == null && hasMore ) {
try {
return load();
}
@@ -234,9 +233,9 @@ public class ConnectedIndexScanner implements IndexScanner {
@Override
@Metered( group = "core", name = "IndexBucketScanner_load" )
public List<HColumn<ByteBuffer, ByteBuffer>> next() {
- List<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
+ List<HColumn<ByteBuffer, ByteBuffer>> returnVal = results;
- lastResults = null;
+ results = null;
return returnVal;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c1157449/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
index bf61846..0efa735 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
@@ -22,12 +22,9 @@ import java.util.Iterator;
import java.util.List;
import java.util.UUID;
-import org.apache.cassandra.utils.FBUtilities;
-
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
import org.apache.usergrid.persistence.cassandra.CassandraService;
-import com.google.common.primitives.UnsignedBytes;
import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.beans.HColumn;
@@ -47,18 +44,19 @@ public class IndexBucketScanner<T> implements IndexScanner {
private final UUID applicationId;
private final Object keyPrefix;
private final ApplicationCF columnFamily;
- private final Object finish;
+ private final ByteBuffer finish;
private final boolean reversed;
private final int pageSize;
private final boolean skipFirst;
private final String bucket;
private final StartToBytes<T> scanStartSerializer;
+ private final T initialStartValue;
/** Pointer to our next start read */
private ByteBuffer start;
- /** Set to the original value to start scanning from */
- private T initialStartValue;
+ private boolean resumedFromCursor = false;
+
/** Iterator for our results from the last page load */
private List<HColumn<ByteBuffer, ByteBuffer>> lastResults;
@@ -68,6 +66,7 @@ public class IndexBucketScanner<T> implements IndexScanner {
+
public IndexBucketScanner( CassandraService cass, ApplicationCF columnFamily, StartToBytes<T> scanStartSerializer,
UUID applicationId, Object keyPrefix, String bucket, T start, T finish,
boolean reversed, int pageSize, boolean skipFirst) {
@@ -76,17 +75,18 @@ public class IndexBucketScanner<T> implements IndexScanner {
this.applicationId = applicationId;
this.keyPrefix = keyPrefix;
this.columnFamily = columnFamily;
- this.finish = finish;
this.reversed = reversed;
this.skipFirst = skipFirst;
this.bucket = bucket;
- //we always add 1 to the page size. This is because we pop the last column for the next page of results
- this.pageSize = pageSize+1;
+ this.pageSize = pageSize;
//the initial value set when we started scanning
+
+ this.finish = scanStartSerializer.toBytes( finish );
this.initialStartValue = start;
- this.start = scanStartSerializer.toBytes( initialStartValue );
+
+ reset();
}
@@ -97,6 +97,7 @@ public class IndexBucketScanner<T> implements IndexScanner {
public void reset() {
hasMore = true;
start = scanStartSerializer.toBytes( initialStartValue );
+ resumedFromCursor = start != null && skipFirst;
}
@@ -118,81 +119,51 @@ public class IndexBucketScanner<T> implements IndexScanner {
//if we skip the first we need to set the load to page size +2, since we'll discard the first
//and start paging at the next entity, otherwise we'll just load the page size we need
- int selectSize = pageSize;
+ int selectSize = pageSize+1;
//we purposefully use instance equality. If it's a pointer to the same value, we need to increase by 1
//since we'll be skipping the first value
- final boolean shouldCheckFirst =
- //we should skip the value it's a cursor resume OR it's a previous page from a stateful iterator
- (this.skipFirst && initialStartValue != null) || start != null;
+ if(resumedFromCursor){
+ selectSize++;
+ }
final List<HColumn<ByteBuffer, ByteBuffer>>
- resultsTree = cass.getColumns( cass.getApplicationKeyspace( applicationId ), columnFamily, rowKey,
+ results = cass.getColumns( cass.getApplicationKeyspace( applicationId ), columnFamily, rowKey,
start, finish, selectSize, reversed );
//remove the first element, it's from a cursor value and we don't want to retain it
// we loaded a full page, there might be more
- if ( resultsTree.size() == selectSize ) {
+ if ( results.size() == selectSize ) {
hasMore = true;
// set the bytebuffer for the next pass
- start = resultsTree.get( resultsTree.size() - 1 ).getName();
+ start = results.remove( results.size() - 1 ).getName();
}
else {
hasMore = false;
}
+
+
//remove the first element since it needs to be skipped AFTER the size check. Otherwise it will fail
//we only want to skip if our byte value are the same as our expected start. Since we aren't stateful you can't
//be sure your start even comes back, and you don't want to erroneously remove columns
- if ( shouldCheckFirst && resultsTree.size() > 0 && start != null) {
- final int startIndex = start.position();
- final int startLength = start.remaining();
-
-
-
- final ByteBuffer returnedBuffer = resultsTree.get( 0 ).getName();
- final int returnedIndex = returnedBuffer.position();
- final int returnedLength = returnedBuffer.remaining();
-
-
- final int compare = FBUtilities.compareUnsigned( start.array(), returnedBuffer.array(), startIndex, returnedIndex, startLength, returnedLength ) ;
-
- //the byte buffers are the same as our seek (which may or may not be the case in the first seek)
- if(compare == 0){
- resultsTree.remove( 0 );
- }
+ if ( resumedFromCursor ) {
+ CassandraColumnUtils.maybeRemoveFirst( results, scanStartSerializer.toBytes( initialStartValue ) );
+ resumedFromCursor = false;
}
- lastResults = resultsTree;
+ lastResults = results;
return lastResults != null && lastResults.size() > 0;
}
- /**
- * Returns true if the 2 byte buffers contain the same bytes, false otherwise
- * @param first
- * @param second
- * @return
- */
- private boolean compareBuffer(final ByteBuffer first, final ByteBuffer second){
- int firstLength = first.remaining();
- int firstPosition = first.position();
- int secondLength = second.remaining();
- int secondPosition = second.position();
-
- final int compare = FBUtilities.compareUnsigned( first.array(), second.array(), firstPosition, secondPosition, firstLength, secondLength);
-
- return compare == 0;
-
-
- }
/*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c1157449/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
index 2bb83b4..a669d8c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
@@ -23,6 +23,8 @@ import java.util.UUID;
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
+import com.fasterxml.uuid.UUIDComparator;
+
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -94,7 +96,13 @@ public class ConnectionIndexSliceParser implements SliceParser {
return 1;
}
- return connectedType.compareTo( ((ConnectionColumn)o).connectedType );
+ final int compare = UUIDComparator.staticCompare( uuid, o.getUUID() );
+
+ if(compare == 0){
+ return connectedType.compareTo( ((ConnectionColumn)o).connectedType );
+ }
+
+ return compare;
}
}
}
[20/25] incubator-usergrid git commit: Refactor of gather iterator to
clean up coordination
Posted by sn...@apache.org.
Refactor of gather iterator to clean up coordination
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d872e030
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d872e030
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d872e030
Branch: refs/heads/master
Commit: d872e0305e85739e1f531793560c298ed6739919
Parents: bbd5637
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jul 8 15:19:13 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jul 8 15:19:13 2015 -0600
----------------------------------------------------------------------
.../query/ir/result/GatherIterator.java | 125 +++++++++++++------
1 file changed, 86 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d872e030/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index 36104f9..c05e3cc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashSet;
-import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.TreeSet;
@@ -39,27 +38,15 @@ import org.apache.usergrid.persistence.query.ir.SearchVisitor;
public class GatherIterator implements ResultIterator {
- private List<Future<Void>> iterators;
- private final ConcurrentResultMerge merge;
- private boolean merged;
+ private final WorkerCoordinator workerCoordinator;
public GatherIterator( final int pageSize, final QueryNode rootNode, final Collection<SearchVisitor> searchVisitors,
final ExecutorService executorService ) {
- this.merge = new ConcurrentResultMerge( pageSize );
+ this.workerCoordinator = new WorkerCoordinator( executorService, searchVisitors, rootNode, pageSize );
- this.iterators = new ArrayList<Future<Void>>( searchVisitors.size() );
- this.merged = false;
-
-
- /**
- * Start our search processing
- */
- for ( SearchVisitor visitor : searchVisitors ) {
- final Future<Void> result = executorService.submit( new VisitorExecutor( rootNode, merge, visitor ) );
- iterators.add( result );
- }
+ this.workerCoordinator.start();
}
@@ -77,37 +64,86 @@ public class GatherIterator implements ResultIterator {
@Override
public boolean hasNext() {
- waitForCompletion();
+ return workerCoordinator.getResults().hasResults();
+ }
+
+
+ @Override
+ public Set<ScanColumn> next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No more elements" );
+ }
- return merge.results.size() > 0;
+ return workerCoordinator.getResults().copyAndClear();
}
- private void waitForCompletion() {
- if ( merged ) {
- return;
+ /**
+ * Coordinator object for all workers
+ */
+ private final class WorkerCoordinator {
+
+ private final ExecutorService executorService;
+ private final Collection<SearchVisitor> searchVisitors;
+ private final int pageSize;
+ private final QueryNode rootNode;
+ private ConcurrentResultMerge merge;
+ private ArrayList<Future<Void>> workers;
+
+
+ private WorkerCoordinator( final ExecutorService executorService,
+ final Collection<SearchVisitor> searchVisitors, final QueryNode rootNode,
+ final int pageSize ) {
+ this.executorService = executorService;
+ this.searchVisitors = searchVisitors;
+ this.rootNode = rootNode;
+ this.pageSize = pageSize;
}
- for ( final Future<Void> future : iterators ) {
- try {
- future.get();
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Unable to aggregate results", e );
+
+ public void start() {
+ this.merge = new ConcurrentResultMerge( pageSize );
+
+
+ this.workers = new ArrayList<Future<Void>>( searchVisitors.size() );
+
+
+ /**
+ * Start our search processing
+ */
+ for ( SearchVisitor visitor : searchVisitors ) {
+ final VisitorExecutor executor = new VisitorExecutor( rootNode, merge, visitor );
+
+// try {
+// executor.call();
+// }
+// catch ( Exception e ) {
+// throw new RuntimeException( e );
+// }
+ final Future<Void> result = executorService.submit( executor);
+ workers.add( result );
}
}
- merged = true;
- }
+ /**
+ * Get the results of the merge, after all workers have finished
+ * @return
+ */
+ public ConcurrentResultMerge getResults() {
- @Override
- public Set<ScanColumn> next() {
- if ( !hasNext() ) {
- throw new NoSuchElementException( "No more elements" );
- }
+ //make sure all our workers are done
+ for ( final Future<Void> future : workers ) {
+ try {
+ future.get();
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to aggregate results", e );
+ }
+ }
- return merge.copyAndClear();
+ return merge;
+ }
}
@@ -157,7 +193,7 @@ public class GatherIterator implements ResultIterator {
*/
private final class ConcurrentResultMerge {
- private final TreeSet<ScanColumn> results;
+ private TreeSet<ScanColumn> results;
private final int maxSize;
@@ -181,14 +217,25 @@ public class GatherIterator implements ResultIterator {
}
}
+
+ /**
+ * Return true if the merge has results
+ */
+ public boolean hasResults() {
+ return results != null && results.size() > 0;
+ }
+
+
/**
* Get the set
*/
- public Set<ScanColumn> copyAndClear() {
+ public synchronized Set<ScanColumn> copyAndClear() {
//create an immutable copy
- final Set<ScanColumn> toReturn = new LinkedHashSet<ScanColumn>( results );
- results.clear();
+ final Set<ScanColumn> toReturn = results ;
+ results = new TreeSet<ScanColumn>();
return toReturn;
}
+
+
}
}
[22/25] incubator-usergrid git commit: Fixes aws properties
Posted by sn...@apache.org.
Fixes aws properties
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a22c9960
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a22c9960
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a22c9960
Branch: refs/heads/master
Commit: a22c996077a4dc952b77f5d270cb2dc5987b241f
Parents: 219a425
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jul 8 16:02:02 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jul 8 16:02:02 2015 -0600
----------------------------------------------------------------------
stack/awscluster/.gitignore | 1 +
stack/awscluster/aws.properties | 19 -------------------
2 files changed, 1 insertion(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a22c9960/stack/awscluster/.gitignore
----------------------------------------------------------------------
diff --git a/stack/awscluster/.gitignore b/stack/awscluster/.gitignore
new file mode 100644
index 0000000..f3eb103
--- /dev/null
+++ b/stack/awscluster/.gitignore
@@ -0,0 +1 @@
+aws.properties
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a22c9960/stack/awscluster/aws.properties
----------------------------------------------------------------------
diff --git a/stack/awscluster/aws.properties b/stack/awscluster/aws.properties
deleted file mode 100644
index 182168a..0000000
--- a/stack/awscluster/aws.properties
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
-#
-accesskey=AKIAJBSDUBUSCVQKLNCQ
-secretkey=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
[25/25] incubator-usergrid git commit: This closes #302
Posted by sn...@apache.org.
This closes #302
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2547ee88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2547ee88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2547ee88
Branch: refs/heads/master
Commit: 2547ee88f5fe68f5037c8793c706fdf783bc5706
Parents: 7fa7f4e fd40f75
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 30 13:19:03 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 30 13:19:03 2015 -0400
----------------------------------------------------------------------
stack/awscluster/.gitignore | 1 +
.../main/resources/usergrid-default.properties | 5 +-
.../src/test/resources/usergrid-test.properties | 3 +
.../persistence/IndexBucketLocator.java | 32 +-
.../apache/usergrid/persistence/Results.java | 10 +-
.../persistence/cassandra/CassandraService.java | 42 +-
.../cassandra/ConnectionRefImpl.java | 10 +
.../cassandra/EntityManagerFactoryImpl.java | 2 +-
.../cassandra/EntityManagerImpl.java | 15 +-
.../persistence/cassandra/GeoIndexManager.java | 99 ++--
.../cassandra/QueryExecutorService.java | 31 ++
.../cassandra/QueryExecutorServiceImpl.java | 118 +++++
.../persistence/cassandra/QueryProcessor.java | 43 +-
.../cassandra/RelationManagerImpl.java | 472 ++++---------------
.../cassandra/SimpleIndexBucketLocatorImpl.java | 4 +-
.../cassandra/index/CassandraColumnUtils.java | 72 +++
.../cassandra/index/ConnectedIndexScanner.java | 66 ++-
.../index/DynamicCompositeComparator.java | 43 ++
.../DynamicCompositeForwardComparator.java | 40 ++
.../DynamicCompositeReverseComparator.java | 39 ++
.../index/DynamicCompositeStartToBytes.java | 48 ++
.../cassandra/index/IndexBucketScanner.java | 102 ++--
.../index/IndexMultiBucketSetLoader.java | 55 ---
.../cassandra/index/IndexScanner.java | 19 +-
.../cassandra/index/NoOpIndexScanner.java | 13 +-
.../cassandra/index/StartToBytes.java | 34 ++
.../cassandra/index/UUIDStartToBytes.java | 41 ++
.../persistence/geo/EntityLocationRef.java | 21 +-
.../persistence/geo/GeoIndexSearcher.java | 18 +-
.../persistence/query/ir/QuerySlice.java | 19 +
.../persistence/query/ir/SearchVisitor.java | 51 +-
.../query/ir/result/AbstractScanColumn.java | 31 +-
.../result/CollectionSearchVisitorFactory.java | 76 +++
.../query/ir/result/CollectionShardFilter.java | 53 +++
.../ir/result/ConnectionIndexSliceParser.java | 37 +-
.../result/ConnectionSearchVisitorFactory.java | 81 ++++
.../query/ir/result/ConnectionShardFilter.java | 50 ++
.../query/ir/result/CursorGenerator.java | 36 ++
.../query/ir/result/EmptyIterator.java | 5 -
.../query/ir/result/GatherIterator.java | 241 ++++++++++
.../query/ir/result/GeoIterator.java | 156 +++---
.../query/ir/result/IntersectionIterator.java | 35 +-
.../query/ir/result/OrderByIterator.java | 144 +++---
.../query/ir/result/ResultIterator.java | 4 +-
.../persistence/query/ir/result/ScanColumn.java | 17 +-
.../ir/result/SearchCollectionVisitor.java | 193 ++++++++
.../ir/result/SearchConnectionVisitor.java | 254 ++++++++++
.../query/ir/result/SearchVisitorFactory.java | 37 ++
.../ir/result/SecondaryIndexSliceParser.java | 252 +++++++++-
.../query/ir/result/ShardFilter.java | 29 ++
.../query/ir/result/ShardFilterIterator.java | 124 +++++
.../query/ir/result/SliceCursorGenerator.java | 55 +++
.../query/ir/result/SliceIterator.java | 85 +---
.../query/ir/result/SliceParser.java | 4 +-
.../query/ir/result/StaticIdIterator.java | 8 +-
.../query/ir/result/SubtractionIterator.java | 20 +-
.../persistence/query/ir/result/UUIDColumn.java | 49 ++
.../query/ir/result/UUIDCursorGenerator.java | 49 ++
.../query/ir/result/UUIDIndexSliceParser.java | 22 +-
.../query/ir/result/UnionIterator.java | 116 +++--
.../main/resources/usergrid-core-context.xml | 5 +
.../usergrid/persistence/CollectionIT.java | 2 +-
.../cassandra/QueryProcessorTest.java | 44 +-
.../SimpleIndexBucketLocatorImplTest.java | 22 +-
.../query/AbstractIteratingQueryIT.java | 3 +
.../query/ir/result/AbstractScanColumnTest.java | 10 +-
.../query/ir/result/InOrderIterator.java | 9 +-
.../query/ir/result/IteratorHelper.java | 4 +-
.../ir/result/ShardFilterIteratorTest.java | 168 +++++++
.../collection/activities/OrderByTest.java | 24 +-
.../apache/usergrid/tools/EntityCleanup.java | 80 ++--
.../usergrid/tools/EntityInsertBenchMark.java | 3 +-
.../usergrid/tools/EntityReadBenchMark.java | 11 +-
.../usergrid/tools/UniqueIndexCleanup.java | 186 ++++----
74 files changed, 3243 insertions(+), 1159 deletions(-)
----------------------------------------------------------------------
[19/25] incubator-usergrid git commit: Configures thread pool
Posted by sn...@apache.org.
Configures thread pool
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bbd56378
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bbd56378
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bbd56378
Branch: refs/heads/master
Commit: bbd563787b07a9bbc2f4a1647573a8c8ffedbc6e
Parents: 9a92fcc
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jul 8 14:46:56 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jul 8 14:46:56 2015 -0600
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +
.../src/test/resources/usergrid-test.properties | 3 +
.../cassandra/EntityManagerFactoryImpl.java | 2 +-
.../cassandra/EntityManagerImpl.java | 5 +-
.../cassandra/QueryExecutorService.java | 31 +++++++
.../cassandra/QueryExecutorServiceImpl.java | 95 ++++++++++++++++++++
.../persistence/cassandra/QueryProcessor.java | 31 +------
.../cassandra/RelationManagerImpl.java | 14 +--
.../query/ir/result/GatherIterator.java | 1 -
.../main/resources/usergrid-core-context.xml | 5 ++
.../cassandra/QueryProcessorTest.java | 44 ++++-----
11 files changed, 176 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 1aeb67f..27dbf4b 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -70,6 +70,9 @@ cassandra.lock.keyspace=Locks
# false to disable test features
usergrid.test=false
+#Number of threads to allow concurrent querying of Cassandra
+usergrid.query.threadcount=100
+
#Properties to control the number of buckets in the index.
usergrid.index.defaultbucketsize=20
usergrid.counter.skipAggregate=false
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/config/src/test/resources/usergrid-test.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/test/resources/usergrid-test.properties b/stack/config/src/test/resources/usergrid-test.properties
index 6b7cfff..c4536bd 100644
--- a/stack/config/src/test/resources/usergrid-test.properties
+++ b/stack/config/src/test/resources/usergrid-test.properties
@@ -44,6 +44,9 @@ cassandra.keyspace.strategy=org.apache.cassandra.locator.SimpleStrategy
cassandra.keyspace.replication=1
+#Our embedded cassandra can't handle the load. Limit the concurrent queries to keep from pushing it over
+usergrid.query.threadcount=5
+
cassandra.username=
cassandra.password=
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 53ccf40..421b286 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -144,7 +144,7 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
private EntityManager _getEntityManager( UUID applicationId ) {
//EntityManagerImpl em = new EntityManagerImpl();
- EntityManager em = applicationContext.getBean( "entityManager", EntityManager.class );
+ EntityManager em = applicationContext.getBean( "entityManager", EntityManager.class );
//em.init(this,cass,counterUtils,applicationId, skipAggregateCounters);
em.setApplicationId( applicationId );
return em;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 51383c2..370caa6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -193,6 +193,9 @@ public class EntityManagerImpl implements EntityManager {
@Resource
private IndexBucketLocator indexBucketLocator;
+ @Resource
+ private QueryExecutorService queryExecutorService;
+
private UUID applicationId;
private Application application;
@@ -270,7 +273,7 @@ public class EntityManagerImpl implements EntityManager {
public RelationManagerImpl getRelationManager( EntityRef entityRef ) {
//RelationManagerImpl rmi = applicationContext.getBean(RelationManagerImpl.class);
RelationManagerImpl rmi = new RelationManagerImpl();
- rmi.init( this, cass, applicationId, entityRef, indexBucketLocator );
+ rmi.init( this, cass, queryExecutorService, applicationId, entityRef, indexBucketLocator );
return rmi;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorService.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorService.java
new file mode 100644
index 0000000..07105b0
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorService.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.cassandra;
+
+
+import java.util.concurrent.ExecutorService;
+
+
+public interface QueryExecutorService {
+
+ /**
+ * Get the executor service
+ * @return
+ */
+ ExecutorService getExecutor();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorServiceImpl.java
new file mode 100644
index 0000000..7376641
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryExecutorServiceImpl.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.cassandra;
+
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ *
+ */
+public class QueryExecutorServiceImpl implements QueryExecutorService {
+
+
+ //we set up a thread pool with 100 execution threads. We purposefully use a synchronous queue and a caller runs so that we simply reject and immediately execute tasks if all 100 threads are occupied.
+
+
+
+ //we make this static, we only want 1 instance of this service in the JVM. Otherwise tests fail
+
+ private int threadCount;
+
+ private static ExecutorService executorService;
+
+
+ @Override
+ public ExecutorService getExecutor() {
+ if(executorService == null){
+ return getExecutorService();
+ }
+
+ return executorService;
+ }
+
+
+ public void setThreadCount( final int threadCount ) {
+ this.threadCount = threadCount;
+ }
+
+
+ /**
+ * Internally synchronized creation
+ * @return
+ */
+ private synchronized ExecutorService getExecutorService(){
+ if(executorService != null){
+ return executorService;
+ }
+
+
+ executorService = new ThreadPoolExecutor( threadCount, threadCount, 30, TimeUnit.SECONDS, new SynchronousQueue<Runnable>( ), new CallerRunsExecutionHandler() );
+ return executorService;
+ }
+
+
+ /**
+ * Execution handler that will run threads in the caller
+ */
+ private static class CallerRunsExecutionHandler implements RejectedExecutionHandler {
+
+ private static final Logger logger = LoggerFactory.getLogger( CallerRunsExecutionHandler.class );
+
+ @Override
+ public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
+ //when a task is rejected due to back pressure, we just want to run it in the caller.
+
+ logger.warn( "Concurrent shard execution rejected the task in executor {}, running it in the caller thread", executor );
+
+ r.run();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
index a2e0f60..10f4fa5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
@@ -22,12 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
import java.util.UUID;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,11 +87,7 @@ public class QueryProcessor {
private final CollectionInfo collectionInfo;
private final EntityManager em;
private final ResultsLoaderFactory loaderFactory;
-
- //we set up a thread pool with 100 execution threads. We purposefully use a synchronous queue and a caller runs so that we simply reject and immediately execute tasks if all 100 threads are occupied.
-
- //we make this static, we only want 1 instance of this service in the JVM. Otherwise tests fail
- private static final ExecutorService executorService = new ThreadPoolExecutor( 100, 100, 30, TimeUnit.SECONDS, new SynchronousQueue<Runnable>( ), new CallerRunsExecutionHandler() );
+ private final QueryExecutorService executorService;
private Operand rootOperand;
private List<SortPredicate> sorts;
@@ -110,11 +100,12 @@ public class QueryProcessor {
private int sliceCount;
- public QueryProcessor( Query query, CollectionInfo collectionInfo, EntityManager em,
+ public QueryProcessor( EntityManager em, QueryExecutorService executorService, Query query, CollectionInfo collectionInfo,
ResultsLoaderFactory loaderFactory ) throws PersistenceException {
setQuery( query );
this.collectionInfo = collectionInfo;
this.em = em;
+ this.executorService = executorService;
this.loaderFactory = loaderFactory;
process();
}
@@ -281,7 +272,7 @@ public class QueryProcessor {
//use the gather iterator to collect all the '
final int resultSetSize = Math.min( size, Query.MAX_LIMIT );
- ResultIterator itr = new GatherIterator(resultSetSize, rootNode, searchVisitorFactory.createVisitors(), executorService );
+ ResultIterator itr = new GatherIterator(resultSetSize, rootNode, searchVisitorFactory.createVisitors(), executorService.getExecutor() );
List<ScanColumn> entityIds = new ArrayList<ScanColumn>( );
@@ -696,19 +687,5 @@ public class QueryProcessor {
}
- private static class CallerRunsExecutionHandler implements RejectedExecutionHandler{
-
- private static final Logger logger = LoggerFactory.getLogger( CallerRunsExecutionHandler.class );
-
- @Override
- public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
- //when a task is rejected due to back pressure, we just want to run it in the caller.
-
- logger.warn( "Concurrent shard execution rejected the task in executor {}, running it in the caller thread", executor );
-
- r.run();
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index 74d1d60..86e1690 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -127,13 +127,14 @@ public class RelationManagerImpl implements RelationManager {
private UUID applicationId;
private EntityRef headEntity;
private IndexBucketLocator indexBucketLocator;
+ private QueryExecutorService executorService;
public RelationManagerImpl() {
}
- public RelationManagerImpl init( EntityManagerImpl em, CassandraService cass, UUID applicationId,
+ public RelationManagerImpl init( EntityManagerImpl em, CassandraService cass, QueryExecutorService executorService, UUID applicationId,
EntityRef headEntity, IndexBucketLocator indexBucketLocator ) {
Assert.notNull( em, "Entity manager cannot be null" );
@@ -148,6 +149,7 @@ public class RelationManagerImpl implements RelationManager {
this.cass = cass;
this.headEntity = headEntity;
this.indexBucketLocator = indexBucketLocator;
+ this.executorService = executorService;
return this;
}
@@ -155,7 +157,7 @@ public class RelationManagerImpl implements RelationManager {
private RelationManagerImpl getRelationManager( EntityRef headEntity ) {
RelationManagerImpl rmi = new RelationManagerImpl();
- rmi.init( em, cass, applicationId, headEntity, indexBucketLocator );
+ rmi.init( em, cass, executorService, applicationId, headEntity, indexBucketLocator );
return rmi;
}
@@ -1711,7 +1713,7 @@ public class RelationManagerImpl implements RelationManager {
// we have something to search with, visit our tree and evaluate the
// results
- QueryProcessor qp = new QueryProcessor( query, collection, em, factory );
+ QueryProcessor qp = new QueryProcessor( em, executorService, query, collection, factory );
CollectionSearchVisitorFactory collectionSearchVisitorFactory = new CollectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, collectionName );
// SearchCollectionVisitor visitor = new SearchCollectionVisitor( this, qp );
@@ -1903,7 +1905,7 @@ public class RelationManagerImpl implements RelationManager {
final ConnectionResultsLoaderFactory factory = new ConnectionResultsLoaderFactory( connectionRef );
- QueryProcessor qp = new QueryProcessor( query, null, em, factory );
+ QueryProcessor qp = new QueryProcessor( em, executorService, query, null,factory );
ConnectionSearchVisitorFactory collectionSearchVisitorFactory = new ConnectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, true, "" );
@@ -1946,7 +1948,7 @@ public class RelationManagerImpl implements RelationManager {
new ConnectionRefImpl( new SimpleEntityRef( connectedEntityType, null ), connectionType, targetEntity );
final ConnectionResultsLoaderFactory factory = new ConnectionResultsLoaderFactory( connectionRef );
- QueryProcessor qp = new QueryProcessor( query, null, em, factory );
+ QueryProcessor qp = new QueryProcessor( em, executorService, query, null, factory );
ConnectionSearchVisitorFactory collectionSearchVisitorFactory = new ConnectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, false, "" );
@@ -1987,7 +1989,7 @@ public class RelationManagerImpl implements RelationManager {
final ConnectionResultsLoaderFactory factory = new ConnectionResultsLoaderFactory( connectionRef );
- QueryProcessor qp = new QueryProcessor( query, null, em, factory );
+ QueryProcessor qp = new QueryProcessor(em, executorService, query, null, factory );
ConnectionSearchVisitorFactory collectionSearchVisitorFactory = new ConnectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, true, "" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index 4187ae0..36104f9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -181,7 +181,6 @@ public class GatherIterator implements ResultIterator {
}
}
-
/**
* Get the set
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml
index 7e69e19..78136ea 100644
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@ -67,6 +67,7 @@
<constructor-arg ref="cassandraHostConfigurator" />
</bean>
+
<bean id="loadBalancingPolicy" class="me.prettyprint.cassandra.connection.DynamicLoadBalancingPolicy"/>
<!-- locking for a single node -->
@@ -196,6 +197,10 @@
<bean id="metricsFactory" class="org.apache.usergrid.metrics.MetricsFactory" scope="singleton"/>
+ <bean id="queryExecutorService" class="org.apache.usergrid.persistence.cassandra.QueryExecutorServiceImpl" scope="singleton">
+ <property name="threadCount" value="${usergrid.query.threadcount}"/>
+ </bean>
+
<!-- scan all job classes -->
<context:component-scan base-package="org.apache.usergrid.batch.job" />
<context:annotation-config />
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbd56378/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
index c1fe7ee..4637da8 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
@@ -62,7 +62,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -88,7 +88,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -114,7 +114,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -140,7 +140,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -166,7 +166,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -192,7 +192,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -221,7 +221,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -250,7 +250,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -279,7 +279,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
WithinNode node = ( WithinNode ) processor.getFirstNode();
@@ -310,7 +310,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -361,7 +361,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
OrNode node = ( OrNode ) processor.getFirstNode();
@@ -407,7 +407,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
OrNode node = ( OrNode ) processor.getFirstNode();
@@ -476,7 +476,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
OrNode rootNode = ( OrNode ) processor.getFirstNode();
@@ -553,7 +553,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
AndNode rootNode = ( AndNode ) processor.getFirstNode();
@@ -600,7 +600,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
NotNode rootNode = ( NotNode ) processor.getFirstNode();
@@ -629,7 +629,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -658,7 +658,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -686,7 +686,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -722,7 +722,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -757,7 +757,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
query.setLimit( limit );
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
OrderByNode node = ( OrderByNode ) processor.getFirstNode();
@@ -783,7 +783,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
query.setLimit( limit );
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
SliceNode node = ( SliceNode ) processor.getFirstNode();
@@ -811,7 +811,7 @@ public class QueryProcessorTest {
Query query = parser.ql().query;
query.setLimit( limit );
- QueryProcessor processor = new QueryProcessor( query, null, null, null );
+ QueryProcessor processor = new QueryProcessor( null, null, query, null, null );;
QueryNode slice = processor.getFirstNode();
[05/25] incubator-usergrid git commit: Comparators partially
implemented.
Posted by sn...@apache.org.
Comparators partially implemented.
Merges iterators. Paging still incomplete.
Merges iterators. Paging still incomplete.
WIP overwrite
WIP overwrite
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/68c7eae1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/68c7eae1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/68c7eae1
Branch: refs/heads/master
Commit: 68c7eae1eb4409b067c8f3c3e4923b6b5e5a8e86
Parents: bb2837c
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Jun 18 11:06:17 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Jun 25 11:14:49 2015 -0600
----------------------------------------------------------------------
.../persistence/cassandra/CassandraService.java | 42 ++--
.../persistence/cassandra/QueryProcessor.java | 15 +-
.../cassandra/index/ConnectedIndexScanner.java | 4 +-
.../index/DynamicCompositeStartToBytes.java | 48 ++++
.../cassandra/index/IndexBucketScanner.java | 81 +++++--
.../cassandra/index/StartToBytes.java | 34 +++
.../cassandra/index/UUIDStartToBytes.java | 41 ++++
.../persistence/query/ir/SearchVisitor.java | 10 +-
.../query/ir/result/AbstractScanColumn.java | 32 +--
.../query/ir/result/GatherIterator.java | 111 +++++++---
.../ir/result/SearchCollectionVisitor.java | 14 +-
.../ir/result/SearchConnectionVisitor.java | 3 +-
.../ir/result/SecondaryIndexSliceParser.java | 220 ++++++++++++++++++-
.../query/ir/result/SliceIterator.java | 17 +-
.../persistence/query/ir/result/UUIDColumn.java | 36 +--
.../query/ir/result/UUIDIndexSliceParser.java | 2 +-
.../query/ir/result/UnionIterator.java | 18 +-
.../usergrid/persistence/CollectionIT.java | 2 +-
18 files changed, 552 insertions(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
index 58d5d0f..12ea338 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
@@ -39,6 +39,7 @@ import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
+import org.apache.usergrid.persistence.cassandra.index.UUIDStartToBytes;
import org.apache.usergrid.persistence.hector.CountingMutator;
import me.prettyprint.cassandra.connection.HConnectionManager;
@@ -1027,29 +1028,27 @@ public class CassandraService {
*
* @throws Exception the exception
*/
- public IndexScanner getIdList( Keyspace ko, Object key, UUID start, UUID finish, int count, boolean reversed,
- IndexBucketLocator locator, UUID applicationId, String collectionName, boolean keepFirst )
+ public IndexScanner getIdList( Object key, UUID start, UUID finish, int count, boolean reversed,
+ final String bucket, UUID applicationId, boolean keepFirst )
throws Exception {
- //TODO, refactor this
- throw new UnsupportedOperationException( "Implement me" );
-
- // if ( count <= 0 ) {
- // count = DEFAULT_COUNT;
- // }
- //
- // if ( NULL_ID.equals( start ) ) {
- // start = null;
- // }
- //
- //
- // final boolean skipFirst = start != null && !keepFirst;
- //
- // IndexScanner scanner =
- // new IndexBucketScanner( this, locator, ENTITY_ID_SETS, applicationId, IndexType.COLLECTION, key, start,
- // finish, reversed, count, skipFirst, collectionName );
- //
- // return scanner;
+ if ( count <= 0 ) {
+ count = DEFAULT_COUNT;
+ }
+
+ if ( NULL_ID.equals( start ) ) {
+ start = null;
+ }
+
+
+ final boolean skipFirst = start != null && !keepFirst;
+
+
+ IndexScanner scanner =
+ new IndexBucketScanner<UUID>( this, ENTITY_ID_SETS, UUIDStartToBytes.INSTANCE, applicationId, key, bucket, start,
+ finish, reversed, count, skipFirst );
+
+ return scanner;
}
@@ -1064,4 +1063,5 @@ public class CassandraService {
}
cluster = null;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
index f55aa67..ebaacf4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
@@ -448,7 +448,7 @@ public class QueryProcessor {
node = newSliceNode();
}
else {
- node = getUnionNode( op );
+ node = getUnionNode( );
}
String fieldName = op.getProperty().getIndexedValue();
@@ -485,7 +485,7 @@ public class QueryProcessor {
checkIndexed( propertyName );
- getUnionNode( op ).setFinish( propertyName, op.getLiteral().getValue(), false );
+ getUnionNode( ).setFinish( propertyName, op.getLiteral().getValue(), false );
}
@@ -502,7 +502,7 @@ public class QueryProcessor {
checkIndexed( propertyName );
- getUnionNode( op ).setFinish( propertyName, op.getLiteral().getValue(), true );
+ getUnionNode( ).setFinish( propertyName, op.getLiteral().getValue(), true );
}
@@ -519,7 +519,7 @@ public class QueryProcessor {
//checkIndexed( fieldName );
Literal<?> literal = op.getLiteral();
- SliceNode node = getUnionNode( op );
+ SliceNode node = getUnionNode( );
// this is an edge case. If we get more edge cases, we need to push
// this down into the literals and let the objects
@@ -554,7 +554,7 @@ public class QueryProcessor {
checkIndexed( propertyName );
- getUnionNode( op ).setStart( propertyName, op.getLiteral().getValue(), false );
+ getUnionNode( ).setStart( propertyName, op.getLiteral().getValue(), false );
}
@@ -570,7 +570,7 @@ public class QueryProcessor {
checkIndexed( propertyName );
- getUnionNode( op ).setStart( propertyName, op.getLiteral().getValue(), true );
+ getUnionNode( ).setStart( propertyName, op.getLiteral().getValue(), true );
}
@@ -578,9 +578,8 @@ public class QueryProcessor {
* Return the current leaf node to add to if it exists. This means that we can compress multiple 'AND'
* operations and ranges into a single node. Otherwise a new node is created and pushed to the stack
*
- * @param current The current operand node
*/
- private SliceNode getUnionNode( EqualityOperand current ) {
+ private SliceNode getUnionNode( ) {
/**
* we only create a new slice node in 3 situations 1. No nodes exist 2.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
index 87726cb..6c6152a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
@@ -174,11 +174,11 @@ public class ConnectedIndexScanner implements IndexScanner {
}
//remove the first element, we need to skip it
- if ( skipFirst && lastResults.size() > 0) {
+ if ( skipFirst && lastResults != null && lastResults.size() > 0) {
lastResults.remove( 0 );
}
- if ( hasMore && lastResults.size() > 0 ) {
+ if ( hasMore && lastResults !=null && lastResults.size() > 0 ) {
// set the bytebuffer for the next pass
lastResults.remove( lastResults.size() - 1 );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeStartToBytes.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeStartToBytes.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeStartToBytes.java
new file mode 100644
index 0000000..072e8ca
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/DynamicCompositeStartToBytes.java
@@ -0,0 +1,48 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.usergrid.persistence.cassandra.index;
+
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import me.prettyprint.cassandra.serializers.UUIDSerializer;
+import me.prettyprint.hector.api.beans.DynamicComposite;
+
+
+/**
+ * Converts a DynamicComposite to bytes
+ */
+public class DynamicCompositeStartToBytes implements StartToBytes<DynamicComposite> {
+
+ public static final DynamicCompositeStartToBytes INSTANCE = new DynamicCompositeStartToBytes();
+
+ private DynamicCompositeStartToBytes(){}
+
+
+ @Override
+ public ByteBuffer toBytes( final DynamicComposite toBytes ) {
+
+ if(toBytes == null){
+ return null;
+ }
+
+ return toBytes.serialize();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
index e568575..bf61846 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
@@ -18,19 +18,16 @@ package org.apache.usergrid.persistence.cassandra.index;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.TreeSet;
import java.util.UUID;
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
+import org.apache.cassandra.utils.FBUtilities;
+
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
import org.apache.usergrid.persistence.cassandra.CassandraService;
+import com.google.common.primitives.UnsignedBytes;
import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.beans.HColumn;
@@ -44,7 +41,7 @@ import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtil
*
* @author tnine
*/
-public class IndexBucketScanner implements IndexScanner {
+public class IndexBucketScanner<T> implements IndexScanner {
private final CassandraService cass;
private final UUID applicationId;
@@ -55,12 +52,13 @@ public class IndexBucketScanner implements IndexScanner {
private final int pageSize;
private final boolean skipFirst;
private final String bucket;
+ private final StartToBytes<T> scanStartSerializer;
/** Pointer to our next start read */
- private Object start;
+ private ByteBuffer start;
/** Set to the original value to start scanning from */
- private Object scanStart;
+ private T initialStartValue;
/** Iterator for our results from the last page load */
private List<HColumn<ByteBuffer, ByteBuffer>> lastResults;
@@ -70,14 +68,14 @@ public class IndexBucketScanner implements IndexScanner {
- public IndexBucketScanner( CassandraService cass, ApplicationCF columnFamily,
- UUID applicationId, Object keyPrefix, String bucket, Object start, Object finish,
+ public IndexBucketScanner( CassandraService cass, ApplicationCF columnFamily, StartToBytes<T> scanStartSerializer,
+ UUID applicationId, Object keyPrefix, String bucket, T start, T finish,
boolean reversed, int pageSize, boolean skipFirst) {
this.cass = cass;
+ this.scanStartSerializer = scanStartSerializer;
this.applicationId = applicationId;
this.keyPrefix = keyPrefix;
this.columnFamily = columnFamily;
- this.start = start;
this.finish = finish;
this.reversed = reversed;
this.skipFirst = skipFirst;
@@ -85,7 +83,10 @@ public class IndexBucketScanner implements IndexScanner {
//we always add 1 to the page size. This is because we pop the last column for the next page of results
this.pageSize = pageSize+1;
- this.scanStart = start;
+
+ //the initial value set when we started scanning
+ this.initialStartValue = start;
+ this.start = scanStartSerializer.toBytes( initialStartValue );
}
@@ -95,7 +96,7 @@ public class IndexBucketScanner implements IndexScanner {
@Override
public void reset() {
hasMore = true;
- start = scanStart;
+ start = scanStartSerializer.toBytes( initialStartValue );
}
@@ -122,12 +123,10 @@ public class IndexBucketScanner implements IndexScanner {
//we purposefully use instance equality. If it's a pointer to the same value, we need to increase by 1
//since we'll be skipping the first value
- final boolean firstPageSkipFirst = this.skipFirst && start == scanStart;
-
- if(firstPageSkipFirst){
- selectSize++;
- }
+ final boolean shouldCheckFirst =
+ //we should skip the value it's a cursor resume OR it's a previous page from a stateful iterator
+ (this.skipFirst && initialStartValue != null) || start != null;
final List<HColumn<ByteBuffer, ByteBuffer>>
resultsTree = cass.getColumns( cass.getApplicationKeyspace( applicationId ), columnFamily, rowKey,
@@ -140,7 +139,6 @@ public class IndexBucketScanner implements IndexScanner {
if ( resultsTree.size() == selectSize ) {
hasMore = true;
-
// set the bytebuffer for the next pass
start = resultsTree.get( resultsTree.size() - 1 ).getName();
}
@@ -149,8 +147,25 @@ public class IndexBucketScanner implements IndexScanner {
}
//remove the first element since it needs to be skipped AFTER the size check. Otherwise it will fail
- if ( firstPageSkipFirst ) {
- resultsTree.remove( 0 );
+ //we only want to skip if our byte value are the same as our expected start. Since we aren't stateful you can't
+ //be sure your start even comes back, and you don't want to erroneously remove columns
+ if ( shouldCheckFirst && resultsTree.size() > 0 && start != null) {
+ final int startIndex = start.position();
+ final int startLength = start.remaining();
+
+
+
+ final ByteBuffer returnedBuffer = resultsTree.get( 0 ).getName();
+ final int returnedIndex = returnedBuffer.position();
+ final int returnedLength = returnedBuffer.remaining();
+
+
+ final int compare = FBUtilities.compareUnsigned( start.array(), returnedBuffer.array(), startIndex, returnedIndex, startLength, returnedLength ) ;
+
+ //the byte buffers are the same as our seek (which may or may not be the case in the first seek)
+ if(compare == 0){
+ resultsTree.remove( 0 );
+ }
}
lastResults = resultsTree;
@@ -159,6 +174,27 @@ public class IndexBucketScanner implements IndexScanner {
}
+ /**
+ * Returns true if the 2 byte buffers contain the same bytes, false otherwise
+ * @param first
+ * @param second
+ * @return
+ */
+ private boolean compareBuffer(final ByteBuffer first, final ByteBuffer second){
+ int firstLength = first.remaining();
+ int firstPosition = first.position();
+
+ int secondLength = second.remaining();
+ int secondPosition = second.position();
+
+ final int compare = FBUtilities.compareUnsigned( first.array(), second.array(), firstPosition, secondPosition, firstLength, secondLength);
+
+ return compare == 0;
+
+
+ }
+
+
/*
* (non-Javadoc)
*
@@ -235,4 +271,5 @@ public class IndexBucketScanner implements IndexScanner {
public boolean isReversed() {
return this.reversed;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/StartToBytes.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/StartToBytes.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/StartToBytes.java
new file mode 100644
index 0000000..c6516b8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/StartToBytes.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.cassandra.index;
+
+
+import java.nio.ByteBuffer;
+
+
+/**
+ * Parse our index values into byte buffer
+ */
+public interface StartToBytes<T> {
+
+ /**
+ * Convert the start scanning type to bytes
+ * @param toBytes
+ * @return
+ */
+ ByteBuffer toBytes(final T toBytes);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/UUIDStartToBytes.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/UUIDStartToBytes.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/UUIDStartToBytes.java
new file mode 100644
index 0000000..2cdf9fd
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/UUIDStartToBytes.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.cassandra.index;
+
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import me.prettyprint.cassandra.serializers.UUIDSerializer;
+
+
+/**
+ * Converts a UUID to bytes
+ */
+public class UUIDStartToBytes implements StartToBytes<UUID> {
+
+ public static final UUIDStartToBytes INSTANCE = new UUIDStartToBytes();
+
+ private UUIDStartToBytes(){}
+
+ private static final UUIDSerializer UUID_SERIALIZER = UUIDSerializer.get();
+
+ @Override
+ public ByteBuffer toBytes( final UUID toBytes ) {
+ return UUID_SERIALIZER.toByteBuffer( toBytes );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
index 410b99f..fc2df4d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
@@ -49,8 +49,6 @@ import org.apache.usergrid.persistence.query.ir.result.UnionIterator;
*/
public abstract class SearchVisitor implements NodeVisitor {
- private static final SecondaryIndexSliceParser COLLECTION_PARSER = new SecondaryIndexSliceParser();
-
protected final Query query;
protected final QueryProcessor queryProcessor;
@@ -162,7 +160,7 @@ public abstract class SearchVisitor implements NodeVisitor {
final int nodeId = node.getId();
- UnionIterator union = new UnionIterator( queryProcessor.getPageSizeHint( node ), nodeId, queryProcessor.getCursorCache( nodeId ) );
+ UnionIterator union = new UnionIterator( queryProcessor.getPageSizeHint( node ), nodeId );
if ( left != null ) {
union.addIterator( left );
@@ -211,7 +209,7 @@ public abstract class SearchVisitor implements NodeVisitor {
if ( subResults == null ) {
QuerySlice firstFieldSlice = new QuerySlice( slice.getPropertyName(), -1 );
subResults =
- new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), COLLECTION_PARSER );
+ new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), new SecondaryIndexSliceParser() );
}
orderIterator = new OrderByIterator( slice, orderByNode.getSecondarySorts(), subResults, em,
@@ -230,7 +228,7 @@ public abstract class SearchVisitor implements NodeVisitor {
scanner = secondaryIndexScan( orderByNode, slice );
}
- SliceIterator joinSlice = new SliceIterator( slice, scanner, COLLECTION_PARSER);
+ SliceIterator joinSlice = new SliceIterator( slice, scanner, new SecondaryIndexSliceParser());
IntersectionIterator union = new IntersectionIterator( queryProcessor.getPageSizeHint( orderByNode ) );
union.addIterator( joinSlice );
@@ -261,7 +259,7 @@ public abstract class SearchVisitor implements NodeVisitor {
for ( QuerySlice slice : node.getAllSlices() ) {
IndexScanner scanner = secondaryIndexScan( node, slice );
- intersections.addIterator( new SliceIterator( slice, scanner, COLLECTION_PARSER) );
+ intersections.addIterator( new SliceIterator( slice, scanner, new SecondaryIndexSliceParser()) );
}
results.push( intersections );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
index 383ead7..2a359be 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
@@ -30,9 +30,9 @@ import org.apache.cassandra.utils.ByteBufferUtil;
*/
public abstract class AbstractScanColumn implements ScanColumn {
- private final UUID uuid;
- private final ByteBuffer buffer;
- private ScanColumn child;
+ protected final UUID uuid;
+ protected final ByteBuffer buffer;
+ protected ScanColumn child;
protected AbstractScanColumn( final UUID uuid, final ByteBuffer columnNameBuffer ) {
@@ -58,13 +58,13 @@ public abstract class AbstractScanColumn implements ScanColumn {
if ( this == o ) {
return true;
}
- if ( !( o instanceof AbstractScanColumn ) ) {
+ if ( !( o instanceof ScanColumn ) ) {
return false;
}
- AbstractScanColumn that = ( AbstractScanColumn ) o;
+ ScanColumn that = ( ScanColumn ) o;
- return uuid.equals( that.uuid );
+ return uuid.equals( that.getUUID() );
}
@@ -94,24 +94,4 @@ public abstract class AbstractScanColumn implements ScanColumn {
return child;
}
-
- /**
- * Comparator for comparing children. A null safe call
- * @param otherScanColumn
- * @return
- */
- protected int compareChildren( final ScanColumn otherScanColumn ) {
-
- if ( otherScanColumn == null ) {
- return 1;
- }
-
- final ScanColumn otherChild = otherScanColumn.getChild();
-
- if ( child != null && otherChild != null ) {
- return child.compareTo( otherChild );
- }
-
- return 0;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index 29912cc..5bc91e0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -1,11 +1,16 @@
package org.apache.usergrid.persistence.query.ir.result;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
+import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
@@ -21,18 +26,20 @@ import org.apache.usergrid.persistence.query.ir.SearchVisitor;
public class GatherIterator implements ResultIterator {
- private final Collection<SearchVisitor> searchVisitors;
private final QueryNode rootNode;
private final int pageSize;
- private Set<ScanColumn> next;
+ private Iterator<ScanColumn> mergedIterators;
+ private Map<UUID, ResultIterator> cursorMap;
+ private List<ResultIterator> iterators;
public GatherIterator(final int pageSize, final QueryNode rootNode, final Collection<SearchVisitor> searchVisitors) {
this.pageSize = pageSize;
this.rootNode = rootNode;
- this.searchVisitors = searchVisitors;
+ this.cursorMap = new HashMap<UUID, ResultIterator>( pageSize );
+ createIterators( searchVisitors );
}
@@ -45,6 +52,14 @@ public class GatherIterator implements ResultIterator {
@Override
public void finalizeCursor( final CursorCache cache, final UUID lastValue ) {
//find the last value in the tree, and return it's cursor
+ final ResultIterator sourceIterator = cursorMap.get( lastValue );
+
+ if(sourceIterator == null){
+ throw new IllegalArgumentException( "Could not find the iterator that provided the candidate with uuid " + lastValue );
+ }
+
+ //delegate to the source iterator
+ sourceIterator.finalizeCursor( cache, lastValue );
}
@@ -56,12 +71,11 @@ public class GatherIterator implements ResultIterator {
@Override
public boolean hasNext() {
-
- if(next() == null){
- advance();
+ if(mergedIterators == null || !mergedIterators.hasNext()){
+ mergeIterators();
}
- return next != null;
+ return mergedIterators.hasNext();
}
@@ -71,38 +85,76 @@ public class GatherIterator implements ResultIterator {
throw new NoSuchElementException( "No more elements" );
}
- final Set<ScanColumn> results = next;
- next = null;
- return results;
+ return getNextPage();
+ }
+
+ private void createIterators(final Collection<SearchVisitor> searchVisitors ){
+
+ this.iterators = new ArrayList<ResultIterator>( searchVisitors.size() );
+
+ for(SearchVisitor visitor: searchVisitors){
+
+ try {
+ rootNode.visit( visitor );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to process query", e );
+ }
+
+ final ResultIterator iterator = visitor.getResults();
+
+ iterators.add( iterator );
+
+ }
+
}
/**
+ * Get the next page of results
+ * @return
+ */
+ private Set<ScanColumn> getNextPage(){
+
+ //try to take from our PageSize
+ LinkedHashSet<ScanColumn> resultSet = new LinkedHashSet<ScanColumn>( pageSize );
+
+ for(int i = 0; i < pageSize && mergedIterators.hasNext(); i ++){
+ resultSet.add( mergedIterators.next() );
+ }
+
+
+ return resultSet;
+ }
+
+ /**
* Advance the iterator
*/
- private void advance(){
+ private void mergeIterators(){
//TODO make this concurrent
+ //clear the cursor map
+ cursorMap.clear();
- final TreeSet<ScanColumn> results = new TreeSet<ScanColumn>( );
-
+ TreeSet<ScanColumn> merged = new TreeSet<ScanColumn>( );
- for(SearchVisitor visitor: searchVisitors){
- merge(results, visitor);
+ for(ResultIterator iterator: this.iterators){
+ merge(merged, iterator);
}
- this.next = results;
+ mergedIterators = merged.iterator();
+
}
+
+
/**
* Merge this interator into our final column results
* @param results
- * @param visitor
+ * @param iterator
*/
- private void merge(final TreeSet<ScanColumn> results, final SearchVisitor visitor){
-
- final ResultIterator iterator = visitor.getResults();
+ private void merge(final TreeSet<ScanColumn> results, final ResultIterator iterator){
//nothing to do, return
@@ -110,19 +162,24 @@ public class GatherIterator implements ResultIterator {
return;
}
-
final Iterator<ScanColumn> nextPage = iterator.next().iterator();
-
//only take from the iterator what we need to create a full page.
for(int i = 0 ; i < pageSize && nextPage.hasNext(); i ++){
- results.add( nextPage.next() );
+ final ScanColumn next = nextPage.next();
+
+ results.add(next);
+ cursorMap.put( next.getUUID(), iterator );
+
+// //results are too large, trim them
+// if(results.size() > pageSize){
+// final ScanColumn toRemove = results.pollLast();
+// cursorMap.remove( toRemove.getUUID() );
+// }
- //results are too large, trim them
- if(results.size() > pageSize){
- results.pollLast();
- }
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index 78abbca..c20baf4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -7,6 +7,7 @@ import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.QueryProcessor;
+import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeStartToBytes;
import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
@@ -77,7 +78,7 @@ public class SearchCollectionVisitor extends SearchVisitor {
// perform the search
else {
columns =
- searchIndexBuckets( indexKey, slice, collection.getName(), queryProcessor.getPageSizeHint( node ) );
+ searchIndexBuckets( indexKey, slice, queryProcessor.getPageSizeHint( node ) );
}
return columns;
@@ -100,10 +101,10 @@ public class SearchCollectionVisitor extends SearchVisitor {
IndexScanner indexScanner = cassandraService
- .getIdList( cassandraService.getApplicationKeyspace( applicationId ),
+ .getIdList(
key( headEntity.getUuid(), DICTIONARY_COLLECTIONS, collectionName ), startId, null,
- queryProcessor.getPageSizeHint( node ), query.isReversed(), indexBucketLocator, applicationId,
- collectionName, node.isForceKeepFirst() );
+ queryProcessor.getPageSizeHint( node ), query.isReversed(), bucket, applicationId,
+ node.isForceKeepFirst() );
this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER ) );
}
@@ -149,10 +150,9 @@ public class SearchCollectionVisitor extends SearchVisitor {
*
* @param indexKey The index key to read
* @param slice Slice set in the query
- * @param collectionName The name of the collection to search
* @param pageSize The page size to load when iterating
*/
- private IndexScanner searchIndexBuckets( Object indexKey, QuerySlice slice, String collectionName, int pageSize )
+ private IndexScanner searchIndexBuckets( Object indexKey, QuerySlice slice, int pageSize )
throws Exception {
DynamicComposite[] range = slice.getRange();
@@ -160,7 +160,7 @@ public class SearchCollectionVisitor extends SearchVisitor {
Object keyPrefix = key( indexKey, slice.getPropertyName() );
IndexScanner scanner =
- new IndexBucketScanner( cassandraService, ENTITY_INDEX, applicationId, keyPrefix, bucket, range[0],
+ new IndexBucketScanner( cassandraService, ENTITY_INDEX, DynamicCompositeStartToBytes.INSTANCE, applicationId, keyPrefix, bucket, range[0],
range[1], slice.isReversed(), pageSize, slice.hasCursor() );
return scanner;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 74ff19f..6f833db 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -13,6 +13,7 @@ import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
import org.apache.usergrid.persistence.cassandra.QueryProcessor;
import org.apache.usergrid.persistence.cassandra.index.ConnectedIndexScanner;
+import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeStartToBytes;
import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
@@ -205,7 +206,7 @@ public class SearchConnectionVisitor extends SearchVisitor {
IndexScanner scanner =
- new IndexBucketScanner( cassandraService, ENTITY_INDEX, applicationId,
+ new IndexBucketScanner( cassandraService, ENTITY_INDEX, DynamicCompositeStartToBytes.INSTANCE, applicationId,
keyPrefix, shardBucket, range[0], range[1], slice.isReversed(), pageSize, slice.hasCursor());
return scanner;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
index e27d99c..b26f684 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
@@ -17,50 +17,85 @@
package org.apache.usergrid.persistence.query.ir.result;
+import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
import java.util.UUID;
+import org.apache.usergrid.utils.UUIDUtils;
+
import me.prettyprint.hector.api.beans.DynamicComposite;
/**
- * Parser for reading and writing secondary index composites
+ * Parser for reading and writing secondary index composites. Instances of this class should not be shared among
+ * iterators.
+ *
+ * It it designed with the following assumptions in mind.
+ *
+ * 1) The slice contains the same data type for every element 2) Evaluating the first parse call for a comparator is
+ * sufficient for subsequent use
*
* @author tnine
*/
public class SecondaryIndexSliceParser implements SliceParser {
+ //the type comparator
+ private Comparator<SecondaryIndexColumn> typeComparator;
+
+
+ public SecondaryIndexSliceParser() {}
+
/* (non-Javadoc)
* @see org.apache.usergrid.persistence.query.ir.result.SliceParser#parse(java.nio.ByteBuffer)
*/
@Override
- public ScanColumn parse( ByteBuffer buff, final boolean isReversed) {
- DynamicComposite composite = DynamicComposite.fromByteBuffer( buff.duplicate() );
+ public ScanColumn parse( ByteBuffer buff, final boolean isReversed ) {
+ final DynamicComposite composite = DynamicComposite.fromByteBuffer( buff.duplicate() );
- throw new UnsupportedOperationException( "Implement me with static comparators" );
+ final UUID uuid = ( UUID ) composite.get( 2 );
+ final Object value = composite.get( 1 );
-// return new SecondaryIndexColumn( ( UUID ) composite.get( 2 ), composite.get( 1 ), buff, null );
+ if ( typeComparator == null ) {
+ typeComparator = getTypeComparator( value, isReversed );
+ }
+
+ return new SecondaryIndexColumn( uuid, value, buff, typeComparator );
}
+ private Comparator<SecondaryIndexColumn> getTypeComparator( final Object value, final boolean isReversed ) {
+
+ final Class clazz = value.getClass();
+ final Comparator<SecondaryIndexColumn> comparator = COMPARATOR_MAP.get( new MapKey( clazz, isReversed ) );
+
+ if ( comparator == null ) {
+ throw new NullPointerException( "comparator was not found for runtime type '" + clazz + "'" );
+ }
+
+ return comparator;
+ }
+
+ /**
+ * Column for our secondary index type
+ */
public static class SecondaryIndexColumn extends AbstractScanColumn {
private final Object value;
- private final Comparator<Object> valueComparator;
+ private final Comparator<SecondaryIndexColumn> valueComparator;
/**
* Create the secondary index column
- * @param uuid
- * @param value
- * @param columnNameBuffer
+ *
* @param valueComparator The comparator for the values
*/
public SecondaryIndexColumn( final UUID uuid, final Object value, final ByteBuffer columnNameBuffer,
- final Comparator<Object> valueComparator ) {
+ final Comparator<SecondaryIndexColumn> valueComparator ) {
super( uuid, columnNameBuffer );
this.value = value;
this.valueComparator = valueComparator;
@@ -74,8 +109,169 @@ public class SecondaryIndexSliceParser implements SliceParser {
@Override
- public int compareTo( final ScanColumn o ) {
- throw new UnsupportedOperationException( "Impelment me" );
+ public int compareTo( final ScanColumn other ) {
+ if ( other == null ) {
+ return 1;
+ }
+
+ return valueComparator.compare( this, ( SecondaryIndexColumn ) other );
+ }
+ }
+
+
+ private static final Map<MapKey, Comparator<SecondaryIndexColumn>> COMPARATOR_MAP =
+ new HashMap<MapKey, Comparator<SecondaryIndexColumn>>();
+
+ static {
+
+ final LongComparator longComparator = new LongComparator();
+ COMPARATOR_MAP.put( new MapKey( Long.class, false ), longComparator );
+ COMPARATOR_MAP.put( new MapKey( Long.class, true ), new ReverseComparator( longComparator ) );
+
+ final StringComparator stringComparator = new StringComparator();
+
+ COMPARATOR_MAP.put( new MapKey( String.class, false ), stringComparator );
+ COMPARATOR_MAP.put( new MapKey( String.class, true ), new ReverseComparator( stringComparator ) );
+
+
+ final UUIDComparator uuidComparator = new UUIDComparator();
+
+ COMPARATOR_MAP.put( new MapKey( UUID.class, false ), uuidComparator );
+ COMPARATOR_MAP.put( new MapKey( UUID.class, true ), new ReverseComparator( uuidComparator ) );
+
+ final BigIntegerComparator bigIntegerComparator = new BigIntegerComparator();
+
+ COMPARATOR_MAP.put( new MapKey( BigInteger.class, false ), bigIntegerComparator );
+ COMPARATOR_MAP.put( new MapKey( BigInteger.class, true ), new ReverseComparator( bigIntegerComparator ) );
+ }
+
+
+ /**
+ * The key for the map
+ */
+ private static final class MapKey {
+ public final Class<?> clazz;
+ public final boolean reversed;
+
+
+ private MapKey( final Class<?> clazz, final boolean reversed ) {
+ this.clazz = clazz;
+ this.reversed = reversed;
+ }
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( !( o instanceof MapKey ) ) {
+ return false;
+ }
+
+ final MapKey mapKey = ( MapKey ) o;
+
+ if ( reversed != mapKey.reversed ) {
+ return false;
+ }
+ return clazz.equals( mapKey.clazz );
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = clazz.hashCode();
+ result = 31 * result + ( reversed ? 1 : 0 );
+ return result;
+ }
+ }
+
+
+ private static abstract class SecondaryIndexColumnComparator implements Comparator<SecondaryIndexColumn> {
+
+ /**
+ * If the result of compare is != 0 it is returned, otherwise the uuids are compared
+ */
+ protected int compareValues( final int compare, final SecondaryIndexColumn first,
+ SecondaryIndexColumn second ) {
+ if ( compare != 0 ) {
+ return compare;
+ }
+
+ return com.fasterxml.uuid.UUIDComparator.staticCompare( first.uuid, second.uuid );
+ }
+ }
+
+
+ private static final class LongComparator extends SecondaryIndexColumnComparator {
+
+ @Override
+ public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
+
+ final Long firstLong = ( Long ) first.value;
+ final Long secondLong = ( Long ) second.value;
+
+
+ return compareValues( Long.compare( firstLong, secondLong ), first, second );
+ }
+ }
+
+
+ private static final class StringComparator extends SecondaryIndexColumnComparator {
+ @Override
+ public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
+
+ if ( first == null && second != null ) {
+ return -1;
+ }
+
+ final String firstString = ( String ) first.value;
+ final String secondString = ( String ) second.value;
+
+
+ return compareValues( firstString.compareTo( secondString ), first, second );
+ }
+ }
+
+
+ private static final class UUIDComparator extends SecondaryIndexColumnComparator {
+ @Override
+ public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
+ final UUID firstUUID = ( UUID ) first.value;
+ final UUID secondUUID = ( UUID ) second.value;
+
+
+ return compareValues( UUIDUtils.compare( firstUUID, secondUUID ), first, second );
+ }
+ }
+
+
+ private static final class BigIntegerComparator extends SecondaryIndexColumnComparator {
+ @Override
+ public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
+ final BigInteger firstInt = ( BigInteger ) first.value;
+ final BigInteger secondInt = ( BigInteger ) second.value;
+
+
+ return compareValues( firstInt.compareTo( secondInt ), first, second );
+ }
+ }
+
+
+ /**
+ * Reversed our comparator
+ */
+ private static final class ReverseComparator implements Comparator<SecondaryIndexColumn> {
+
+ private final Comparator<SecondaryIndexColumn> comparator;
+
+
+ private ReverseComparator( final Comparator<SecondaryIndexColumn> comparator ) {this.comparator = comparator;}
+
+
+ @Override
+ public int compare( final SecondaryIndexColumn first, final SecondaryIndexColumn second ) {
+ return comparator.compare( first, second ) * -1;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
index 3f052da..a386159 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -26,6 +26,7 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.persistence.cassandra.CursorCache;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.exceptions.QueryIterationException;
@@ -127,9 +128,9 @@ public class SliceIterator implements ResultIterator {
while ( results.hasNext() ) {
- ByteBuffer colName = results.next().getName().duplicate();
+ final ByteBuffer colName = results.next().getName().duplicate();
- ScanColumn parsed = parser.parse( colName, isReversed );
+ final ScanColumn parsed = parser.parse( colName, isReversed );
//skip this value, the parser has discarded it
if ( parsed == null ) {
@@ -215,22 +216,22 @@ public class SliceIterator implements ResultIterator {
//this is a bug
if ( scanner.hasNext() ) {
logger.error(
- "An iterator attempted to access a slice that was not iterated over. This will result in the" +
- " cursor construction failing" );
+ "An iterator attempted to access a slice that was not iterated over. This will result in the"
+ + " cursor construction failing" );
throw new QueryIterationException(
- "An iterator attempted to access a slice that was not iterated over. This will result in the" +
- " cursor construction failing" );
+ "An iterator attempted to access a slice that was not iterated over. This will result in the"
+ + " cursor construction failing" );
}
final ByteBuffer sliceCursor = slice.getCursor();
//we've never loaded anything, just re-use the existing slice
- if (last == null && sliceCursor != null ) {
+ if ( last == null && sliceCursor != null ) {
bytes = sliceCursor;
}
//use the last column we loaded. This way our scan returns nothing next time since start == finish
- else if(last != null) {
+ else if ( last != null ) {
bytes = last.getCursorValue();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
index a5e09c1..640b391 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
@@ -4,53 +4,35 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.nio.ByteBuffer;
import java.util.UUID;
+import org.apache.usergrid.persistence.cassandra.Serializers;
import org.apache.usergrid.utils.UUIDUtils;
/**
* Used as a comparator for columns
*/
-class UUIDColumn implements ScanColumn{
+class UUIDColumn extends AbstractScanColumn{
- private final UUID uuid;
private final int compareReversed;
- private ScanColumn child;
-
- UUIDColumn( final UUID uuid, final int compareReversed ) {
- this.uuid = uuid;
+ protected UUIDColumn( final UUID uuid, final ByteBuffer columnNameBuffer, final int compareReversed ) {
+ super( uuid, columnNameBuffer );
this.compareReversed = compareReversed;
}
- @Override
- public UUID getUUID() {
- return uuid;
- }
-
-
- @Override
- public ByteBuffer getCursorValue() {
- return null;
- }
-
-
- @Override
- public void setChild( final ScanColumn childColumn ) {
- this.child = childColumn;
+ public UUIDColumn( final UUID uuid, final int compareReversed ) {
+ super(uuid, Serializers.ue.toByteBuffer( uuid ));
+ this.compareReversed = compareReversed;
}
- @Override
- public ScanColumn getChild() {
- return child;
- }
@Override
public int compareTo( final ScanColumn other ) {
-
return UUIDUtils.compare( uuid, other.getUUID() ) * compareReversed;
-
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
index 91644ce..2c5b1ba 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
@@ -33,6 +33,6 @@ public class UUIDIndexSliceParser implements SliceParser {
@Override
public ScanColumn parse( final ByteBuffer columnNameBytes, final boolean isReversed ) {
final int compare = isReversed? -1: 1;
- return new UUIDColumn( ue.fromByteBuffer( columnNameBytes.duplicate() ), compare );
+ return new UUIDColumn( ue.fromByteBuffer( columnNameBytes.duplicate() ), columnNameBytes, compare );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
index 635ca97..aca9852 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -133,7 +133,7 @@ public class UnionIterator extends MultiIterator {
private final List<ScanColumn> list;
- private UUIDColumn min;
+// private UUIDColumn min;
public SortedColumnList( final int maxSize, final UUID minUuid ) {
@@ -141,9 +141,9 @@ public class UnionIterator extends MultiIterator {
this.list = new ArrayList<ScanColumn>( maxSize );
this.maxSize = maxSize;
- if ( minUuid != null ) {
- min = new UUIDColumn( minUuid, 1 ) ;
- }
+// if ( minUuid != null ) {
+// min = new UUIDColumn( minUuid, 1 ) ;
+// }
}
@@ -152,9 +152,9 @@ public class UnionIterator extends MultiIterator {
*/
public void add( ScanColumn col ) {
//less than our min, don't add
- if ( min != null && min.compareTo( col ) >= 0 ) {
- return;
- }
+// if ( min != null && min.compareTo( col ) >= 0 ) {
+// return;
+// }
int index = Collections.binarySearch( this.list, col );
@@ -215,7 +215,7 @@ public class UnionIterator extends MultiIterator {
}
final UUID oldMin = this.list.get( size - 1 ).getUUID();
- min = new UUIDColumn( oldMin, 1 );
+// min = new UUIDColumn( oldMin, 1 );
}
@@ -228,7 +228,7 @@ public class UnionIterator extends MultiIterator {
public void reset(){
clear();
- this.min = null;
+// this.min = null;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68c7eae1/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
index ac58bd7..99fed21 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
@@ -1444,7 +1444,7 @@ public class CollectionIT extends AbstractCoreIT {
Query query = Query.fromQL( s );
Results r = em.searchCollection( em.getApplicationRef(), "loveobjects", query );
- assertTrue( r.size() == 1 );
+ assertEquals(1, r.size() );
String username = ( String ) ( ( Map ) r.getEntities().get( 0 ).getProperty( "Recipient" ) ).get( "Username" );
// selection results should be a list of lists
[18/25] incubator-usergrid git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-752
Posted by sn...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-752
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9a92fcc8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9a92fcc8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9a92fcc8
Branch: refs/heads/master
Commit: 9a92fcc83963e40ff14c1670d881b0790d9cf643
Parents: b2e1fe6 648b599
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jul 8 11:37:13 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jul 8 11:37:13 2015 -0600
----------------------------------------------------------------------
CHANGELOG | 6 +
LICENSE | 49 ++-
NOTICE | 3 +-
.../css/arsmarquette/ARSMaquettePro-Light.otf | Bin 184600 -> 0 bytes
.../css/arsmarquette/ARSMaquettePro-Medium.otf | Bin 188020 -> 0 bytes
.../css/arsmarquette/ARSMaquettePro-Regular.otf | Bin 188096 -> 0 bytes
portal/css/main.css | 15 -
portal/css/main.min.css | 2 +-
release/release-candidate.sh | 10 -
stack/LICENSE-2.0.txt | 202 ------------
stack/rest/pom.xml | 11 +-
.../applications/assets/AssetResourceIT.java | 2 +-
.../src/test/resources/cat-larger-than-6mb.jpg | Bin 9799257 -> 0 bytes
.../src/test/resources/ship-larger-than-6mb.gif | Bin 0 -> 7407487 bytes
.../org/apache/usergrid/tools/ExportAdmins.java | 328 ++++++++++---------
.../org/apache/usergrid/tools/ImportAdmins.java | 297 ++++++++---------
.../org/apache/usergrid/tools/UserManager.java | 22 ++
stack/tools/src/main/resources/log4j.properties | 6 +-
18 files changed, 401 insertions(+), 552 deletions(-)
----------------------------------------------------------------------
[24/25] incubator-usergrid git commit: Cleans up unused applicationId
Posted by sn...@apache.org.
Cleans up unused applicationId
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fd40f757
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fd40f757
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fd40f757
Branch: refs/heads/master
Commit: fd40f757b901af0963c9c8a973bf0c96e6a920b3
Parents: d1ca419
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Jul 10 10:48:10 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Jul 10 10:48:10 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/cassandra/RelationManagerImpl.java | 2 +-
.../persistence/query/ir/result/ShardFilterIteratorTest.java | 7 -------
2 files changed, 1 insertion(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fd40f757/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index 587f17d..79d3cf2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -221,7 +221,7 @@ public class RelationManagerImpl implements RelationManager {
if ( "location.coordinates".equals( entry.getPath() ) ) {
EntityLocationRef loc = new EntityLocationRef( indexUpdate.getEntity(), entry.getTimestampUuid(),
entry.getValue().toString() );
- batchRemoveLocationFromCollectionIndex( indexUpdate.getBatch(), indexBucketLocator, applicationId,
+ batchRemoveLocationFromCollectionIndex( indexUpdate.getBatch(), indexBucketLocator,
index_name, loc );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fd40f757/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIteratorTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIteratorTest.java
index 6fa686f..648b137 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIteratorTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIteratorTest.java
@@ -52,13 +52,6 @@ public class ShardFilterIteratorTest {
final IndexBucketLocator indexBucketLocator = new SimpleIndexBucketLocatorImpl( 20 );
-
- final UUID applicationId = UUIDUtils.newTimeUUID();
-
-
- final String components = "things";
-
-
final Set<ScanColumn> allColumns = new LinkedHashSet<ScanColumn>( size );
[12/25] incubator-usergrid git commit: Fixes cursor generation for
geo points
Posted by sn...@apache.org.
Fixes cursor generation for geo points
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b462b6ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b462b6ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b462b6ac
Branch: refs/heads/master
Commit: b462b6ac4a88d41578ffca71126bb1dc76ea0dbf
Parents: 595aa71
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jul 1 14:57:22 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jul 1 14:57:22 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/query/ir/result/GeoIterator.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b462b6ac/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
index 20d1f69..6b5f527 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
@@ -290,7 +290,7 @@ public class GeoIterator implements ResultIterator {
@Override
public void addToCursor( final CursorCache cache ) {
-
+ geoCursorGenerator.addToCursor( cache, this );
}
[13/25] incubator-usergrid git commit: Fixes bug with order by in
connections
Posted by sn...@apache.org.
Fixes bug with order by in connections
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/84d77605
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/84d77605
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/84d77605
Branch: refs/heads/master
Commit: 84d77605dd1cc4d5a0b638686a2166c2430a0ca3
Parents: b462b6a
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Jul 2 17:22:34 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Jul 2 17:22:34 2015 -0600
----------------------------------------------------------------------
.../query/ir/result/ConnectionShardFilter.java | 16 +++++++++-------
.../query/ir/result/SearchConnectionVisitor.java | 3 +--
.../query/ir/result/ShardFilterIterator.java | 2 +-
.../persistence/query/AbstractIteratingQueryIT.java | 3 +++
4 files changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d77605/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
index 39f3683..f56dc2b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionShardFilter.java
@@ -48,15 +48,17 @@ public final class ConnectionShardFilter implements ShardFilter {
public boolean isInShard( final ScanColumn scanColumn ) {
- final UUID entityId = scanColumn.getUUID();
- final ConnectionRefImpl hashRef = new ConnectionRefImpl( searchConnection.getConnectingEntityType(), searchConnection.getConnectedEntityId(), searchConnection.getConnectionType(), searchConnection.getConnectingEntityType(), entityId );
+ //shard hashing is currently based on source. this is a placeholder for when this is fixed.
+// UUID[] indexIds = searchConnection.getIndexIds();
+//
+// final String shard = indexBucketLocator.getBucket(indexIds[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE] );
+//
+// return expectedBucket.equals( shard );
- final UUID hashId = hashRef.getConnectionSearchShardId();
+ return true;
+//
+ }
- //not for our current processing shard, discard
- final String shard = indexBucketLocator.getBucket( hashId );
- return expectedBucket.equals( shard );
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d77605/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 5505914..5949ee4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -73,8 +73,7 @@ public class SearchConnectionVisitor extends SearchVisitor {
@Override
protected IndexScanner secondaryIndexScan( QueryNode node, QuerySlice slice ) throws Exception {
- UUID
- id = ConnectionRefImpl.getIndexId( ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE, headEntity,
+ final UUID id = ConnectionRefImpl.getIndexId( ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE, headEntity,
connection.getConnectionType(), connection.getConnectedEntityType(), new ConnectedEntityRef[0] );
Object key = key( id, INDEX_CONNECTIONS );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d77605/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIterator.java
index 3a97fb7..5406e44 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ShardFilterIterator.java
@@ -103,7 +103,7 @@ public class ShardFilterIterator implements ResultIterator {
final Set<ScanColumn> results = new LinkedHashSet<ScanColumn>( );
- while(resultsIterator.hasNext()){
+ while(results.size() < pageSize && resultsIterator.hasNext() ){
final Iterator<ScanColumn> scanColumns = resultsIterator.next().iterator();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d77605/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
index f6b2661..ef8c3a1 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
@@ -372,6 +372,7 @@ public abstract class AbstractIteratingQueryIT {
io.doSetup();
int size = 2000;
+// int size = 3;
int queryLimit = Query.MAX_LIMIT;
// the number of entities that should be written including an intersection
@@ -390,6 +391,8 @@ public abstract class AbstractIteratingQueryIT {
String name = String.valueOf( i );
boolean intersect1 = i % intersectIncrement == 0;
boolean intersect2 = i % secondIncrement == 0;
+// boolean intersect1 = i == 1;
+// boolean intersect2 = i == 1;
entity.put( "name", name );
// if we hit the increment, set this to true
[07/25] incubator-usergrid git commit: Wraps connection all scanners
with shard filters to ensure we only process entities relevant to our shards
Posted by sn...@apache.org.
Wraps connection all scanners with shard filters to ensure we only process entities relevant to our shards
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/aa794884
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/aa794884
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/aa794884
Branch: refs/heads/master
Commit: aa79488419bbd1c6a3ff3bfb8d187fb30bd9e046
Parents: 68c7eae
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jun 24 13:20:22 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Jun 25 11:14:52 2015 -0600
----------------------------------------------------------------------
.../cassandra/RelationManagerImpl.java | 12 +-
.../cassandra/index/ConnectedIndexScanner.java | 11 +-
.../persistence/query/ir/SearchVisitor.java | 2 +-
.../result/ConnectionSearchVisitorFactory.java | 2 +-
.../query/ir/result/GatherIterator.java | 6 -
.../ir/result/SearchConnectionVisitor.java | 10 +-
.../query/ir/result/SliceIterator.java | 25 +++-
.../query/ir/result/SliceShardIterator.java | 118 +++++++++++++++++++
.../query/ir/result/UnionIterator.java | 94 ++++++++++++---
9 files changed, 236 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa794884/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index 977dad0..ebe2aec 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -1910,10 +1910,6 @@ public class RelationManagerImpl implements RelationManager {
ConnectionRefImpl connectionRef =
new ConnectionRefImpl( sourceEntity, connectionType, new SimpleEntityRef( connectedEntityType, null ) );
- // EntityRef connectedEntity) {
- // ConnectionRefImpl connectionRef = new ConnectionRefImpl(new ConnectedEntityRefImpl(connectionType,
- // connectedEntityType, null, true), sourceEntity );
-
final ConnectionResultsLoaderFactory factory = new ConnectionResultsLoaderFactory( connectionRef );
@@ -1921,8 +1917,6 @@ public class RelationManagerImpl implements RelationManager {
ConnectionSearchVisitorFactory collectionSearchVisitorFactory = new ConnectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, true, "" );
-// SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, true );
-
return qp.getResults( collectionSearchVisitorFactory );
}
@@ -1967,8 +1961,6 @@ public class RelationManagerImpl implements RelationManager {
ConnectionSearchVisitorFactory collectionSearchVisitorFactory = new ConnectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, false, "" );
-// SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, false );
-
return qp.getResults( collectionSearchVisitorFactory );
}
@@ -2007,9 +1999,7 @@ public class RelationManagerImpl implements RelationManager {
QueryProcessor qp = new QueryProcessor( query, null, em, factory );
- ConnectionSearchVisitorFactory collectionSearchVisitorFactory = new ConnectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, false, "" );
-
-// SearchConnectionVisitor visitor = new SearchConnectionVisitor( this, qp, connectionRef, true );
+ ConnectionSearchVisitorFactory collectionSearchVisitorFactory = new ConnectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, true, "" );
return qp.getResults( collectionSearchVisitorFactory );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa794884/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
index 6c6152a..4b1a049 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.persistence.cassandra.index;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
@@ -134,14 +135,16 @@ public class ConnectedIndexScanner implements IndexScanner {
//go through each connection type until we exhaust the result sets
while ( currentConnectionType != null ) {
+ final int lastResultsSize = lastResults == null? 0: lastResults.size();
+
//only load a delta size to get this next page
- int selectSize = totalSelectSize - lastResults.size();
+ final int selectSize = totalSelectSize - lastResultsSize;
- Object key = key( entityId, dictionaryType, currentConnectionType );
+ final Object key = key( entityId, dictionaryType, currentConnectionType );
- List<HColumn<ByteBuffer, ByteBuffer>> results =
+ final List<HColumn<ByteBuffer, ByteBuffer>> results =
cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_COMPOSITE_DICTIONARIES, key,
start, null, selectSize, reversed );
@@ -180,7 +183,7 @@ public class ConnectedIndexScanner implements IndexScanner {
if ( hasMore && lastResults !=null && lastResults.size() > 0 ) {
// set the bytebuffer for the next pass
- lastResults.remove( lastResults.size() - 1 );
+ start = lastResults.remove( lastResults.size() - 1 ).getName();
}
return lastResults != null && lastResults.size() > 0;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa794884/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
index fc2df4d..271fc67 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
@@ -160,7 +160,7 @@ public abstract class SearchVisitor implements NodeVisitor {
final int nodeId = node.getId();
- UnionIterator union = new UnionIterator( queryProcessor.getPageSizeHint( node ), nodeId );
+ UnionIterator union = new UnionIterator( queryProcessor.getPageSizeHint( node ), nodeId, queryProcessor.getCursorCache( nodeId ) );
if ( left != null ) {
union.addIterator( left );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa794884/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
index 54eeb01..6a340b9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionSearchVisitorFactory.java
@@ -41,7 +41,7 @@ public class ConnectionSearchVisitorFactory implements SearchVisitorFactory {
private final boolean outgoing;
private final String[] prefix;
-
+ //cass, indexBucketLocator, qp, applicationId, headEntity, connectionRef, false, ""
public ConnectionSearchVisitorFactory( final CassandraService cassandraService,
final IndexBucketLocator indexBucketLocator,
final QueryProcessor queryProcessor, final UUID applicationId,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa794884/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index 5bc91e0..eaae6b2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -171,12 +171,6 @@ public class GatherIterator implements ResultIterator {
results.add(next);
cursorMap.put( next.getUUID(), iterator );
-// //results are too large, trim them
-// if(results.size() > pageSize){
-// final ScanColumn toRemove = results.pollLast();
-// cursorMap.remove( toRemove.getUUID() );
-// }
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa794884/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 6f833db..ff08245 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -53,7 +53,7 @@ public class SearchConnectionVisitor extends SearchVisitor {
* @param queryProcessor They query processor to use
* @param applicationId
* @param connection The connection refernce
- * @param outgoing The direction to search. True if we should search from source->target edges. False if we
+ * @param outgoing The direction to search. True if we should search from source->target edges. False if we are searching target<-source
*/
public SearchConnectionVisitor( final CassandraService cassandraService,
final IndexBucketLocator indexBucketLocator, final QueryProcessor queryProcessor,
@@ -120,6 +120,9 @@ public class SearchConnectionVisitor extends SearchVisitor {
@Override
public void visit( AllNode node ) throws Exception {
+
+ //todo, use a cache for this...
+
QuerySlice slice = node.getSlice();
queryProcessor.applyCursorAndSort( slice );
@@ -179,7 +182,10 @@ public class SearchConnectionVisitor extends SearchVisitor {
new ConnectedIndexScanner( cassandraService, dictionaryType, applicationId, entityIdToUse, connectionTypes,
start, slice.isReversed(), size, skipFirst );
- this.results.push( new SliceIterator( slice, connectionScanner, connectionParser ) );
+ //we have to create our wrapper so validate the data we read is correct for our shard
+ final SliceShardIterator.ShardBucketValidator validator = new SliceShardIterator.ShardBucketValidator(indexBucketLocator, bucket, applicationId, IndexBucketLocator.IndexType.CONNECTION, "" );
+
+ this.results.push( new SliceShardIterator( validator, slice, connectionScanner, connectionParser ));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa794884/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
index a386159..4f5bff5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -46,10 +46,10 @@ public class SliceIterator implements ResultIterator {
private final LinkedHashMap<UUID, ScanColumn> cols;
private final QuerySlice slice;
- private final SliceParser parser;
- private final IndexScanner scanner;
+ protected final SliceParser parser;
+ protected final IndexScanner scanner;
private final int pageSize;
- private final boolean isReversed;
+ protected final boolean isReversed;
/**
@@ -128,9 +128,9 @@ public class SliceIterator implements ResultIterator {
while ( results.hasNext() ) {
- final ByteBuffer colName = results.next().getName().duplicate();
+ final HColumn<ByteBuffer, ByteBuffer> column = results.next();
- final ScanColumn parsed = parser.parse( colName, isReversed );
+ final ScanColumn parsed = parse(column);
//skip this value, the parser has discarded it
if ( parsed == null ) {
@@ -151,6 +151,21 @@ public class SliceIterator implements ResultIterator {
}
+ /**
+ * Parses the column. If the column should be discarded, null should be returned
+ * @param column
+ * @return
+ */
+ protected ScanColumn parse( HColumn<ByteBuffer, ByteBuffer> column){
+
+ final ByteBuffer colName = column.getName().duplicate();
+
+ final ScanColumn parsed = parser.parse( colName, isReversed );
+
+ return parsed;
+ }
+
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa794884/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardIterator.java
new file mode 100644
index 0000000..279ba2c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardIterator.java
@@ -0,0 +1,118 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
+import org.apache.usergrid.persistence.query.ir.QuerySlice;
+
+import me.prettyprint.hector.api.beans.HColumn;
+
+
+/**
+ * An iterator that will check if the parsed column is part of this shard. This is required due to a legacy storage
+ * format
+ *
+ * Connections are not sharded by target entity, as a result, we get data partition mismatches when performing
+ * intersections and seeks. This is meant to discard target entities that are not part of the current shard
+ *
+ * @author tnine
+ */
+public class SliceShardIterator extends SliceIterator {
+
+ private static final Logger logger = LoggerFactory.getLogger( SliceShardIterator.class );
+
+ private final ShardBucketValidator shardBucketValidator;
+
+
+ /**
+ * @param slice The slice used in the scanner
+ * @param scanner The scanner to use to read the cols
+ * @param parser The parser for the scanner results
+ */
+ public SliceShardIterator( final ShardBucketValidator shardBucketValidator, final QuerySlice slice,
+ final IndexScanner scanner, final SliceParser parser ) {
+ super( slice, scanner, parser );
+
+ this.shardBucketValidator = shardBucketValidator;
+ }
+
+
+ /**
+ * Parses the column. If the column should be discarded, null should be returned
+ */
+ protected ScanColumn parse( HColumn<ByteBuffer, ByteBuffer> column ) {
+
+ final ByteBuffer colName = column.getName().duplicate();
+
+ final ScanColumn parsed = parser.parse( colName, isReversed );
+
+ if(parsed == null){
+ return null;
+ }
+
+ final UUID entityId = parsed.getUUID();
+
+
+ //not for our current processing shard, discard
+ if(!shardBucketValidator.isInShard( entityId )){
+ return null;
+ }
+
+ return parsed;
+ }
+
+
+ /**
+ * Class that performs validation on an entity to ensure it's in the shard we expecte
+ */
+ public static final class ShardBucketValidator {
+ private final IndexBucketLocator indexBucketLocator;
+ private final String expectedBucket;
+ private final UUID applicationId;
+ private final IndexBucketLocator.IndexType type;
+ private final String[] components;
+
+
+ public ShardBucketValidator( final IndexBucketLocator indexBucketLocator, final String expectedBucket,
+ final UUID applicationId, final IndexBucketLocator.IndexType type,
+ final String... components ) {
+ this.indexBucketLocator = indexBucketLocator;
+ this.expectedBucket = expectedBucket;
+ this.applicationId = applicationId;
+ this.type = type;
+ this.components = components;
+ }
+
+
+ public boolean isInShard( final UUID entityId ) {
+ //not for our current processing shard, discard
+ final String shard = indexBucketLocator.getBucket( applicationId, type, entityId, components );
+
+ return expectedBucket.equals( shard );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa794884/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
index aca9852..b6e44d8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -28,6 +28,8 @@ import java.util.UUID;
import org.apache.usergrid.persistence.cassandra.CursorCache;
+import com.fasterxml.uuid.UUIDComparator;
+
import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
@@ -131,19 +133,19 @@ public class UnionIterator extends MultiIterator {
private final int maxSize;
- private final List<ScanColumn> list;
+ private final List<UnionScanColumn> list;
-// private UUIDColumn min;
+ private UnionScanColumn min;
public SortedColumnList( final int maxSize, final UUID minUuid ) {
//we need to allocate the extra space if required
- this.list = new ArrayList<ScanColumn>( maxSize );
+ this.list = new ArrayList<UnionScanColumn>( maxSize );
this.maxSize = maxSize;
-// if ( minUuid != null ) {
-// min = new UUIDColumn( minUuid, 1 ) ;
-// }
+ if ( minUuid != null ) {
+ min = new UnionScanColumn(new UUIDColumn( minUuid, 1 )) ;
+ }
}
@@ -151,12 +153,15 @@ public class UnionIterator extends MultiIterator {
* Add the column to this list
*/
public void add( ScanColumn col ) {
+
+ final UnionScanColumn unionScanColumn = new UnionScanColumn(col);
+
//less than our min, don't add
-// if ( min != null && min.compareTo( col ) >= 0 ) {
-// return;
-// }
+ if ( min != null && min.compareTo( unionScanColumn ) >= 0 ) {
+ return;
+ }
- int index = Collections.binarySearch( this.list, col );
+ int index = Collections.binarySearch( this.list, unionScanColumn );
//already present
if ( index > -1 ) {
@@ -170,7 +175,7 @@ public class UnionIterator extends MultiIterator {
return;
}
- this.list.add( index, col );
+ this.list.add( index, unionScanColumn );
final int size = this.list.size();
@@ -214,8 +219,7 @@ public class UnionIterator extends MultiIterator {
return;
}
- final UUID oldMin = this.list.get( size - 1 ).getUUID();
-// min = new UUIDColumn( oldMin, 1 );
+ min = this.list.get( size - 1 );
}
@@ -228,7 +232,69 @@ public class UnionIterator extends MultiIterator {
public void reset(){
clear();
-// this.min = null;
+ this.min = null;
+ }
+ }
+
+ private static class UnionScanColumn implements ScanColumn{
+
+ private final ScanColumn delegate;
+ private ScanColumn child;
+
+
+ private UnionScanColumn( final ScanColumn delegate ) {
+ super();
+ this.delegate = delegate;}
+
+
+ @Override
+ public int compareTo( final ScanColumn o ) {
+ return UUIDComparator.staticCompare( delegate.getUUID(), o.getUUID() );
+ }
+
+
+ @Override
+ public UUID getUUID() {
+ return delegate.getUUID();
+ }
+
+
+ @Override
+ public ByteBuffer getCursorValue() {
+ return ue.toByteBuffer( delegate.getUUID() );
+ }
+
+
+ @Override
+ public void setChild( final ScanColumn childColumn ) {
+ //intentionally a no-op, since child is on the delegate
+ }
+
+
+ @Override
+ public ScanColumn getChild() {
+ return delegate.getChild();
+ }
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( !( o instanceof UnionScanColumn ) ) {
+ return false;
+ }
+
+ final UnionScanColumn that = ( UnionScanColumn ) o;
+
+ return delegate.getUUID().equals( that.delegate.getUUID() );
+ }
+
+
+ @Override
+ public int hashCode() {
+ return delegate.getUUID().hashCode();
}
}
}
[21/25] incubator-usergrid git commit: Adds slice duplicate.
Otherwise cursor concurrency errors occur.
Posted by sn...@apache.org.
Adds slice duplicate. Otherwise cursor concurrency errors occur.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/219a425a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/219a425a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/219a425a
Branch: refs/heads/master
Commit: 219a425a656d33af1c9defbb08b6d73d574dadeb
Parents: d872e03
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jul 8 15:55:12 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Jul 8 15:55:12 2015 -0600
----------------------------------------------------------------------
.../persistence/cassandra/QueryProcessor.java | 2 +-
.../persistence/query/ir/QuerySlice.java | 19 +++++++++++++++++++
.../persistence/query/ir/SearchVisitor.java | 4 ++--
3 files changed, 22 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/219a425a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
index 10f4fa5..538f6d6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
@@ -220,7 +220,7 @@ public class QueryProcessor {
* Apply cursor position and sort order to this slice. This should only be invoke at evaluation time to ensure that
* the IR tree has already been fully constructed
*/
- public void applyCursorAndSort( QuerySlice slice ) {
+ public void applyCursorAndSort( QuerySlice slice ) {
// apply the sort first, since this can change the hash code
SortPredicate sort = getSort( slice.getPropertyName() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/219a425a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/QuerySlice.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/QuerySlice.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/QuerySlice.java
index 77c0a6b..2b7d23c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/QuerySlice.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/QuerySlice.java
@@ -53,6 +53,25 @@ public class QuerySlice {
}
+
+ /**
+ * Create a deep copy of the query slice from the original query slice
+ * @param original
+ */
+ private QuerySlice(final QuerySlice original){
+ this.propertyName = original.propertyName;
+ this.nodeId = original.nodeId;
+ this.start = original.start;
+ this.finish = original.finish;
+ this.cursor = original.cursor;
+ this.reversed = original.reversed;
+ }
+
+
+ public QuerySlice duplicate(){
+ return new QuerySlice( this );
+ }
+
/** Reverse this slice. Flips the reversed switch and correctly changes the start and finish */
public void reverse() {
reversed = !reversed;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/219a425a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
index 2fe2e37..8c5fe2b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
@@ -229,7 +229,7 @@ public abstract class SearchVisitor implements NodeVisitor {
scanner = new NoOpIndexScanner();
}
else {
- scanner = secondaryIndexScan( orderByNode, slice );
+ scanner = secondaryIndexScan( orderByNode, slice.duplicate() );
}
final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( slice );
@@ -264,7 +264,7 @@ public abstract class SearchVisitor implements NodeVisitor {
IntersectionIterator intersections = new IntersectionIterator( queryProcessor.getPageSizeHint( node ) );
for ( QuerySlice slice : node.getAllSlices() ) {
- IndexScanner scanner = secondaryIndexScan( node, slice );
+ IndexScanner scanner = secondaryIndexScan( node, slice.duplicate() );
final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( slice );
[09/25] incubator-usergrid git commit: Finished cursor generation
refactor. Added gets for slice shard filter iterator.
Posted by sn...@apache.org.
Finished cursor generation refactor. Added gets for slice shard filter iterator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/efab0b93
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/efab0b93
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/efab0b93
Branch: refs/heads/master
Commit: efab0b935cfc60b44b7d0c79233367a55611dc84
Parents: f087924
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jun 30 15:13:04 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jun 30 15:13:04 2015 -0600
----------------------------------------------------------------------
.../persistence/cassandra/QueryProcessor.java | 3 +-
.../persistence/query/ir/SearchVisitor.java | 16 +-
.../query/ir/result/AbstractScanColumn.java | 12 +-
.../ir/result/ConnectionIndexSliceParser.java | 16 +-
.../query/ir/result/CursorGenerator.java | 36 +++++
.../query/ir/result/EmptyIterator.java | 5 -
.../query/ir/result/GatherIterator.java | 18 ---
.../query/ir/result/GeoIterator.java | 131 +++++++++-------
.../query/ir/result/IntersectionIterator.java | 35 +++--
.../query/ir/result/OrderByIterator.java | 26 ++--
.../query/ir/result/ResultIterator.java | 4 +-
.../persistence/query/ir/result/ScanColumn.java | 9 ++
.../ir/result/SearchCollectionVisitor.java | 11 +-
.../ir/result/SearchConnectionVisitor.java | 5 +-
.../ir/result/SecondaryIndexSliceParser.java | 13 +-
.../query/ir/result/SliceCursorGenerator.java | 55 +++++++
.../query/ir/result/SliceIterator.java | 57 -------
.../ir/result/SliceShardFilterIterator.java | 5 -
.../query/ir/result/StaticIdIterator.java | 7 +-
.../query/ir/result/SubtractionIterator.java | 20 +--
.../persistence/query/ir/result/UUIDColumn.java | 9 +-
.../query/ir/result/UUIDCursorGenerator.java | 49 ++++++
.../query/ir/result/UUIDIndexSliceParser.java | 9 +-
.../query/ir/result/UnionIterator.java | 51 +++---
.../query/AbstractIteratingQueryIT.java | 3 +-
.../query/ir/result/AbstractScanColumnTest.java | 4 +-
.../query/ir/result/InOrderIterator.java | 9 +-
.../query/ir/result/IteratorHelper.java | 4 +-
.../ir/result/SliceShardFilterIteratorTest.java | 155 +++++++++++++++++++
.../apache/usergrid/tools/EntityCleanup.java | 2 +-
.../usergrid/tools/UniqueIndexCleanup.java | 2 +-
31 files changed, 523 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
index ebaacf4..32fc07c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
@@ -285,8 +285,9 @@ public class QueryProcessor {
int resultSize = Math.min( entityIds.size(), size );
entityIds = entityIds.subList( 0, resultSize );
+ //set our cursor on the last results
if ( resultSize == size ) {
- itr.finalizeCursor( resultsCursor, entityIds.get( resultSize - 1 ).getUUID() );
+ entityIds.get( resultSize -1 ).addToCursor( resultsCursor );
}
}
if ( logger.isDebugEnabled() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
index 271fc67..fdeca6d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
@@ -33,6 +33,7 @@ import org.apache.usergrid.persistence.query.ir.result.IntersectionIterator;
import org.apache.usergrid.persistence.query.ir.result.OrderByIterator;
import org.apache.usergrid.persistence.query.ir.result.ResultIterator;
import org.apache.usergrid.persistence.query.ir.result.SecondaryIndexSliceParser;
+import org.apache.usergrid.persistence.query.ir.result.SliceCursorGenerator;
import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
import org.apache.usergrid.persistence.query.ir.result.StaticIdIterator;
import org.apache.usergrid.persistence.query.ir.result.SubtractionIterator;
@@ -208,8 +209,11 @@ public abstract class SearchVisitor implements NodeVisitor {
//only order by with no query, start scanning the first field
if ( subResults == null ) {
QuerySlice firstFieldSlice = new QuerySlice( slice.getPropertyName(), -1 );
+
+ final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( firstFieldSlice );
+
subResults =
- new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), new SecondaryIndexSliceParser() );
+ new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), new SecondaryIndexSliceParser( sliceCursorGenerator ) );
}
orderIterator = new OrderByIterator( slice, orderByNode.getSecondarySorts(), subResults, em,
@@ -228,7 +232,10 @@ public abstract class SearchVisitor implements NodeVisitor {
scanner = secondaryIndexScan( orderByNode, slice );
}
- SliceIterator joinSlice = new SliceIterator( slice, scanner, new SecondaryIndexSliceParser());
+ final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( slice );
+
+ SliceIterator joinSlice = new SliceIterator( slice, scanner, new SecondaryIndexSliceParser(
+ sliceCursorGenerator ));
IntersectionIterator union = new IntersectionIterator( queryProcessor.getPageSizeHint( orderByNode ) );
union.addIterator( joinSlice );
@@ -259,7 +266,10 @@ public abstract class SearchVisitor implements NodeVisitor {
for ( QuerySlice slice : node.getAllSlices() ) {
IndexScanner scanner = secondaryIndexScan( node, slice );
- intersections.addIterator( new SliceIterator( slice, scanner, new SecondaryIndexSliceParser()) );
+ final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( slice );
+
+ intersections.addIterator( new SliceIterator( slice, scanner, new SecondaryIndexSliceParser(
+ sliceCursorGenerator )) );
}
results.push( intersections );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
index 2a359be..35672d9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
@@ -22,6 +22,8 @@ import java.util.UUID;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.usergrid.persistence.cassandra.CursorCache;
+
/**
*
@@ -32,12 +34,15 @@ public abstract class AbstractScanColumn implements ScanColumn {
protected final UUID uuid;
protected final ByteBuffer buffer;
+ protected final CursorGenerator sliceCursorGenerator;
protected ScanColumn child;
- protected AbstractScanColumn( final UUID uuid, final ByteBuffer columnNameBuffer ) {
+ protected AbstractScanColumn( final UUID uuid, final ByteBuffer columnNameBuffer,
+ final CursorGenerator sliceCursorGenerator ) {
this.uuid = uuid;
this.buffer = columnNameBuffer;
+ this.sliceCursorGenerator = sliceCursorGenerator;
}
@@ -94,4 +99,9 @@ public abstract class AbstractScanColumn implements ScanColumn {
return child;
}
+
+ @Override
+ public void addToCursor( final CursorCache cache ) {
+ this.sliceCursorGenerator.addToCursor( cache, this );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
index a669d8c..1a3f44d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
@@ -21,7 +21,6 @@ import java.nio.ByteBuffer;
import java.util.UUID;
import org.apache.usergrid.persistence.Schema;
-import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
import com.fasterxml.uuid.UUIDComparator;
@@ -36,11 +35,15 @@ import me.prettyprint.hector.api.beans.DynamicComposite;
public class ConnectionIndexSliceParser implements SliceParser {
private final String connectedEntityType;
+ private final SliceCursorGenerator sliceCurosrGenerator;
- /** @param connectedEntityType Could be null if we want to return all types */
- public ConnectionIndexSliceParser( String connectedEntityType ) {
+ /**
+ * @param connectedEntityType Could be null if we want to return all types
+ * @param sliceCurosrGenerator */
+ public ConnectionIndexSliceParser( String connectedEntityType, final SliceCursorGenerator sliceCurosrGenerator ) {
this.connectedEntityType = connectedEntityType;
+ this.sliceCurosrGenerator = sliceCurosrGenerator;
}
@@ -65,7 +68,7 @@ public class ConnectionIndexSliceParser implements SliceParser {
return null;
}
- return new ConnectionColumn( ( UUID ) composite.get( 0 ), connectedType, buff );
+ return new ConnectionColumn( ( UUID ) composite.get( 0 ), connectedType, buff, sliceCurosrGenerator );
// return composite;
// return null;
}
@@ -78,8 +81,9 @@ public class ConnectionIndexSliceParser implements SliceParser {
private final String connectedType;
- public ConnectionColumn( UUID uuid, String connectedType, ByteBuffer column) {
- super( uuid, column );
+ public ConnectionColumn( UUID uuid, String connectedType, ByteBuffer column,
+ final SliceCursorGenerator sliceCursorGenerator ) {
+ super( uuid, column, sliceCursorGenerator );
this.connectedType = connectedType;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CursorGenerator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CursorGenerator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CursorGenerator.java
new file mode 100644
index 0000000..b128cea
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CursorGenerator.java
@@ -0,0 +1,36 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import org.apache.usergrid.persistence.cassandra.CursorCache;
+
+
+/**
+ * Interface to generate a cursor value from the specified column
+ */
+public interface CursorGenerator<T extends ScanColumn> {
+
+ /**
+ * Use the generator to add this value to the cursor cache
+ * @param cache
+ */
+ void addToCursor( final CursorCache cache, final T column );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/EmptyIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/EmptyIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/EmptyIterator.java
index 933bd2a..d4adbd3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/EmptyIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/EmptyIterator.java
@@ -32,11 +32,6 @@ public class EmptyIterator implements ResultIterator {
}
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastValue ) {
- //no op
- }
-
@Override
public Iterator<Set<ScanColumn>> iterator() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index eaae6b2..9d2e4da 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -31,14 +31,12 @@ public class GatherIterator implements ResultIterator {
private Iterator<ScanColumn> mergedIterators;
- private Map<UUID, ResultIterator> cursorMap;
private List<ResultIterator> iterators;
public GatherIterator(final int pageSize, final QueryNode rootNode, final Collection<SearchVisitor> searchVisitors) {
this.pageSize = pageSize;
this.rootNode = rootNode;
- this.cursorMap = new HashMap<UUID, ResultIterator>( pageSize );
createIterators( searchVisitors );
}
@@ -49,19 +47,6 @@ public class GatherIterator implements ResultIterator {
}
- @Override
- public void finalizeCursor( final CursorCache cache, final UUID lastValue ) {
- //find the last value in the tree, and return it's cursor
- final ResultIterator sourceIterator = cursorMap.get( lastValue );
-
- if(sourceIterator == null){
- throw new IllegalArgumentException( "Could not find the iterator that provided the candidate with uuid " + lastValue );
- }
-
- //delegate to the source iterator
- sourceIterator.finalizeCursor( cache, lastValue );
- }
-
@Override
public Iterator<Set<ScanColumn>> iterator() {
@@ -133,8 +118,6 @@ public class GatherIterator implements ResultIterator {
private void mergeIterators(){
//TODO make this concurrent
- //clear the cursor map
- cursorMap.clear();
TreeSet<ScanColumn> merged = new TreeSet<ScanColumn>( );
@@ -169,7 +152,6 @@ public class GatherIterator implements ResultIterator {
final ScanColumn next = nextPage.next();
results.add(next);
- cursorMap.put( next.getUUID(), iterator );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
index 587bb49..20d1f69 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
@@ -140,11 +140,13 @@ public class GeoIterator implements ResultIterator {
lastCellsSearched = results.lastSearchedGeoCells;
+ final GeoCursorGenerator cursorGenerator = new GeoCursorGenerator( slice, lastCellsSearched );
+
for (final EntityLocationRef location : locations) {
final UUID id = location.getUuid();
- final LocationScanColumn locationScan = new LocationScanColumn(location);
+ final LocationScanColumn locationScan = new LocationScanColumn(location, cursorGenerator );
idOrder.put(id, locationScan);
lastLoaded.add(locationScan);
@@ -212,60 +214,6 @@ public class GeoIterator implements ResultIterator {
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(
- * org.apache.usergrid.persistence.cassandra.CursorCache, java.util.UUID)
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID uuid ) {
-
- LocationScanColumn col = idOrder.get( uuid );
-
- if ( col == null ) {
- throw new IllegalArgumentException( "Could not generate cursor for column because column could not be found" );
- }
-
- final EntityLocationRef location = col.location;
-
- if ( location == null ) {
- return;
- }
-
- final int sliceHash = slice.hashCode();
-
- // get our next distance
- final double latitude = location.getLatitude();
-
- final double longitude = location.getLongitude();
-
- // now create a string value for this
- final StringBuilder builder = new StringBuilder();
-
- builder.append( uuid ).append( DELIM );
- builder.append( latitude ).append( DELIM );
- builder.append( longitude );
-
- if ( lastCellsSearched != null ) {
- builder.append( DELIM );
-
- for ( String geoCell : lastCellsSearched ) {
- builder.append( geoCell ).append( TILE_DELIM );
- }
-
- int length = builder.length();
-
- builder.delete( length - TILE_DELIM.length() - 1, length );
- }
-
- ByteBuffer buff = se.toByteBuffer( builder.toString() );
-
-
- cache.setNextCursor( sliceHash, buff );
- }
-
/** Get the last cells searched in the iteraton */
public List<String> getLastCellsSearched() {
@@ -310,11 +258,14 @@ public class GeoIterator implements ResultIterator {
private class LocationScanColumn implements ScanColumn {
private final EntityLocationRef location;
+ private final GeoCursorGenerator geoCursorGenerator;
private ScanColumn child;
- public LocationScanColumn( EntityLocationRef location ) {
+
+ public LocationScanColumn( EntityLocationRef location, final GeoCursorGenerator geoCursorGenerator ) {
this.location = location;
+ this.geoCursorGenerator = geoCursorGenerator;
}
@@ -338,6 +289,12 @@ public class GeoIterator implements ResultIterator {
@Override
+ public void addToCursor( final CursorCache cache ) {
+
+ }
+
+
+ @Override
public ScanColumn getChild() {
return this.child;
}
@@ -385,4 +342,66 @@ public class GeoIterator implements ResultIterator {
return locationCompare;
}
}
+
+ private static final class GeoCursorGenerator implements CursorGenerator<LocationScanColumn>{
+
+ private final QuerySlice slice;
+ private final List<String> lastCellsSearched;
+
+
+ private GeoCursorGenerator( final QuerySlice slice, final List<String> lastCellsSearched ) {
+ this.slice = slice;
+ this.lastCellsSearched = lastCellsSearched;
+ }
+
+
+ @Override
+ public void addToCursor( final CursorCache cache, final LocationScanColumn col ) {
+
+
+
+ if ( col == null ) {
+ throw new IllegalArgumentException( "Could not generate cursor for column because column could not be found" );
+ }
+
+ final EntityLocationRef location = col.location;
+
+ if ( location == null ) {
+ return;
+ }
+
+ final int sliceHash = slice.hashCode();
+
+ // get our next distance
+ final double latitude = location.getLatitude();
+
+ final double longitude = location.getLongitude();
+
+ // now create a string value for this
+ final StringBuilder builder = new StringBuilder();
+
+ builder.append( col.getUUID() ).append( DELIM );
+ builder.append( latitude ).append( DELIM );
+ builder.append( longitude );
+
+ if ( lastCellsSearched != null ) {
+ builder.append( DELIM );
+
+ for ( String geoCell : lastCellsSearched ) {
+ builder.append( geoCell ).append( TILE_DELIM );
+ }
+
+ int length = builder.length();
+
+ builder.delete( length - TILE_DELIM.length() - 1, length );
+ }
+
+ ByteBuffer buff = se.toByteBuffer( builder.toString() );
+
+
+ cache.setNextCursor( sliceHash, buff );
+
+ }
+ }
}
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/IntersectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/IntersectionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/IntersectionIterator.java
index 1fea546..5f5767f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/IntersectionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/IntersectionIterator.java
@@ -150,21 +150,22 @@ public class IntersectionIterator extends MultiIterator {
return results;
}
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(
- * org.apache.usergrid.persistence.cassandra.CursorCache)
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
- ResultIterator itr = iterators.get( 0 );
-
- //We can only create a cursor on our root level value in the intersection iterator.
- if ( itr != null ) {
- itr.finalizeCursor( cache, lastLoaded );
- }
- }
+ //TODO, replace columns with slice parser here
+
+// /*
+// * (non-Javadoc)
+// *
+// * @see
+// * org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(
+// * org.apache.usergrid.persistence.cassandra.CursorCache)
+// */
+// @Override
+// public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
+// ResultIterator itr = iterators.get( 0 );
+//
+// //We can only create a cursor on our root level value in the intersection iterator.
+// if ( itr != null ) {
+// itr.finalizeCursor( cache, lastLoaded );
+// }
+// }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
index 49847c9..6d05bd6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
@@ -134,19 +134,19 @@ public class OrderByIterator extends MergeIterator {
// no op
}
-
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastValue ) {
- int sliceHash = slice.hashCode();
-
- ByteBuffer bytes = ue.toByteBuffer( lastValue );
-
- if ( bytes == null ) {
- return;
- }
-
- cache.setNextCursor( sliceHash, bytes );
- }
+//
+// @Override
+// public void finalizeCursor( CursorCache cache, UUID lastValue ) {
+// int sliceHash = slice.hashCode();
+//
+// ByteBuffer bytes = ue.toByteBuffer( lastValue );
+//
+// if ( bytes == null ) {
+// return;
+// }
+//
+// cache.setNextCursor( sliceHash, bytes );
+// }
/** A Sorted set with a max size. When a new entry is added, the max is removed */
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ResultIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ResultIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ResultIterator.java
index 01a048c..f62ccf4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ResultIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ResultIterator.java
@@ -37,8 +37,6 @@ public interface ResultIterator extends Iterable<Set<ScanColumn>>, Iterator<Set<
/** Reset this iterator to the start to begin iterating again */
- public void reset();
+ void reset();
- /** Finalize the cursor for this results. Pass in the uuid of the last entity loaded. */
- public void finalizeCursor( CursorCache cache, UUID lastValue );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
index 18277e4..577d34a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.nio.ByteBuffer;
import java.util.UUID;
+import org.apache.usergrid.persistence.cassandra.CursorCache;
import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
@@ -40,6 +41,14 @@ public interface ScanColumn extends Comparable<ScanColumn> {
*/
void setChild( final ScanColumn childColumn );
+
+ /**
+ * Use the generator to add this value to the cursor cache
+ * @param cache
+ */
+ void addToCursor( final CursorCache cache );
+
+
/**
* Returns the childl column if present, can return null
* @return
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index a3fe116..3dfce2c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -36,8 +36,6 @@ import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtil
public class SearchCollectionVisitor extends SearchVisitor {
- private static final UUIDIndexSliceParser UUID_PARSER = new UUIDIndexSliceParser();
-
private final CollectionInfo collection;
@@ -95,8 +93,13 @@ public class SearchCollectionVisitor extends SearchVisitor {
UUID startId = null;
+ final UUIDCursorGenerator uuidCursorGenerator = new UUIDCursorGenerator( slice.hashCode() );
+ final UUIDIndexSliceParser uuidIndexSliceParser = new UUIDIndexSliceParser( uuidCursorGenerator );
+
+
if ( slice.hasCursor() ) {
- startId = UUID_PARSER.parse( slice.getCursor(), false ).getUUID();
+
+ startId = uuidIndexSliceParser.parse( slice.getCursor(), false ).getUUID();
}
@@ -105,7 +108,7 @@ public class SearchCollectionVisitor extends SearchVisitor {
queryProcessor.getPageSizeHint( node ), query.isReversed(), bucket, applicationId,
node.isForceKeepFirst() );
- this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER ) );
+ this.results.push( new SliceIterator( slice, indexScanner, uuidIndexSliceParser ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 7e7ddf6..3923bf4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -17,7 +17,6 @@ import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeStartToBy
import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
-import org.apache.usergrid.persistence.geo.CollectionGeoSearch;
import org.apache.usergrid.persistence.geo.ConnectionGeoSearch;
import org.apache.usergrid.persistence.geo.model.Point;
import org.apache.usergrid.persistence.query.ir.AllNode;
@@ -171,8 +170,10 @@ public class SearchConnectionVisitor extends SearchVisitor {
final String connectionType = connection.getConnectionType();
+ final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( slice );
- final ConnectionIndexSliceParser connectionParser = new ConnectionIndexSliceParser( targetType );
+ final ConnectionIndexSliceParser connectionParser = new ConnectionIndexSliceParser( targetType,
+ sliceCursorGenerator );
final Iterator<String> connectionTypes;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
index b26f684..d309c3d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
@@ -45,8 +45,12 @@ public class SecondaryIndexSliceParser implements SliceParser {
//the type comparator
private Comparator<SecondaryIndexColumn> typeComparator;
+ private final SliceCursorGenerator sliceCursorGenerator;
- public SecondaryIndexSliceParser() {}
+
+ public SecondaryIndexSliceParser( final SliceCursorGenerator sliceCursorGenerator ) {
+ this.sliceCursorGenerator = sliceCursorGenerator;
+ }
/* (non-Javadoc)
@@ -63,7 +67,7 @@ public class SecondaryIndexSliceParser implements SliceParser {
typeComparator = getTypeComparator( value, isReversed );
}
- return new SecondaryIndexColumn( uuid, value, buff, typeComparator );
+ return new SecondaryIndexColumn( uuid, value, buff, typeComparator, sliceCursorGenerator );
}
@@ -95,8 +99,9 @@ public class SecondaryIndexSliceParser implements SliceParser {
* @param valueComparator The comparator for the values
*/
public SecondaryIndexColumn( final UUID uuid, final Object value, final ByteBuffer columnNameBuffer,
- final Comparator<SecondaryIndexColumn> valueComparator ) {
- super( uuid, columnNameBuffer );
+ final Comparator<SecondaryIndexColumn> valueComparator,
+ final SliceCursorGenerator sliceCursorGenerator) {
+ super( uuid, columnNameBuffer, sliceCursorGenerator );
this.value = value;
this.valueComparator = valueComparator;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
new file mode 100644
index 0000000..cad0a0f
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
@@ -0,0 +1,55 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.nio.ByteBuffer;
+
+import org.apache.usergrid.persistence.cassandra.CursorCache;
+import org.apache.usergrid.persistence.query.ir.QuerySlice;
+
+
+/**
+ * A cursor generator for the specified query slice
+ */
+public class SliceCursorGenerator implements CursorGenerator<ScanColumn> {
+
+ private final QuerySlice slice;
+
+
+ public SliceCursorGenerator( final QuerySlice slice ) {this.slice = slice;}
+
+
+ @Override
+ public void addToCursor( final CursorCache cache, final ScanColumn col ) {
+
+ if ( col == null ) {
+ return;
+ }
+
+ final int sliceHash = slice.hashCode();
+
+
+ ByteBuffer bytes = col.getCursorValue();
+
+
+ cache.setNextCursor( sliceHash, bytes );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
index 3f9e86c..73cecd7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -44,7 +44,6 @@ public class SliceIterator implements ResultIterator {
private static final Logger logger = LoggerFactory.getLogger( SliceIterator.class );
- private final LinkedHashMap<UUID, ScanColumn> cols;
private final QuerySlice slice;
protected final SliceParser parser;
protected final IndexScanner scanner;
@@ -83,7 +82,6 @@ public class SliceIterator implements ResultIterator {
this.parser = parser;
this.scanner = scanner;
this.pageSize = scanner.getPageSize();
- this.cols = new LinkedHashMap<UUID, ScanColumn>( this.pageSize );
this.parsedCols = new LinkedHashSet<ScanColumn>( this.pageSize );
this.isReversed = scanner.isReversed();
}
@@ -122,8 +120,6 @@ public class SliceIterator implements ResultIterator {
Iterator<HColumn<ByteBuffer, ByteBuffer>> results = scanner.next().iterator();
- cols.clear();
-
parsedCols.clear();
while ( results.hasNext() ) {
@@ -141,7 +137,6 @@ public class SliceIterator implements ResultIterator {
}
last = parsed;
- cols.put( parsed.getUUID(), parsed );
parsedCols.add( parsed );
}
@@ -196,56 +191,4 @@ public class SliceIterator implements ResultIterator {
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor()
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
- final int sliceHash = slice.hashCode();
-
- ByteBuffer bytes = null;
-
- ScanColumn col = cols.get( lastLoaded );
-
-
- //the column came from the current page
- if ( col != null ) {
- bytes = col.getCursorValue();
- }
- else {
-
- //check if we reached the end of our iterator. If we did, set the last value into the cursor. Otherwise
- //this is a bug
- if ( scanner.hasNext() ) {
- logger.error(
- "An iterator attempted to access a slice that was not iterated over. This will result in the"
- + " cursor construction failing" );
- throw new QueryIterationException(
- "An iterator attempted to access a slice that was not iterated over. This will result in the"
- + " cursor construction failing" );
- }
-
- final ByteBuffer sliceCursor = slice.getCursor();
-
- //we've never loaded anything, just re-use the existing slice
- if ( last == null && sliceCursor != null ) {
- bytes = sliceCursor;
- }
-
- //use the last column we loaded. This way our scan returns nothing next time since start == finish
- else if ( last != null ) {
- bytes = last.getCursorValue();
- }
- }
-
-
- if ( bytes == null ) {
- return;
- }
-
- cache.setNextCursor( sliceHash, bytes );
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
index fb0d295..1664627 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
@@ -72,11 +72,6 @@ public class SliceShardFilterIterator implements ResultIterator {
}
- @Override
- public void finalizeCursor( final CursorCache cache, final UUID lastValue ) {
- resultsIterator.finalizeCursor( cache, lastValue );
- }
-
@Override
public Iterator<Set<ScanColumn>> iterator() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
index 1ed6949..4c28bf3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
@@ -37,7 +37,7 @@ public class StaticIdIterator implements ResultIterator {
*
*/
public StaticIdIterator( UUID id ) {
- final ScanColumn col = new UUIDColumn( id, 1 );
+ final ScanColumn col = new UUIDColumn( id, 1, new UUIDCursorGenerator( -1 ) );
ids = Collections.singleton( col );
}
@@ -49,11 +49,6 @@ public class StaticIdIterator implements ResultIterator {
}
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastValue ) {
- //no cursor, it's a static list
- }
-
@Override
public Iterator<Set<ScanColumn>> iterator() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SubtractionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SubtractionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SubtractionIterator.java
index 9cfcf0a..001ac27 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SubtractionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SubtractionIterator.java
@@ -100,14 +100,14 @@ public class SubtractionIterator extends MergeIterator {
}
- /* (non-Javadoc)
- * @see org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(org.apache.usergrid.persistence.cassandra
- * .CursorCache)
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
- //we can only keep a cursor on our keep result set, we must subtract from every page of keep when loading
- // results
- keepIterator.finalizeCursor( cache, lastLoaded );
- }
+// /* (non-Javadoc)
+// * @see org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(org.apache.usergrid.persistence.cassandra
+// * .CursorCache)
+// */
+// @Override
+// public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
+// //we can only keep a cursor on our keep result set, we must subtract from every page of keep when loading
+// // results
+// keepIterator.finalizeCursor( cache, lastLoaded );
+// }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
index 640b391..264a9b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
@@ -15,14 +15,9 @@ class UUIDColumn extends AbstractScanColumn{
private final int compareReversed;
- protected UUIDColumn( final UUID uuid, final ByteBuffer columnNameBuffer, final int compareReversed ) {
- super( uuid, columnNameBuffer );
- this.compareReversed = compareReversed;
- }
-
- public UUIDColumn( final UUID uuid, final int compareReversed ) {
- super(uuid, Serializers.ue.toByteBuffer( uuid ));
+ public UUIDColumn( final UUID uuid, final int compareReversed, final CursorGenerator<UUIDColumn> sliceCursorGenerator ) {
+ super(uuid, Serializers.ue.toByteBuffer( uuid ), sliceCursorGenerator );
this.compareReversed = compareReversed;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDCursorGenerator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDCursorGenerator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDCursorGenerator.java
new file mode 100644
index 0000000..236db05
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDCursorGenerator.java
@@ -0,0 +1,49 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.cassandra.CursorCache;
+
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+
+
+/**
+ * Cursor generator that takes a uuid column and
+ */
+public class UUIDCursorGenerator<T extends ScanColumn> implements CursorGenerator<T> {
+
+ private final int id;
+
+
+ public UUIDCursorGenerator( final int id ) {this.id = id;}
+
+
+ @Override
+ public void addToCursor( final CursorCache cache, final T column ) {
+
+ final UUID uuid = column.getUUID();
+ final ByteBuffer buff = ue.toByteBuffer( uuid );
+ cache.setNextCursor( id, buff );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
index 2c5b1ba..de80b0b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
@@ -29,10 +29,17 @@ import static org.apache.usergrid.persistence.cassandra.Serializers.*;
public class UUIDIndexSliceParser implements SliceParser {
+ private final CursorGenerator<UUIDColumn> uuidCursorGenerator;
+
+
+ public UUIDIndexSliceParser( final UUIDCursorGenerator<UUIDColumn> uuidCursorGenerator ) {
+ this.uuidCursorGenerator = uuidCursorGenerator;
+ }
+
@Override
public ScanColumn parse( final ByteBuffer columnNameBytes, final boolean isReversed ) {
final int compare = isReversed? -1: 1;
- return new UUIDColumn( ue.fromByteBuffer( columnNameBytes.duplicate() ), columnNameBytes, compare );
+ return new UUIDColumn( ue.fromByteBuffer( columnNameBytes.duplicate() ), compare, uuidCursorGenerator);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
index b6e44d8..58e0e9a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -41,8 +41,6 @@ public class UnionIterator extends MultiIterator {
private SortedColumnList list;
- private final int id;
-
/**
* @param pageSize The page size to return
@@ -51,16 +49,17 @@ public class UnionIterator extends MultiIterator {
*/
public UnionIterator( int pageSize, int id, ByteBuffer minUuid ) {
super( pageSize );
-
- this.id = id;
-
UUID parseMinUuid = null;
if(minUuid != null) {
parseMinUuid = ue.fromByteBuffer( minUuid );
}
- list = new SortedColumnList( pageSize, parseMinUuid );
+ final UUIDCursorGenerator uuidCursorGenerator = new UUIDCursorGenerator( id );
+
+ list = new SortedColumnList( pageSize, parseMinUuid, uuidCursorGenerator );
+
+
}
@@ -98,21 +97,6 @@ public class UnionIterator extends MultiIterator {
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(
- * org.apache.usergrid.persistence.cassandra.CursorCache)
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
-
- ByteBuffer buff = ue.toByteBuffer( lastLoaded );
- cache.setNextCursor( id, buff );
- //get our scan column and put them in the cache
- //we finalize the cursor of the min
- }
@Override
@@ -135,16 +119,21 @@ public class UnionIterator extends MultiIterator {
private final List<UnionScanColumn> list;
+ private final UUIDCursorGenerator uuidCursorGenerator;
+
private UnionScanColumn min;
- public SortedColumnList( final int maxSize, final UUID minUuid ) {
+ public SortedColumnList( final int maxSize, final UUID minUuid, final UUIDCursorGenerator uuidCursorGenerator ) {
+
+
+ this.uuidCursorGenerator = uuidCursorGenerator;
//we need to allocate the extra space if required
this.list = new ArrayList<UnionScanColumn>( maxSize );
this.maxSize = maxSize;
if ( minUuid != null ) {
- min = new UnionScanColumn(new UUIDColumn( minUuid, 1 )) ;
+ min = new UnionScanColumn(new UUIDColumn( minUuid, 1, uuidCursorGenerator), uuidCursorGenerator ) ;
}
}
@@ -154,7 +143,7 @@ public class UnionIterator extends MultiIterator {
*/
public void add( ScanColumn col ) {
- final UnionScanColumn unionScanColumn = new UnionScanColumn(col);
+ final UnionScanColumn unionScanColumn = new UnionScanColumn(col, uuidCursorGenerator );
//less than our min, don't add
if ( min != null && min.compareTo( unionScanColumn ) >= 0 ) {
@@ -239,12 +228,16 @@ public class UnionIterator extends MultiIterator {
private static class UnionScanColumn implements ScanColumn{
private final ScanColumn delegate;
+ private final UUIDCursorGenerator uuidCursorGenerator;
private ScanColumn child;
- private UnionScanColumn( final ScanColumn delegate ) {
+
+ private UnionScanColumn( final ScanColumn delegate, final UUIDCursorGenerator uuidCursorGenerator ) {
super();
- this.delegate = delegate;}
+ this.delegate = delegate;
+ this.uuidCursorGenerator = uuidCursorGenerator;
+ }
@Override
@@ -272,6 +265,12 @@ public class UnionIterator extends MultiIterator {
@Override
+ public void addToCursor( final CursorCache cache ) {
+ this.uuidCursorGenerator.addToCursor( cache, this );
+ }
+
+
+ @Override
public ScanColumn getChild() {
return delegate.getChild();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
index f6b2661..1d0d0cb 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
@@ -371,7 +371,8 @@ public abstract class AbstractIteratingQueryIT {
io.doSetup();
- int size = 2000;
+// int size = 2000;
+ int size = 2;
int queryLimit = Query.MAX_LIMIT;
// the number of entities that should be written including an intersection
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
index de46836..c7b3c49 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
@@ -21,6 +21,8 @@ import java.nio.ByteBuffer;
import java.util.UUID;
import org.junit.Test;
+
+import org.apache.usergrid.persistence.query.ir.QuerySlice;
import org.apache.usergrid.utils.UUIDUtils;
import static junit.framework.Assert.assertNull;
@@ -94,7 +96,7 @@ public class AbstractScanColumnTest {
private class TestScanColumn extends AbstractScanColumn {
protected TestScanColumn( final UUID uuid, final ByteBuffer buffer ) {
- super( uuid, buffer );
+ super( uuid, buffer, new SliceCursorGenerator( new QuerySlice( "foo", 1 ) ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
index b4cd906..87ddfe7 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
@@ -51,7 +51,7 @@ public class InOrderIterator implements ResultIterator {
/** Add a uuid to the list */
public void add( UUID... ids ) {
for ( UUID current : ids ) {
- uuids.add( new UUIDColumn( current, 1 ) );
+ uuids.add( new UUIDColumn( current, 1, new UUIDCursorGenerator<UUIDColumn>( 1 ) ) );
}
}
@@ -120,12 +120,5 @@ public class InOrderIterator implements ResultIterator {
}
- /* (non-Javadoc)
- * @see org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(org.apache.usergrid.persistence.cassandra
- * .CursorCache)
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
index 13ad542..f75b495 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
@@ -28,7 +28,9 @@ import java.util.UUID;
*/
public class IteratorHelper {
+ private static final UUIDCursorGenerator<UUIDColumn> CURSOR_GENERATOR = new UUIDCursorGenerator<UUIDColumn>( 1 );
+
public static ScanColumn uuidColumn( UUID value ) {
- return new UUIDColumn( value, 1 );
+ return new UUIDColumn( value, 1 , CURSOR_GENERATOR);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java
new file mode 100644
index 0000000..9079582
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java
@@ -0,0 +1,155 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.SimpleIndexBucketLocatorImpl;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Simple test to test UUID
+ */
+public class SliceShardFilterIteratorTest {
+
+ @Test
+ public void testIndexValues() {
+
+ int size = 100;
+
+ final Multimap<String, ScanColumn> shards = HashMultimap.create();
+
+ final IndexBucketLocator indexBucketLocator = new SimpleIndexBucketLocatorImpl( 20 );
+
+
+ final UUID applicationId = UUIDUtils.newTimeUUID();
+
+ final IndexBucketLocator.IndexType indexType = IndexBucketLocator.IndexType.COLLECTION;
+
+ final String components = "things";
+
+
+ final Set<ScanColumn> allColumns = new LinkedHashSet<ScanColumn>( size );
+
+
+ final UUIDCursorGenerator uuidCursorGenerator = new UUIDCursorGenerator( 1 );
+
+ for ( int i = 0; i < size; i++ ) {
+ final UUID entityId = UUIDUtils.newTimeUUID();
+
+ final String shard = indexBucketLocator.getBucket( applicationId, indexType, entityId, components );
+
+ final UUIDColumn uuidColumn = new UUIDColumn( entityId, 1, uuidCursorGenerator );
+
+ //add the shard to our assertion set
+ shards.put( shard, uuidColumn );
+
+ allColumns.add( uuidColumn );
+ }
+
+ //now create an iterator with all the uuid sand verity they're correct.
+
+
+ for ( final String shard : shards.keySet() ) {
+ //create a copy of our expected uuids
+ final Set<ScanColumn> expected = new HashSet<ScanColumn>( shards.get( shard ) );
+
+
+ final TestIterator testIterator = new TestIterator( new HashSet<ScanColumn>( shards.get( shard ) ) );
+
+
+ final SliceShardFilterIterator.ShardBucketValidator shardBucketValidator =
+ new SliceShardFilterIterator.ShardBucketValidator( indexBucketLocator, shard, applicationId,
+ indexType, components );
+
+
+ //now iterate over everything and remove it from expected
+ final SliceShardFilterIterator sliceShardFilterIterator = new SliceShardFilterIterator( shardBucketValidator, testIterator, 10 );
+
+ //keep removing
+ while(sliceShardFilterIterator.hasNext()){
+
+ //check each scan column from our results
+ for(final ScanColumn column : sliceShardFilterIterator.next()){
+
+ final boolean contained = expected.remove( column );
+
+ assertTrue("Column should be present", contained);
+
+ }
+
+
+ }
+
+ assertTrue("expected should be empty", expected.isEmpty());
+ }
+
+ }
+
+
+ private static final class TestIterator implements ResultIterator {
+
+
+ private final Set<ScanColumn> scanColumns;
+ private boolean completed;
+
+
+ private TestIterator( final Set<ScanColumn> scanColumns ) {this.scanColumns = scanColumns;}
+
+
+ @Override
+ public void reset() {
+ //no op
+ }
+
+
+ @Override
+ public Iterator<Set<ScanColumn>> iterator() {
+ return this;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+ return !completed;
+ }
+
+
+ @Override
+ public Set<ScanColumn> next() {
+ completed = true;
+ return scanColumns;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
index b023d5e..368119c 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
@@ -126,7 +126,7 @@ public class EntityCleanup extends ToolBase {
key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
indexBucketLocator, applicationId, collectionName, false );
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
+ SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser( uuidCursorGenerator ) );
while ( itr.hasNext() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 8375f33..6b669b9 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -181,7 +181,7 @@ public class UniqueIndexCleanup extends ToolBase {
key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
indexBucketLocator, applicationId, collectionName, false );
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
+ SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser( uuidCursorGenerator ) );
while ( itr.hasNext() ) {