You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/06/30 23:13:08 UTC
incubator-usergrid git commit: Finished cursor generation refactor.
Added gets for slice shard filter iterator.
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-752 f087924dd -> efab0b935
Finished cursor generation refactor. Added gets for slice shard filter iterator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/efab0b93
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/efab0b93
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/efab0b93
Branch: refs/heads/USERGRID-752
Commit: efab0b935cfc60b44b7d0c79233367a55611dc84
Parents: f087924
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jun 30 15:13:04 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jun 30 15:13:04 2015 -0600
----------------------------------------------------------------------
.../persistence/cassandra/QueryProcessor.java | 3 +-
.../persistence/query/ir/SearchVisitor.java | 16 +-
.../query/ir/result/AbstractScanColumn.java | 12 +-
.../ir/result/ConnectionIndexSliceParser.java | 16 +-
.../query/ir/result/CursorGenerator.java | 36 +++++
.../query/ir/result/EmptyIterator.java | 5 -
.../query/ir/result/GatherIterator.java | 18 ---
.../query/ir/result/GeoIterator.java | 131 +++++++++-------
.../query/ir/result/IntersectionIterator.java | 35 +++--
.../query/ir/result/OrderByIterator.java | 26 ++--
.../query/ir/result/ResultIterator.java | 4 +-
.../persistence/query/ir/result/ScanColumn.java | 9 ++
.../ir/result/SearchCollectionVisitor.java | 11 +-
.../ir/result/SearchConnectionVisitor.java | 5 +-
.../ir/result/SecondaryIndexSliceParser.java | 13 +-
.../query/ir/result/SliceCursorGenerator.java | 55 +++++++
.../query/ir/result/SliceIterator.java | 57 -------
.../ir/result/SliceShardFilterIterator.java | 5 -
.../query/ir/result/StaticIdIterator.java | 7 +-
.../query/ir/result/SubtractionIterator.java | 20 +--
.../persistence/query/ir/result/UUIDColumn.java | 9 +-
.../query/ir/result/UUIDCursorGenerator.java | 49 ++++++
.../query/ir/result/UUIDIndexSliceParser.java | 9 +-
.../query/ir/result/UnionIterator.java | 51 +++---
.../query/AbstractIteratingQueryIT.java | 3 +-
.../query/ir/result/AbstractScanColumnTest.java | 4 +-
.../query/ir/result/InOrderIterator.java | 9 +-
.../query/ir/result/IteratorHelper.java | 4 +-
.../ir/result/SliceShardFilterIteratorTest.java | 155 +++++++++++++++++++
.../apache/usergrid/tools/EntityCleanup.java | 2 +-
.../usergrid/tools/UniqueIndexCleanup.java | 2 +-
31 files changed, 523 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
index ebaacf4..32fc07c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
@@ -285,8 +285,9 @@ public class QueryProcessor {
int resultSize = Math.min( entityIds.size(), size );
entityIds = entityIds.subList( 0, resultSize );
+ //set our cursor on the last results
if ( resultSize == size ) {
- itr.finalizeCursor( resultsCursor, entityIds.get( resultSize - 1 ).getUUID() );
+ entityIds.get( resultSize -1 ).addToCursor( resultsCursor );
}
}
if ( logger.isDebugEnabled() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
index 271fc67..fdeca6d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/SearchVisitor.java
@@ -33,6 +33,7 @@ import org.apache.usergrid.persistence.query.ir.result.IntersectionIterator;
import org.apache.usergrid.persistence.query.ir.result.OrderByIterator;
import org.apache.usergrid.persistence.query.ir.result.ResultIterator;
import org.apache.usergrid.persistence.query.ir.result.SecondaryIndexSliceParser;
+import org.apache.usergrid.persistence.query.ir.result.SliceCursorGenerator;
import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
import org.apache.usergrid.persistence.query.ir.result.StaticIdIterator;
import org.apache.usergrid.persistence.query.ir.result.SubtractionIterator;
@@ -208,8 +209,11 @@ public abstract class SearchVisitor implements NodeVisitor {
//only order by with no query, start scanning the first field
if ( subResults == null ) {
QuerySlice firstFieldSlice = new QuerySlice( slice.getPropertyName(), -1 );
+
+ final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( firstFieldSlice );
+
subResults =
- new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), new SecondaryIndexSliceParser() );
+ new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), new SecondaryIndexSliceParser( sliceCursorGenerator ) );
}
orderIterator = new OrderByIterator( slice, orderByNode.getSecondarySorts(), subResults, em,
@@ -228,7 +232,10 @@ public abstract class SearchVisitor implements NodeVisitor {
scanner = secondaryIndexScan( orderByNode, slice );
}
- SliceIterator joinSlice = new SliceIterator( slice, scanner, new SecondaryIndexSliceParser());
+ final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( slice );
+
+ SliceIterator joinSlice = new SliceIterator( slice, scanner, new SecondaryIndexSliceParser(
+ sliceCursorGenerator ));
IntersectionIterator union = new IntersectionIterator( queryProcessor.getPageSizeHint( orderByNode ) );
union.addIterator( joinSlice );
@@ -259,7 +266,10 @@ public abstract class SearchVisitor implements NodeVisitor {
for ( QuerySlice slice : node.getAllSlices() ) {
IndexScanner scanner = secondaryIndexScan( node, slice );
- intersections.addIterator( new SliceIterator( slice, scanner, new SecondaryIndexSliceParser()) );
+ final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( slice );
+
+ intersections.addIterator( new SliceIterator( slice, scanner, new SecondaryIndexSliceParser(
+ sliceCursorGenerator )) );
}
results.push( intersections );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
index 2a359be..35672d9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
@@ -22,6 +22,8 @@ import java.util.UUID;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.usergrid.persistence.cassandra.CursorCache;
+
/**
*
@@ -32,12 +34,15 @@ public abstract class AbstractScanColumn implements ScanColumn {
protected final UUID uuid;
protected final ByteBuffer buffer;
+ protected final CursorGenerator sliceCursorGenerator;
protected ScanColumn child;
- protected AbstractScanColumn( final UUID uuid, final ByteBuffer columnNameBuffer ) {
+ protected AbstractScanColumn( final UUID uuid, final ByteBuffer columnNameBuffer,
+ final CursorGenerator sliceCursorGenerator ) {
this.uuid = uuid;
this.buffer = columnNameBuffer;
+ this.sliceCursorGenerator = sliceCursorGenerator;
}
@@ -94,4 +99,9 @@ public abstract class AbstractScanColumn implements ScanColumn {
return child;
}
+
+ @Override
+ public void addToCursor( final CursorCache cache ) {
+ this.sliceCursorGenerator.addToCursor( cache, this );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
index a669d8c..1a3f44d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ConnectionIndexSliceParser.java
@@ -21,7 +21,6 @@ import java.nio.ByteBuffer;
import java.util.UUID;
import org.apache.usergrid.persistence.Schema;
-import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
import com.fasterxml.uuid.UUIDComparator;
@@ -36,11 +35,15 @@ import me.prettyprint.hector.api.beans.DynamicComposite;
public class ConnectionIndexSliceParser implements SliceParser {
private final String connectedEntityType;
+ private final SliceCursorGenerator sliceCurosrGenerator;
- /** @param connectedEntityType Could be null if we want to return all types */
- public ConnectionIndexSliceParser( String connectedEntityType ) {
+ /**
+ * @param connectedEntityType Could be null if we want to return all types
+ * @param sliceCurosrGenerator */
+ public ConnectionIndexSliceParser( String connectedEntityType, final SliceCursorGenerator sliceCurosrGenerator ) {
this.connectedEntityType = connectedEntityType;
+ this.sliceCurosrGenerator = sliceCurosrGenerator;
}
@@ -65,7 +68,7 @@ public class ConnectionIndexSliceParser implements SliceParser {
return null;
}
- return new ConnectionColumn( ( UUID ) composite.get( 0 ), connectedType, buff );
+ return new ConnectionColumn( ( UUID ) composite.get( 0 ), connectedType, buff, sliceCurosrGenerator );
// return composite;
// return null;
}
@@ -78,8 +81,9 @@ public class ConnectionIndexSliceParser implements SliceParser {
private final String connectedType;
- public ConnectionColumn( UUID uuid, String connectedType, ByteBuffer column) {
- super( uuid, column );
+ public ConnectionColumn( UUID uuid, String connectedType, ByteBuffer column,
+ final SliceCursorGenerator sliceCursorGenerator ) {
+ super( uuid, column, sliceCursorGenerator );
this.connectedType = connectedType;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CursorGenerator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CursorGenerator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CursorGenerator.java
new file mode 100644
index 0000000..b128cea
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CursorGenerator.java
@@ -0,0 +1,36 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import org.apache.usergrid.persistence.cassandra.CursorCache;
+
+
+/**
+ * Interface to generate a cursor value from the specified column
+ */
+public interface CursorGenerator<T extends ScanColumn> {
+
+ /**
+ * Use the generator to add this value to the cursor cache
+ * @param cache
+ */
+ void addToCursor( final CursorCache cache, final T column );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/EmptyIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/EmptyIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/EmptyIterator.java
index 933bd2a..d4adbd3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/EmptyIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/EmptyIterator.java
@@ -32,11 +32,6 @@ public class EmptyIterator implements ResultIterator {
}
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastValue ) {
- //no op
- }
-
@Override
public Iterator<Set<ScanColumn>> iterator() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index eaae6b2..9d2e4da 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -31,14 +31,12 @@ public class GatherIterator implements ResultIterator {
private Iterator<ScanColumn> mergedIterators;
- private Map<UUID, ResultIterator> cursorMap;
private List<ResultIterator> iterators;
public GatherIterator(final int pageSize, final QueryNode rootNode, final Collection<SearchVisitor> searchVisitors) {
this.pageSize = pageSize;
this.rootNode = rootNode;
- this.cursorMap = new HashMap<UUID, ResultIterator>( pageSize );
createIterators( searchVisitors );
}
@@ -49,19 +47,6 @@ public class GatherIterator implements ResultIterator {
}
- @Override
- public void finalizeCursor( final CursorCache cache, final UUID lastValue ) {
- //find the last value in the tree, and return it's cursor
- final ResultIterator sourceIterator = cursorMap.get( lastValue );
-
- if(sourceIterator == null){
- throw new IllegalArgumentException( "Could not find the iterator that provided the candidate with uuid " + lastValue );
- }
-
- //delegate to the source iterator
- sourceIterator.finalizeCursor( cache, lastValue );
- }
-
@Override
public Iterator<Set<ScanColumn>> iterator() {
@@ -133,8 +118,6 @@ public class GatherIterator implements ResultIterator {
private void mergeIterators(){
//TODO make this concurrent
- //clear the cursor map
- cursorMap.clear();
TreeSet<ScanColumn> merged = new TreeSet<ScanColumn>( );
@@ -169,7 +152,6 @@ public class GatherIterator implements ResultIterator {
final ScanColumn next = nextPage.next();
results.add(next);
- cursorMap.put( next.getUUID(), iterator );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
index 587bb49..20d1f69 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
@@ -140,11 +140,13 @@ public class GeoIterator implements ResultIterator {
lastCellsSearched = results.lastSearchedGeoCells;
+ final GeoCursorGenerator cursorGenerator = new GeoCursorGenerator( slice, lastCellsSearched );
+
for (final EntityLocationRef location : locations) {
final UUID id = location.getUuid();
- final LocationScanColumn locationScan = new LocationScanColumn(location);
+ final LocationScanColumn locationScan = new LocationScanColumn(location, cursorGenerator );
idOrder.put(id, locationScan);
lastLoaded.add(locationScan);
@@ -212,60 +214,6 @@ public class GeoIterator implements ResultIterator {
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(
- * org.apache.usergrid.persistence.cassandra.CursorCache, java.util.UUID)
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID uuid ) {
-
- LocationScanColumn col = idOrder.get( uuid );
-
- if ( col == null ) {
- throw new IllegalArgumentException( "Could not generate cursor for column because column could not be found" );
- }
-
- final EntityLocationRef location = col.location;
-
- if ( location == null ) {
- return;
- }
-
- final int sliceHash = slice.hashCode();
-
- // get our next distance
- final double latitude = location.getLatitude();
-
- final double longitude = location.getLongitude();
-
- // now create a string value for this
- final StringBuilder builder = new StringBuilder();
-
- builder.append( uuid ).append( DELIM );
- builder.append( latitude ).append( DELIM );
- builder.append( longitude );
-
- if ( lastCellsSearched != null ) {
- builder.append( DELIM );
-
- for ( String geoCell : lastCellsSearched ) {
- builder.append( geoCell ).append( TILE_DELIM );
- }
-
- int length = builder.length();
-
- builder.delete( length - TILE_DELIM.length() - 1, length );
- }
-
- ByteBuffer buff = se.toByteBuffer( builder.toString() );
-
-
- cache.setNextCursor( sliceHash, buff );
- }
-
/** Get the last cells searched in the iteraton */
public List<String> getLastCellsSearched() {
@@ -310,11 +258,14 @@ public class GeoIterator implements ResultIterator {
private class LocationScanColumn implements ScanColumn {
private final EntityLocationRef location;
+ private final GeoCursorGenerator geoCursorGenerator;
private ScanColumn child;
- public LocationScanColumn( EntityLocationRef location ) {
+
+ public LocationScanColumn( EntityLocationRef location, final GeoCursorGenerator geoCursorGenerator ) {
this.location = location;
+ this.geoCursorGenerator = geoCursorGenerator;
}
@@ -338,6 +289,12 @@ public class GeoIterator implements ResultIterator {
@Override
+ public void addToCursor( final CursorCache cache ) {
+
+ }
+
+
+ @Override
public ScanColumn getChild() {
return this.child;
}
@@ -385,4 +342,66 @@ public class GeoIterator implements ResultIterator {
return locationCompare;
}
}
+
+ private static final class GeoCursorGenerator implements CursorGenerator<LocationScanColumn>{
+
+ private final QuerySlice slice;
+ private final List<String> lastCellsSearched;
+
+
+ private GeoCursorGenerator( final QuerySlice slice, final List<String> lastCellsSearched ) {
+ this.slice = slice;
+ this.lastCellsSearched = lastCellsSearched;
+ }
+
+
+ @Override
+ public void addToCursor( final CursorCache cache, final LocationScanColumn col ) {
+
+
+
+ if ( col == null ) {
+ throw new IllegalArgumentException( "Could not generate cursor for column because column could not be found" );
+ }
+
+ final EntityLocationRef location = col.location;
+
+ if ( location == null ) {
+ return;
+ }
+
+ final int sliceHash = slice.hashCode();
+
+ // get our next distance
+ final double latitude = location.getLatitude();
+
+ final double longitude = location.getLongitude();
+
+ // now create a string value for this
+ final StringBuilder builder = new StringBuilder();
+
+ builder.append( col.getUUID() ).append( DELIM );
+ builder.append( latitude ).append( DELIM );
+ builder.append( longitude );
+
+ if ( lastCellsSearched != null ) {
+ builder.append( DELIM );
+
+ for ( String geoCell : lastCellsSearched ) {
+ builder.append( geoCell ).append( TILE_DELIM );
+ }
+
+ int length = builder.length();
+
+ builder.delete( length - TILE_DELIM.length() - 1, length );
+ }
+
+ ByteBuffer buff = se.toByteBuffer( builder.toString() );
+
+
+ cache.setNextCursor( sliceHash, buff );
+
+ }
+ }
}
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/IntersectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/IntersectionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/IntersectionIterator.java
index 1fea546..5f5767f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/IntersectionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/IntersectionIterator.java
@@ -150,21 +150,22 @@ public class IntersectionIterator extends MultiIterator {
return results;
}
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(
- * org.apache.usergrid.persistence.cassandra.CursorCache)
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
- ResultIterator itr = iterators.get( 0 );
-
- //We can only create a cursor on our root level value in the intersection iterator.
- if ( itr != null ) {
- itr.finalizeCursor( cache, lastLoaded );
- }
- }
+ //TODO, replace columns with slice parser here
+
+// /*
+// * (non-Javadoc)
+// *
+// * @see
+// * org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(
+// * org.apache.usergrid.persistence.cassandra.CursorCache)
+// */
+// @Override
+// public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
+// ResultIterator itr = iterators.get( 0 );
+//
+// //We can only create a cursor on our root level value in the intersection iterator.
+// if ( itr != null ) {
+// itr.finalizeCursor( cache, lastLoaded );
+// }
+// }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
index 49847c9..6d05bd6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
@@ -134,19 +134,19 @@ public class OrderByIterator extends MergeIterator {
// no op
}
-
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastValue ) {
- int sliceHash = slice.hashCode();
-
- ByteBuffer bytes = ue.toByteBuffer( lastValue );
-
- if ( bytes == null ) {
- return;
- }
-
- cache.setNextCursor( sliceHash, bytes );
- }
+//
+// @Override
+// public void finalizeCursor( CursorCache cache, UUID lastValue ) {
+// int sliceHash = slice.hashCode();
+//
+// ByteBuffer bytes = ue.toByteBuffer( lastValue );
+//
+// if ( bytes == null ) {
+// return;
+// }
+//
+// cache.setNextCursor( sliceHash, bytes );
+// }
/** A Sorted set with a max size. When a new entry is added, the max is removed */
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ResultIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ResultIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ResultIterator.java
index 01a048c..f62ccf4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ResultIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ResultIterator.java
@@ -37,8 +37,6 @@ public interface ResultIterator extends Iterable<Set<ScanColumn>>, Iterator<Set<
/** Reset this iterator to the start to begin iterating again */
- public void reset();
+ void reset();
- /** Finalize the cursor for this results. Pass in the uuid of the last entity loaded. */
- public void finalizeCursor( CursorCache cache, UUID lastValue );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
index 18277e4..577d34a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.query.ir.result;
import java.nio.ByteBuffer;
import java.util.UUID;
+import org.apache.usergrid.persistence.cassandra.CursorCache;
import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeComparator;
@@ -40,6 +41,14 @@ public interface ScanColumn extends Comparable<ScanColumn> {
*/
void setChild( final ScanColumn childColumn );
+
+ /**
+ * Use the generator to add this value to the cursor cache
+ * @param cache
+ */
+ void addToCursor( final CursorCache cache );
+
+
/**
* Returns the childl column if present, can return null
* @return
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index a3fe116..3dfce2c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -36,8 +36,6 @@ import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtil
public class SearchCollectionVisitor extends SearchVisitor {
- private static final UUIDIndexSliceParser UUID_PARSER = new UUIDIndexSliceParser();
-
private final CollectionInfo collection;
@@ -95,8 +93,13 @@ public class SearchCollectionVisitor extends SearchVisitor {
UUID startId = null;
+ final UUIDCursorGenerator uuidCursorGenerator = new UUIDCursorGenerator( slice.hashCode() );
+ final UUIDIndexSliceParser uuidIndexSliceParser = new UUIDIndexSliceParser( uuidCursorGenerator );
+
+
if ( slice.hasCursor() ) {
- startId = UUID_PARSER.parse( slice.getCursor(), false ).getUUID();
+
+ startId = uuidIndexSliceParser.parse( slice.getCursor(), false ).getUUID();
}
@@ -105,7 +108,7 @@ public class SearchCollectionVisitor extends SearchVisitor {
queryProcessor.getPageSizeHint( node ), query.isReversed(), bucket, applicationId,
node.isForceKeepFirst() );
- this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER ) );
+ this.results.push( new SliceIterator( slice, indexScanner, uuidIndexSliceParser ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 7e7ddf6..3923bf4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -17,7 +17,6 @@ import org.apache.usergrid.persistence.cassandra.index.DynamicCompositeStartToBy
import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
-import org.apache.usergrid.persistence.geo.CollectionGeoSearch;
import org.apache.usergrid.persistence.geo.ConnectionGeoSearch;
import org.apache.usergrid.persistence.geo.model.Point;
import org.apache.usergrid.persistence.query.ir.AllNode;
@@ -171,8 +170,10 @@ public class SearchConnectionVisitor extends SearchVisitor {
final String connectionType = connection.getConnectionType();
+ final SliceCursorGenerator sliceCursorGenerator = new SliceCursorGenerator( slice );
- final ConnectionIndexSliceParser connectionParser = new ConnectionIndexSliceParser( targetType );
+ final ConnectionIndexSliceParser connectionParser = new ConnectionIndexSliceParser( targetType,
+ sliceCursorGenerator );
final Iterator<String> connectionTypes;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
index b26f684..d309c3d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SecondaryIndexSliceParser.java
@@ -45,8 +45,12 @@ public class SecondaryIndexSliceParser implements SliceParser {
//the type comparator
private Comparator<SecondaryIndexColumn> typeComparator;
+ private final SliceCursorGenerator sliceCursorGenerator;
- public SecondaryIndexSliceParser() {}
+
+ public SecondaryIndexSliceParser( final SliceCursorGenerator sliceCursorGenerator ) {
+ this.sliceCursorGenerator = sliceCursorGenerator;
+ }
/* (non-Javadoc)
@@ -63,7 +67,7 @@ public class SecondaryIndexSliceParser implements SliceParser {
typeComparator = getTypeComparator( value, isReversed );
}
- return new SecondaryIndexColumn( uuid, value, buff, typeComparator );
+ return new SecondaryIndexColumn( uuid, value, buff, typeComparator, sliceCursorGenerator );
}
@@ -95,8 +99,9 @@ public class SecondaryIndexSliceParser implements SliceParser {
* @param valueComparator The comparator for the values
*/
public SecondaryIndexColumn( final UUID uuid, final Object value, final ByteBuffer columnNameBuffer,
- final Comparator<SecondaryIndexColumn> valueComparator ) {
- super( uuid, columnNameBuffer );
+ final Comparator<SecondaryIndexColumn> valueComparator,
+ final SliceCursorGenerator sliceCursorGenerator) {
+ super( uuid, columnNameBuffer, sliceCursorGenerator );
this.value = value;
this.valueComparator = valueComparator;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
new file mode 100644
index 0000000..cad0a0f
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
@@ -0,0 +1,55 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.nio.ByteBuffer;
+
+import org.apache.usergrid.persistence.cassandra.CursorCache;
+import org.apache.usergrid.persistence.query.ir.QuerySlice;
+
+
+/**
+ * A cursor generator for the specified query slice
+ */
+public class SliceCursorGenerator implements CursorGenerator<ScanColumn> {
+
+ private final QuerySlice slice;
+
+
+ public SliceCursorGenerator( final QuerySlice slice ) {this.slice = slice;}
+
+
+ @Override
+ public void addToCursor( final CursorCache cache, final ScanColumn col ) {
+
+ if ( col == null ) {
+ return;
+ }
+
+ final int sliceHash = slice.hashCode();
+
+
+ ByteBuffer bytes = col.getCursorValue();
+
+
+ cache.setNextCursor( sliceHash, bytes );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
index 3f9e86c..73cecd7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -44,7 +44,6 @@ public class SliceIterator implements ResultIterator {
private static final Logger logger = LoggerFactory.getLogger( SliceIterator.class );
- private final LinkedHashMap<UUID, ScanColumn> cols;
private final QuerySlice slice;
protected final SliceParser parser;
protected final IndexScanner scanner;
@@ -83,7 +82,6 @@ public class SliceIterator implements ResultIterator {
this.parser = parser;
this.scanner = scanner;
this.pageSize = scanner.getPageSize();
- this.cols = new LinkedHashMap<UUID, ScanColumn>( this.pageSize );
this.parsedCols = new LinkedHashSet<ScanColumn>( this.pageSize );
this.isReversed = scanner.isReversed();
}
@@ -122,8 +120,6 @@ public class SliceIterator implements ResultIterator {
Iterator<HColumn<ByteBuffer, ByteBuffer>> results = scanner.next().iterator();
- cols.clear();
-
parsedCols.clear();
while ( results.hasNext() ) {
@@ -141,7 +137,6 @@ public class SliceIterator implements ResultIterator {
}
last = parsed;
- cols.put( parsed.getUUID(), parsed );
parsedCols.add( parsed );
}
@@ -196,56 +191,4 @@ public class SliceIterator implements ResultIterator {
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor()
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
- final int sliceHash = slice.hashCode();
-
- ByteBuffer bytes = null;
-
- ScanColumn col = cols.get( lastLoaded );
-
-
- //the column came from the current page
- if ( col != null ) {
- bytes = col.getCursorValue();
- }
- else {
-
- //check if we reached the end of our iterator. If we did, set the last value into the cursor. Otherwise
- //this is a bug
- if ( scanner.hasNext() ) {
- logger.error(
- "An iterator attempted to access a slice that was not iterated over. This will result in the"
- + " cursor construction failing" );
- throw new QueryIterationException(
- "An iterator attempted to access a slice that was not iterated over. This will result in the"
- + " cursor construction failing" );
- }
-
- final ByteBuffer sliceCursor = slice.getCursor();
-
- //we've never loaded anything, just re-use the existing slice
- if ( last == null && sliceCursor != null ) {
- bytes = sliceCursor;
- }
-
- //use the last column we loaded. This way our scan returns nothing next time since start == finish
- else if ( last != null ) {
- bytes = last.getCursorValue();
- }
- }
-
-
- if ( bytes == null ) {
- return;
- }
-
- cache.setNextCursor( sliceHash, bytes );
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
index fb0d295..1664627 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIterator.java
@@ -72,11 +72,6 @@ public class SliceShardFilterIterator implements ResultIterator {
}
- @Override
- public void finalizeCursor( final CursorCache cache, final UUID lastValue ) {
- resultsIterator.finalizeCursor( cache, lastValue );
- }
-
@Override
public Iterator<Set<ScanColumn>> iterator() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
index 1ed6949..4c28bf3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/StaticIdIterator.java
@@ -37,7 +37,7 @@ public class StaticIdIterator implements ResultIterator {
*
*/
public StaticIdIterator( UUID id ) {
- final ScanColumn col = new UUIDColumn( id, 1 );
+ final ScanColumn col = new UUIDColumn( id, 1, new UUIDCursorGenerator( -1 ) );
ids = Collections.singleton( col );
}
@@ -49,11 +49,6 @@ public class StaticIdIterator implements ResultIterator {
}
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastValue ) {
- //no cursor, it's a static list
- }
-
@Override
public Iterator<Set<ScanColumn>> iterator() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SubtractionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SubtractionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SubtractionIterator.java
index 9cfcf0a..001ac27 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SubtractionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SubtractionIterator.java
@@ -100,14 +100,14 @@ public class SubtractionIterator extends MergeIterator {
}
- /* (non-Javadoc)
- * @see org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(org.apache.usergrid.persistence.cassandra
- * .CursorCache)
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
- //we can only keep a cursor on our keep result set, we must subtract from every page of keep when loading
- // results
- keepIterator.finalizeCursor( cache, lastLoaded );
- }
+// /* (non-Javadoc)
+// * @see org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(org.apache.usergrid.persistence.cassandra
+// * .CursorCache)
+// */
+// @Override
+// public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
+// //we can only keep a cursor on our keep result set, we must subtract from every page of keep when loading
+// // results
+// keepIterator.finalizeCursor( cache, lastLoaded );
+// }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
index 640b391..264a9b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
@@ -15,14 +15,9 @@ class UUIDColumn extends AbstractScanColumn{
private final int compareReversed;
- protected UUIDColumn( final UUID uuid, final ByteBuffer columnNameBuffer, final int compareReversed ) {
- super( uuid, columnNameBuffer );
- this.compareReversed = compareReversed;
- }
-
- public UUIDColumn( final UUID uuid, final int compareReversed ) {
- super(uuid, Serializers.ue.toByteBuffer( uuid ));
+ public UUIDColumn( final UUID uuid, final int compareReversed, final CursorGenerator<UUIDColumn> sliceCursorGenerator ) {
+ super(uuid, Serializers.ue.toByteBuffer( uuid ), sliceCursorGenerator );
this.compareReversed = compareReversed;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDCursorGenerator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDCursorGenerator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDCursorGenerator.java
new file mode 100644
index 0000000..236db05
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDCursorGenerator.java
@@ -0,0 +1,49 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.cassandra.CursorCache;
+
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+
+
+/**
+ * Cursor generator that takes a uuid column and
+ */
+public class UUIDCursorGenerator<T extends ScanColumn> implements CursorGenerator<T> {
+
+ private final int id;
+
+
+ public UUIDCursorGenerator( final int id ) {this.id = id;}
+
+
+ @Override
+ public void addToCursor( final CursorCache cache, final T column ) {
+
+ final UUID uuid = column.getUUID();
+ final ByteBuffer buff = ue.toByteBuffer( uuid );
+ cache.setNextCursor( id, buff );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
index 2c5b1ba..de80b0b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDIndexSliceParser.java
@@ -29,10 +29,17 @@ import static org.apache.usergrid.persistence.cassandra.Serializers.*;
public class UUIDIndexSliceParser implements SliceParser {
+ private final CursorGenerator<UUIDColumn> uuidCursorGenerator;
+
+
+ public UUIDIndexSliceParser( final UUIDCursorGenerator<UUIDColumn> uuidCursorGenerator ) {
+ this.uuidCursorGenerator = uuidCursorGenerator;
+ }
+
@Override
public ScanColumn parse( final ByteBuffer columnNameBytes, final boolean isReversed ) {
final int compare = isReversed? -1: 1;
- return new UUIDColumn( ue.fromByteBuffer( columnNameBytes.duplicate() ), columnNameBytes, compare );
+ return new UUIDColumn( ue.fromByteBuffer( columnNameBytes.duplicate() ), compare, uuidCursorGenerator);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
index b6e44d8..58e0e9a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -41,8 +41,6 @@ public class UnionIterator extends MultiIterator {
private SortedColumnList list;
- private final int id;
-
/**
* @param pageSize The page size to return
@@ -51,16 +49,17 @@ public class UnionIterator extends MultiIterator {
*/
public UnionIterator( int pageSize, int id, ByteBuffer minUuid ) {
super( pageSize );
-
- this.id = id;
-
UUID parseMinUuid = null;
if(minUuid != null) {
parseMinUuid = ue.fromByteBuffer( minUuid );
}
- list = new SortedColumnList( pageSize, parseMinUuid );
+ final UUIDCursorGenerator uuidCursorGenerator = new UUIDCursorGenerator( id );
+
+ list = new SortedColumnList( pageSize, parseMinUuid, uuidCursorGenerator );
+
+
}
@@ -98,21 +97,6 @@ public class UnionIterator extends MultiIterator {
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(
- * org.apache.usergrid.persistence.cassandra.CursorCache)
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
-
- ByteBuffer buff = ue.toByteBuffer( lastLoaded );
- cache.setNextCursor( id, buff );
- //get our scan column and put them in the cache
- //we finalize the cursor of the min
- }
@Override
@@ -135,16 +119,21 @@ public class UnionIterator extends MultiIterator {
private final List<UnionScanColumn> list;
+ private final UUIDCursorGenerator uuidCursorGenerator;
+
private UnionScanColumn min;
- public SortedColumnList( final int maxSize, final UUID minUuid ) {
+ public SortedColumnList( final int maxSize, final UUID minUuid, final UUIDCursorGenerator uuidCursorGenerator ) {
+
+
+ this.uuidCursorGenerator = uuidCursorGenerator;
//we need to allocate the extra space if required
this.list = new ArrayList<UnionScanColumn>( maxSize );
this.maxSize = maxSize;
if ( minUuid != null ) {
- min = new UnionScanColumn(new UUIDColumn( minUuid, 1 )) ;
+ min = new UnionScanColumn(new UUIDColumn( minUuid, 1, uuidCursorGenerator), uuidCursorGenerator ) ;
}
}
@@ -154,7 +143,7 @@ public class UnionIterator extends MultiIterator {
*/
public void add( ScanColumn col ) {
- final UnionScanColumn unionScanColumn = new UnionScanColumn(col);
+ final UnionScanColumn unionScanColumn = new UnionScanColumn(col, uuidCursorGenerator );
//less than our min, don't add
if ( min != null && min.compareTo( unionScanColumn ) >= 0 ) {
@@ -239,12 +228,16 @@ public class UnionIterator extends MultiIterator {
private static class UnionScanColumn implements ScanColumn{
private final ScanColumn delegate;
+ private final UUIDCursorGenerator uuidCursorGenerator;
private ScanColumn child;
- private UnionScanColumn( final ScanColumn delegate ) {
+
+ private UnionScanColumn( final ScanColumn delegate, final UUIDCursorGenerator uuidCursorGenerator ) {
super();
- this.delegate = delegate;}
+ this.delegate = delegate;
+ this.uuidCursorGenerator = uuidCursorGenerator;
+ }
@Override
@@ -272,6 +265,12 @@ public class UnionIterator extends MultiIterator {
@Override
+ public void addToCursor( final CursorCache cache ) {
+ this.uuidCursorGenerator.addToCursor( cache, this );
+ }
+
+
+ @Override
public ScanColumn getChild() {
return delegate.getChild();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
index f6b2661..1d0d0cb 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
@@ -371,7 +371,8 @@ public abstract class AbstractIteratingQueryIT {
io.doSetup();
- int size = 2000;
+// int size = 2000;
+ int size = 2;
int queryLimit = Query.MAX_LIMIT;
// the number of entities that should be written including an intersection
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
index de46836..c7b3c49 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumnTest.java
@@ -21,6 +21,8 @@ import java.nio.ByteBuffer;
import java.util.UUID;
import org.junit.Test;
+
+import org.apache.usergrid.persistence.query.ir.QuerySlice;
import org.apache.usergrid.utils.UUIDUtils;
import static junit.framework.Assert.assertNull;
@@ -94,7 +96,7 @@ public class AbstractScanColumnTest {
private class TestScanColumn extends AbstractScanColumn {
protected TestScanColumn( final UUID uuid, final ByteBuffer buffer ) {
- super( uuid, buffer );
+ super( uuid, buffer, new SliceCursorGenerator( new QuerySlice( "foo", 1 ) ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
index b4cd906..87ddfe7 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/InOrderIterator.java
@@ -51,7 +51,7 @@ public class InOrderIterator implements ResultIterator {
/** Add a uuid to the list */
public void add( UUID... ids ) {
for ( UUID current : ids ) {
- uuids.add( new UUIDColumn( current, 1 ) );
+ uuids.add( new UUIDColumn( current, 1, new UUIDCursorGenerator<UUIDColumn>( 1 ) ) );
}
}
@@ -120,12 +120,5 @@ public class InOrderIterator implements ResultIterator {
}
- /* (non-Javadoc)
- * @see org.apache.usergrid.persistence.query.ir.result.ResultIterator#finalizeCursor(org.apache.usergrid.persistence.cassandra
- * .CursorCache)
- */
- @Override
- public void finalizeCursor( CursorCache cache, UUID lastLoaded ) {
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
index 13ad542..f75b495 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/IteratorHelper.java
@@ -28,7 +28,9 @@ import java.util.UUID;
*/
public class IteratorHelper {
+ private static final UUIDCursorGenerator<UUIDColumn> CURSOR_GENERATOR = new UUIDCursorGenerator<UUIDColumn>( 1 );
+
public static ScanColumn uuidColumn( UUID value ) {
- return new UUIDColumn( value, 1 );
+ return new UUIDColumn( value, 1 , CURSOR_GENERATOR);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java
new file mode 100644
index 0000000..9079582
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ir/result/SliceShardFilterIteratorTest.java
@@ -0,0 +1,155 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.SimpleIndexBucketLocatorImpl;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Simple test to test UUID
+ */
+public class SliceShardFilterIteratorTest {
+
+ @Test
+ public void testIndexValues() {
+
+ int size = 100;
+
+ final Multimap<String, ScanColumn> shards = HashMultimap.create();
+
+ final IndexBucketLocator indexBucketLocator = new SimpleIndexBucketLocatorImpl( 20 );
+
+
+ final UUID applicationId = UUIDUtils.newTimeUUID();
+
+ final IndexBucketLocator.IndexType indexType = IndexBucketLocator.IndexType.COLLECTION;
+
+ final String components = "things";
+
+
+ final Set<ScanColumn> allColumns = new LinkedHashSet<ScanColumn>( size );
+
+
+ final UUIDCursorGenerator uuidCursorGenerator = new UUIDCursorGenerator( 1 );
+
+ for ( int i = 0; i < size; i++ ) {
+ final UUID entityId = UUIDUtils.newTimeUUID();
+
+ final String shard = indexBucketLocator.getBucket( applicationId, indexType, entityId, components );
+
+ final UUIDColumn uuidColumn = new UUIDColumn( entityId, 1, uuidCursorGenerator );
+
+ //add the shard to our assertion set
+ shards.put( shard, uuidColumn );
+
+ allColumns.add( uuidColumn );
+ }
+
+ //now create an iterator with all the uuid sand verity they're correct.
+
+
+ for ( final String shard : shards.keySet() ) {
+ //create a copy of our expected uuids
+ final Set<ScanColumn> expected = new HashSet<ScanColumn>( shards.get( shard ) );
+
+
+ final TestIterator testIterator = new TestIterator( new HashSet<ScanColumn>( shards.get( shard ) ) );
+
+
+ final SliceShardFilterIterator.ShardBucketValidator shardBucketValidator =
+ new SliceShardFilterIterator.ShardBucketValidator( indexBucketLocator, shard, applicationId,
+ indexType, components );
+
+
+ //now iterate over everything and remove it from expected
+ final SliceShardFilterIterator sliceShardFilterIterator = new SliceShardFilterIterator( shardBucketValidator, testIterator, 10 );
+
+ //keep removing
+ while(sliceShardFilterIterator.hasNext()){
+
+ //check each scan column from our results
+ for(final ScanColumn column : sliceShardFilterIterator.next()){
+
+ final boolean contained = expected.remove( column );
+
+ assertTrue("Column should be present", contained);
+
+ }
+
+
+ }
+
+ assertTrue("expected should be empty", expected.isEmpty());
+ }
+
+ }
+
+
+ private static final class TestIterator implements ResultIterator {
+
+
+ private final Set<ScanColumn> scanColumns;
+ private boolean completed;
+
+
+ private TestIterator( final Set<ScanColumn> scanColumns ) {this.scanColumns = scanColumns;}
+
+
+ @Override
+ public void reset() {
+ //no op
+ }
+
+
+ @Override
+ public Iterator<Set<ScanColumn>> iterator() {
+ return this;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+ return !completed;
+ }
+
+
+ @Override
+ public Set<ScanColumn> next() {
+ completed = true;
+ return scanColumns;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
index b023d5e..368119c 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
@@ -126,7 +126,7 @@ public class EntityCleanup extends ToolBase {
key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
indexBucketLocator, applicationId, collectionName, false );
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
+ SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser( uuidCursorGenerator ) );
while ( itr.hasNext() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/efab0b93/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 8375f33..6b669b9 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -181,7 +181,7 @@ public class UniqueIndexCleanup extends ToolBase {
key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
indexBucketLocator, applicationId, collectionName, false );
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
+ SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser( uuidCursorGenerator ) );
while ( itr.hasNext() ) {